From 39a6e83a22e184c3a4fad080ef9fe496d0dc8a0a Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 4 Apr 2017 21:10:34 -0600 Subject: [PATCH 001/229] Merging transfer strategy stuff with main. --- .../ds3client/integration/Util.java | 8 + .../com/spectralogic/ds3client/IntValue.java | 28 - .../ds3client/helpers/JobImpl_Test.java | 728 -------------- .../integration/DataIntegrity_Test.java | 2 + .../integration/GetJobManagement_Test.java | 794 ++++++++++++++- .../integration/PutJobManagement_Test.java | 748 +++++++++++++- .../ds3client/integration/Smoke_Test.java | 185 ++++ .../test/helpers/Ds3ClientShim.java | 22 +- .../Ds3ClientShimWithFailedGetObject.java | 1 - .../samples/BulkPutWithMetadata.java | 2 +- .../ds3client/helpers/AutoCloseableCache.java | 71 -- .../ds3client/helpers/ChunkTransferrer.java | 118 --- .../ds3client/helpers/Ds3ClientHelpers.java | 141 ++- .../helpers/Ds3ClientHelpersImpl.java | 394 +++++--- .../ds3client/helpers/FileObjectGetter.java | 3 +- .../ds3client/helpers/JobImpl.java | 217 +--- .../ds3client/helpers/JobPart.java | 3 + .../helpers/JobPartTrackerFactory.java | 2 +- .../ds3client/helpers/JobState.java | 61 +- .../ds3client/helpers/ReadJobImpl.java | 270 +---- .../ds3client/helpers/WriteJobImpl.java | 228 +---- .../helpers/events/FailureEvent.java | 114 +++ .../helpers/events/MetadataEvent.java | 36 + .../helpers/strategy/BlobStrategy.java | 76 -- .../helpers/strategy/StrategyUtils.java | 42 +- .../blobstrategy/AbstractBlobStrategy.java | 78 ++ ...ckPearlChunkAttemptRetryDelayBehavior.java | 43 + .../strategy/blobstrategy/BlobStrategy.java | 35 + .../blobstrategy/BlobStrategyMaker.java | 30 + .../ChunkAttemptRetryBehavior.java | 35 + .../ChunkAttemptRetryDelayBehavior.java | 29 + ...DefinedChunkAttemptRetryDelayBehavior.java | 51 + ...inueForeverChunkAttemptsRetryBehavior.java | 34 + .../GetSequentialBlobStrategy.java} | 73 +- .../MaxChunkAttemptsRetryBehavior.java | 31 + .../PutSequentialBlobStrategy.java} | 70 +- .../channelstrategy/ChannelPreparable.java | 36 + .../channelstrategy/ChannelStrategy.java | 46 + .../NullChannelPreparable.java | 40 + .../RandomAccessChannelStrategy.java | 139 +++ .../SeekableByteChannelDecorator.java | 95 ++ .../SequentialChannelStrategy.java | 118 +++ .../SequentialFileReaderChannelStrategy.java | 68 ++ .../SequentialFileWriterChannelStrategy.java | 69 ++ .../TruncatingChannelPreparable.java | 46 + .../transferstrategy/AbstractObserver.java | 36 + .../AbstractTransferStrategy.java | 192 ++++ .../BlobTransferredEventObserver.java | 31 + .../transferstrategy/ChecksumEvent.java | 46 + .../transferstrategy/ChecksumObserver.java | 51 + ...ContinueForeverTransferRetryDecorator.java | 46 + .../DataTransferredObserver.java | 41 + .../transferstrategy/EventDispatcher.java | 138 +++ .../transferstrategy/EventDispatcherImpl.java | 319 ++++++ .../FailureEventObserver.java | 52 + .../GetJobNetworkFailureRetryDecorator.java | 171 ++++ .../GetJobPartialBlobTransferMethod.java | 84 ++ .../GetJobTransferMethod.java | 97 ++ ...MaxNumObjectTransferAttemptsDecorator.java | 68 ++ .../MetaDataReceivedObserver.java | 71 ++ .../MultiThreadedTransferStrategy.java | 44 + .../ObjectCompletedObserver.java | 51 + .../strategy/transferstrategy/Observer.java | 24 + .../PutJobTransferMethod.java | 134 +++ .../transferstrategy}/RangeHelper.java | 23 +- .../SingleThreadedTransferStrategy.java | 43 + .../transferstrategy/TransferMethod.java | 32 + .../transferstrategy/TransferMethodMaker.java | 24 + .../TransferRetryDecorator.java | 20 + .../transferstrategy/TransferStrategy.java | 30 + .../TransferStrategyBuilder.java | 941 ++++++++++++++++++ .../transferstrategy/UpdateStrategy.java | 23 + .../WaitingForChunksObserver.java | 41 + .../utils/hashing/ChecksumUtils.java | 61 ++ .../helpers/AutoCloseableCache_Test.java | 148 --- .../RangeHelperImpl_Test.java | 2 +- 76 files changed, 6354 insertions(+), 2190 deletions(-) delete mode 100644 ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/IntValue.java delete mode 100644 ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/JobImpl_Test.java delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/AutoCloseableCache.java delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferrer.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/MetadataEvent.java delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/BlobStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/AbstractBlobStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlackPearlChunkAttemptRetryDelayBehavior.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategyMaker.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryBehavior.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryDelayBehavior.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ClientDefinedChunkAttemptRetryDelayBehavior.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ContinueForeverChunkAttemptsRetryBehavior.java rename ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/{GetStreamerStrategy.java => blobstrategy/GetSequentialBlobStrategy.java} (79%) create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/MaxChunkAttemptsRetryBehavior.java rename ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/{PutStreamerStrategy.java => blobstrategy/PutSequentialBlobStrategy.java} (71%) create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelPreparable.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/NullChannelPreparable.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/RandomAccessChannelStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileReaderChannelStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileWriterChannelStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/TruncatingChannelPreparable.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractObserver.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/BlobTransferredEventObserver.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumEvent.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ContinueForeverTransferRetryDecorator.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MaxNumObjectTransferAttemptsDecorator.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MetaDataReceivedObserver.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/Observer.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java rename ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/{ => strategy/transferstrategy}/RangeHelper.java (74%) create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethod.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethodMaker.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferRetryDecorator.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/UpdateStrategy.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/WaitingForChunksObserver.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/hashing/ChecksumUtils.java delete mode 100644 ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/AutoCloseableCache_Test.java rename ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/{ => strategy/transferstrategy}/RangeHelperImpl_Test.java (99%) diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java index a28570dd7..166f73532 100644 --- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java +++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java @@ -113,4 +113,12 @@ public static void deleteAllContents(final Ds3Client client, final String bucket client.deleteBucket(new DeleteBucketRequest(bucketName)); } + public static void deleteBucketContents(final Ds3Client client, final String bucketName) throws IOException { + final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); + + final Iterable objects = helpers.listObjects(bucketName); + for(final Contents contents : objects) { + client.deleteObject(new DeleteObjectRequest(bucketName, contents.getKey())); + } + } } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/IntValue.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/IntValue.java deleted file mode 100644 index 8a79b14ef..000000000 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/IntValue.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * ****************************************************************************** - * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client; - -public final class IntValue { - private int intValue = 0; - - public int increment() { - return ++intValue; - } - - public int getValue() { - return intValue; - } -} diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/JobImpl_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/JobImpl_Test.java deleted file mode 100644 index 3db21e966..000000000 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/JobImpl_Test.java +++ /dev/null @@ -1,728 +0,0 @@ -/* - * ****************************************************************************** - * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.helpers; - -import com.spectralogic.ds3client.Ds3Client; -import com.spectralogic.ds3client.Ds3ClientImpl; -import com.spectralogic.ds3client.commands.DeleteObjectRequest; -import com.spectralogic.ds3client.integration.Util; -import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShim; -import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds; -import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil; -import com.spectralogic.ds3client.models.ChecksumType; -import com.spectralogic.ds3client.models.Contents; -import com.spectralogic.ds3client.models.bulk.Ds3Object; -import com.spectralogic.ds3client.utils.ResourceUtils; -import com.spectralogic.ds3client.IntValue; - -import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Before; -import org.junit.After; -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import static com.spectralogic.ds3client.integration.Util.deleteAllContents; - -public class JobImpl_Test { - private static final Logger LOG = LoggerFactory.getLogger(JobImpl_Test.class); - - private static final Ds3Client client = Util.fromEnv(); - private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client); - private static final String BUCKET_NAME = "JobImpl_Test"; - private static final String TEST_ENV_NAME = "JobImpl_Test"; - private static TempStorageIds envStorageIds; - private static UUID dataPolicyId; - - private static final String DIR_NAME = "largeFiles/"; - private static final String[] FILE_NAMES = new String[]{"lesmis-copies.txt"}; - - private long bookSize = -1; - private final List bookTitles = new ArrayList<>(); - private final List objects = new ArrayList<>(); - - private static Path dirPath; - - @BeforeClass - public static void startup() throws Exception { - dataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, client); - envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, dataPolicyId, client); - setupBucket(dataPolicyId); - dirPath = ResourceUtils.loadFileResource(DIR_NAME); - } - - @AfterClass - public static void teardown() throws IOException { - try { - deleteAllContents(client, BUCKET_NAME); - } finally { - TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, client); - client.close(); - } - } - - @Before - public void beforeRunningTestMethod() { - try { - for (final String book : FILE_NAMES) { - final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); - bookSize = Files.size(objPath); - final Ds3Object obj = new Ds3Object(book, bookSize); - bookTitles.add(book); - objects.add(obj); - } - } catch (final Throwable t) { - fail("Error running beforeRunningTestMethod: " + t.getMessage()); - } - } - - @After - public void afterRunningTestMethod() { - bookTitles.clear(); - objects.clear(); - - try { - deleteBigFileFromBlackPearlBucket(); - } catch (final Throwable t) { - fail("Error deleting file from black pearl: " + t.getMessage()); - } - } - - private void deleteBigFileFromBlackPearlBucket() throws IOException { - final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); - - final Iterable objects = helpers.listObjects(BUCKET_NAME); - for (final Contents contents : objects) { - if (contents.getKey().equals(FILE_NAMES[0])) { - client.deleteObject(new DeleteObjectRequest(BUCKET_NAME, contents.getKey())); - } - } - } - - /** - * Creates the test bucket with the specified data policy to prevent cascading test failure - * when there are multiple data policies - */ - private static void setupBucket(final UUID dataPolicy) { - try { - HELPERS.ensureBucketExists(BUCKET_NAME, dataPolicy); - } catch (final Exception e) { - LOG.error("Setting up test environment failed: " + e.getMessage()); - } - } - - /** - * This test verifies that object completed callback handler are added to the correct set. - * There are 2 sets: 1 for events we create to let us know when to close channels we have opened; - * and 1 for events clients have registered. It's important that we fire the events that close - * channels before calling client events, so that clients can rely on the channels having been - * closed. - * @throws IOException - * @throws URISyntaxException - * @throws NoSuchMethodException - * @throws IllegalAccessException - * @throws InvocationTargetException - * @throws NoSuchFieldException - * @throws ClassNotFoundException - */ - @Test - public void testWriteObjectCompletionEventsPopulateCorrectJobPartTracker() - throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, NoSuchFieldException, ClassNotFoundException - { - final IntValue intValue = new IntValue(); - - - - final ObjectCompletedListener objectCompletedListener = new ObjectCompletedListener() { - private int numCompletedObjects = 0; - - @Override - public void objectCompleted(final String name) { - intValue.increment(); - assertTrue(bookTitles.contains(name)); - assertEquals(1, ++numCompletedObjects); - } - }; - - // This is used just to make sure the callback handler ends up in the right place. - final DataTransferredListener dataTransferredListener = new DataTransferredListener() { - @Override - public void dataTransferred(final long size) { - - } - }; - - final JobImplJobPartDecoratorPair jobImplJobPartDecoratorPair = createJobPartDecoratorAndEnsureObjectPartTrackersPopulated( - objectCompletedListener, - dataTransferredListener, - AccessMode.ForWriting); - - // trigger the callback - - jobImplJobPartDecoratorPair.getJobPartTrackerDecorator().completePart(FILE_NAMES[0], new ObjectPart(0, bookSize)); - - assertEquals(1, intValue.getValue()); - } - - private enum AccessMode { - ForReading, - ForWriting - } - - private JobImplJobPartDecoratorPair createJobPartDecoratorAndEnsureObjectPartTrackersPopulated - ( - final ObjectCompletedListener objectCompletedListener, - final DataTransferredListener dataTransferredListener, - final AccessMode accessMode - ) - throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, NoSuchFieldException, ClassNotFoundException - { - final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client); - - final int maxNumBlockAllocationRetries = 1; - final int maxNumObjectTransferAttempts = 3; - final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim, - maxNumBlockAllocationRetries, - maxNumObjectTransferAttempts); - - final Ds3ClientHelpers.Job job; - - if (accessMode == AccessMode.ForWriting) { - job = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects); - } else { - job = ds3ClientHelpers.startReadJob(BUCKET_NAME, objects); - } - - job.attachObjectCompletedListener(objectCompletedListener); - - job.attachDataTransferredListener(dataTransferredListener); - return getJobImplJobPartDecoratorAndEnsureObjectPartTrackersPopulated(job); - } - - private JobImplJobPartDecoratorPair getJobImplJobPartDecoratorAndEnsureObjectPartTrackersPopulated(final Ds3ClientHelpers.Job job) - throws NoSuchFieldException, IllegalAccessException - { - // Check that the client has one callback registered - final Field partTrackerField = job.getClass().getSuperclass().getDeclaredField("jobPartTracker"); - partTrackerField.setAccessible(true); - final JobImpl.JobPartTrackerDecorator jobPartTrackerDecorator = (JobImpl.JobPartTrackerDecorator) partTrackerField.get(job); - - final Field clientJobPartTrackerField = jobPartTrackerDecorator.getClass().getDeclaredField("clientJobPartTracker"); - clientJobPartTrackerField.setAccessible(true); - final JobPartTrackerImpl clientJobPartTrackerImpl = (JobPartTrackerImpl) clientJobPartTrackerField.get(jobPartTrackerDecorator); - - final Field clientTackersField = clientJobPartTrackerImpl.getClass().getDeclaredField("trackers"); - clientTackersField.setAccessible(true); - final Map clientTrackers = (Map) clientTackersField.get(clientJobPartTrackerImpl); - final ObjectPartTrackerImpl clientObjectPartTrackerImpl = (ObjectPartTrackerImpl) clientTrackers.get(FILE_NAMES[0]); - - final Field clientObjectCompletedListenersField = clientObjectPartTrackerImpl.getClass().getDeclaredField("objectCompletedListeners"); - clientObjectCompletedListenersField.setAccessible(true); - final Set clientObjectCompletedListeners = (Set) clientObjectCompletedListenersField.get(clientObjectPartTrackerImpl); - - assertEquals(1, clientObjectCompletedListeners.size()); - - // Data transfer listeners - final Field clientDataTransferListenersField = clientObjectPartTrackerImpl.getClass().getDeclaredField("dataTransferredListeners"); - clientDataTransferListenersField.setAccessible(true); - final Set clientDataTransferredListeners = (Set) clientDataTransferListenersField.get(clientObjectPartTrackerImpl); - - assertEquals(1, clientDataTransferredListeners.size()); - - // Check that we have no internal callbacks registered. Registering internal callbacks doesn't happen until you - // call transfer - - final Field internalJobPartTrackerField = jobPartTrackerDecorator.getClass().getDeclaredField("internalJobPartTracker"); - internalJobPartTrackerField.setAccessible(true); - final JobPartTrackerImpl internalJobPartTrackerImpl = (JobPartTrackerImpl) internalJobPartTrackerField.get(jobPartTrackerDecorator); - - final Field internalTackersField = internalJobPartTrackerImpl.getClass().getDeclaredField("trackers"); - internalTackersField.setAccessible(true); - final Map internalTrackers = (Map) internalTackersField.get(internalJobPartTrackerImpl); - final ObjectPartTrackerImpl internalObjectPartTrackerImpl = (ObjectPartTrackerImpl) internalTrackers.get(FILE_NAMES[0]); - - final Field internalObjectCompletedListenersField = internalObjectPartTrackerImpl.getClass().getDeclaredField("objectCompletedListeners"); - internalObjectCompletedListenersField.setAccessible(true); - final Set internalObjectCompletedListeners = (Set) internalObjectCompletedListenersField.get(internalObjectPartTrackerImpl); - - assertEquals(0, internalObjectCompletedListeners.size()); - - // Data transfer listeners - final Field internalDataTransferListenersField = internalObjectPartTrackerImpl.getClass().getDeclaredField("dataTransferredListeners"); - internalDataTransferListenersField.setAccessible(true); - final Set internalDataTransferredListeners = (Set) internalDataTransferListenersField.get(internalObjectPartTrackerImpl); - - assertEquals(0, internalDataTransferredListeners.size()); - - return new JobImplJobPartDecoratorPair(jobPartTrackerDecorator, (JobImpl) job); - } - - private final class JobImplJobPartDecoratorPair { - private final JobImpl.JobPartTrackerDecorator jobPartTrackerDecorator; - private final JobImpl job; - - private JobImplJobPartDecoratorPair(final JobImpl.JobPartTrackerDecorator jobPartTrackerDecorator, - final JobImpl job) { - this.jobPartTrackerDecorator = jobPartTrackerDecorator; - this.job = job; - } - - public JobImpl.JobPartTrackerDecorator getJobPartTrackerDecorator() { - return jobPartTrackerDecorator; - } - - public JobImpl getJob() { - return job; - } - } - - - - /** - * This test verifies that there is 1 object completed callback handler registered in the internal job part - * tracker. This callback handler does not exist until you call transfer on the write job. - * @throws NoSuchMethodException - * @throws IOException - * @throws ClassNotFoundException - * @throws URISyntaxException - * @throws IllegalAccessException - * @throws InvocationTargetException - * @throws NoSuchFieldException - */ - @Test - public void testWriteObjectCompletionPopulatesInternalJobPartTrackers() - throws NoSuchMethodException, IOException, ClassNotFoundException, URISyntaxException, IllegalAccessException, InvocationTargetException, NoSuchFieldException - { - final IntValue intValue = new IntValue(); - - final ObjectCompletedListener objectCompletedListener = new ObjectCompletedListener() { - private int numCompletedObjects = 0; - - @Override - public void objectCompleted(final String name) { - intValue.increment(); - assertTrue(bookTitles.contains(name)); - assertEquals(1, ++numCompletedObjects); - } - }; - - final DataTransferredListener dataTransferredListener = new DataTransferredListener() { - @Override - public void dataTransferred(final long size) { - - } - }; - - final JobImplJobPartDecoratorPair jobImplJobPartDecoratorPair = createJobPartDecoratorAndEnsureObjectPartTrackersPopulated( - objectCompletedListener, - dataTransferredListener, - AccessMode.ForWriting); - - jobImplJobPartDecoratorPair.getJob().transfer(new FileObjectPutter(dirPath)); - - final Field partTrackerField = jobImplJobPartDecoratorPair.getJob().getClass().getSuperclass().getDeclaredField("jobPartTracker"); - partTrackerField.setAccessible(true); - final JobImpl.JobPartTrackerDecorator jobPartTrackerDecorator = (JobImpl.JobPartTrackerDecorator)partTrackerField.get(jobImplJobPartDecoratorPair.getJob()); - - final Field internalJobPartTrackerField = jobPartTrackerDecorator.getClass().getDeclaredField("internalJobPartTracker"); - internalJobPartTrackerField.setAccessible(true); - final JobPartTrackerImpl internalJobPartTrackerImpl = (JobPartTrackerImpl)internalJobPartTrackerField.get(jobPartTrackerDecorator); - - final Field internalTackersField = internalJobPartTrackerImpl.getClass().getDeclaredField("trackers"); - internalTackersField.setAccessible(true); - final Map internalTrackers = (Map)internalTackersField.get(internalJobPartTrackerImpl); - final ObjectPartTrackerImpl internalObjectPartTrackerImpl = (ObjectPartTrackerImpl)internalTrackers.get(FILE_NAMES[0]); - - final Field internalObjectCompletedListenersField = internalObjectPartTrackerImpl.getClass().getDeclaredField("objectCompletedListeners"); - internalObjectCompletedListenersField.setAccessible(true); - final Set internalObjectCompletedListeners = (Set)internalObjectCompletedListenersField.get(internalObjectPartTrackerImpl); - - assertEquals(1, internalObjectCompletedListeners.size()); - } - - @Test - public void testWriteObjectCompletionFiresInternalHandlersFirst() - throws NoSuchMethodException, IOException, ClassNotFoundException, URISyntaxException, IllegalAccessException, InvocationTargetException, NoSuchFieldException { - final IntValue intValue = new IntValue(); - - final ObjectCompletedListener objectCompletedListener = new ObjectCompletedListener() { - private int numCompletedObjects = 0; - - @Override - public void objectCompleted(final String name) { - intValue.increment(); - assertTrue(bookTitles.contains(name)); - assertEquals(1, ++numCompletedObjects); - } - }; - - final DataTransferredListener dataTransferredListener = new DataTransferredListener() { - @Override - public void dataTransferred(final long size) { - - } - }; - - final JobImplJobPartDecoratorPair jobImplJobPartDecoratorPair = createJobPartDecoratorAndEnsureObjectPartTrackersPopulated( - objectCompletedListener, - dataTransferredListener, - AccessMode.ForWriting); - - final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); - - final JobPartDecoratorInterceptor jobPartDecoratorInterceptor = new JobPartDecoratorInterceptor(jobImplJobPartDecoratorPair); - jobPartDecoratorInterceptor.transfer(new FileObjectPutter(dirPath)); - assertTrue(jobPartDecoratorInterceptor.bothHandlersFired()); - } - - private static final class JobPartDecoratorInterceptor implements JobPartTracker { - private final JobImplJobPartDecoratorPair jobImplJobPartDecoratorPair; - private final ObjectCompletedCallbackTracker objectCompletedCallbackTracker; - - private JobPartDecoratorInterceptor(final JobImplJobPartDecoratorPair jobImplJobPartDecoratorPair) - throws NoSuchFieldException, IllegalAccessException - { - this.jobImplJobPartDecoratorPair = jobImplJobPartDecoratorPair; - this.objectCompletedCallbackTracker = new ObjectCompletedCallbackTracker(); - - // Replace the job part trackers in JobPartTrackerDecorator with JobPartTrackerInterceptor - final JobImpl.JobPartTrackerDecorator jobPartTrackerDecorator = jobImplJobPartDecoratorPair.getJobPartTrackerDecorator(); - - final Field internalJobPartTrackerField = jobPartTrackerDecorator.getClass().getDeclaredField("internalJobPartTracker"); - internalJobPartTrackerField.setAccessible(true); - final JobPartTrackerInterceptor internalJobPartTrackerInterceptor = new JobPartTrackerInterceptor( - (JobPartTracker)internalJobPartTrackerField.get(jobPartTrackerDecorator), - objectCompletedCallbackTracker.getInternalObjectCompletedHandler()); - internalJobPartTrackerField.set(jobPartTrackerDecorator, internalJobPartTrackerInterceptor); - - final Field clientJobPartTrackerField = jobPartTrackerDecorator.getClass().getDeclaredField("clientJobPartTracker"); - clientJobPartTrackerField.setAccessible(true); - final JobPartTrackerInterceptor clientJobPartTrackerInterceptor = new JobPartTrackerInterceptor( - (JobPartTracker)clientJobPartTrackerField.get(jobPartTrackerDecorator), - objectCompletedCallbackTracker.getClientObjectCompletedHandler()); - clientJobPartTrackerField.set(jobPartTrackerDecorator, clientJobPartTrackerInterceptor); - } - - @Override - public void completePart(final String key, final ObjectPart objectPart) { - jobImplJobPartDecoratorPair.getJobPartTrackerDecorator().completePart(key, objectPart); - } - - @Override - public boolean containsPart(final String key, final ObjectPart objectPart) { - return jobImplJobPartDecoratorPair.getJobPartTrackerDecorator().containsPart(key, objectPart); - } - - @Override - public JobPartTracker attachDataTransferredListener(final DataTransferredListener listener) { - return jobImplJobPartDecoratorPair.getJobPartTrackerDecorator().attachDataTransferredListener(listener); - } - - @Override - public JobPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) { - return jobImplJobPartDecoratorPair.getJobPartTrackerDecorator().attachObjectCompletedListener(listener); - } - - @Override - public void removeDataTransferredListener(final DataTransferredListener listener) { - jobImplJobPartDecoratorPair.getJobPartTrackerDecorator().removeDataTransferredListener(listener); - } - - @Override - public void removeObjectCompletedListener(final ObjectCompletedListener listener) { - jobImplJobPartDecoratorPair.getJobPartTrackerDecorator().removeObjectCompletedListener(listener); - } - - private void transfer(final Ds3ClientHelpers.ObjectChannelBuilder channelBuilder) throws IOException { - jobImplJobPartDecoratorPair.getJob().transfer(channelBuilder); - } - - private boolean bothHandlersFired() { - return objectCompletedCallbackTracker.bothHandlersFired(); - } - } - - private static class ObjectCompletedCallbackTracker { - private final BooleanValue internalObjectCompletedHandlerCalled = new BooleanValue(); - private final BooleanValue clientObjectCompletedHandlerCalled = new BooleanValue(); - - private final ObjectCompletedCallbackRecorder clientObjectCompletedHandler = new ObjectCompletedCallbackRecorder() { - @Override - public void onObjectCompletedCalled() { - clientObjectCompletedHandlerCalled.setValue(true); - - if ( ! internalObjectCompletedHandlerCalled.getValue()) { - throw new IllegalStateException("Client object completed handler called before closing channels."); - } - } - }; - - private final ObjectCompletedCallbackRecorder internalObjectCompletedHandler = new ObjectCompletedCallbackRecorder() { - @Override - public void onObjectCompletedCalled() { - internalObjectCompletedHandlerCalled.setValue(true); - - if (clientObjectCompletedHandlerCalled.getValue()) { - throw new IllegalStateException("Client object completed handler called before closing channels."); - } - } - }; - - public ObjectCompletedCallbackRecorder getClientObjectCompletedHandler() { - return clientObjectCompletedHandler; - } - - public ObjectCompletedCallbackRecorder getInternalObjectCompletedHandler() { - return internalObjectCompletedHandler; - } - - public boolean bothHandlersFired() { - return internalObjectCompletedHandlerCalled.getValue() && clientObjectCompletedHandlerCalled.getValue(); - } - } - - private static class BooleanValue { - private boolean value = false; - - private boolean getValue() { - return this.value; - } - - private boolean setValue(final boolean value) { - this.value = value; - return getValue(); - } - } - - private interface ObjectCompletedCallbackRecorder { - void onObjectCompletedCalled(); - } - - private static final class JobPartTrackerInterceptor implements JobPartTracker { - private final JobPartTracker jobPartTracker; - private final ObjectCompletedCallbackRecorder objectCompletedCallbackRecorder; - - private JobPartTrackerInterceptor(final JobPartTracker jobPartTracker, final ObjectCompletedCallbackRecorder objectCompletedCallbackRecorder) { - this.jobPartTracker = jobPartTracker; - this.objectCompletedCallbackRecorder = objectCompletedCallbackRecorder; - } - - @Override - public void completePart(final String key, final ObjectPart objectPart) { - objectCompletedCallbackRecorder.onObjectCompletedCalled(); - jobPartTracker.completePart(key, objectPart); - } - - @Override - public boolean containsPart(final String key, final ObjectPart objectPart) { - return jobPartTracker.containsPart(key, objectPart); - } - - @Override - public JobPartTracker attachDataTransferredListener(final DataTransferredListener listener) { - return jobPartTracker.attachDataTransferredListener(listener); - } - - @Override - public JobPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) { - return jobPartTracker.attachObjectCompletedListener(listener); - } - - @Override - public void removeDataTransferredListener(final DataTransferredListener listener) { - jobPartTracker.removeDataTransferredListener(listener); - } - - @Override - public void removeObjectCompletedListener(final ObjectCompletedListener listener) { - jobPartTracker.removeObjectCompletedListener(listener); - } - } - - @Test - public void testWriteObjectRemovesEventsFromCorrectPartTracker() - throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, NoSuchFieldException, ClassNotFoundException - { - final IntValue intValue = new IntValue(); - - final ObjectCompletedListener objectCompletedListener = new ObjectCompletedListener() { - private int numCompletedObjects = 0; - - @Override - public void objectCompleted(final String name) { - intValue.increment(); - assertTrue(bookTitles.contains(name)); - assertEquals(1, ++numCompletedObjects); - } - }; - - // This is used just to make sure the callback handler ends up in the right place. - final DataTransferredListener dataTransferredListener = new DataTransferredListener() { - @Override - public void dataTransferred(final long size) { - - } - }; - - final JobImplJobPartDecoratorPair jobImplJobPartDecoratorPair = createJobPartDecoratorAndEnsureObjectPartTrackersPopulated( - objectCompletedListener, - dataTransferredListener, - AccessMode.ForWriting); - - jobImplJobPartDecoratorPair.getJob().removeObjectCompletedListener(objectCompletedListener); - - final JobImpl job = jobImplJobPartDecoratorPair.getJob(); - - final Field partTrackerField = job.getClass().getSuperclass().getDeclaredField("jobPartTracker"); - partTrackerField.setAccessible(true); - final JobImpl.JobPartTrackerDecorator jobPartTrackerDecorator = (JobImpl.JobPartTrackerDecorator) partTrackerField.get(job); - - final Field clientJobPartTrackerField = jobPartTrackerDecorator.getClass().getDeclaredField("clientJobPartTracker"); - clientJobPartTrackerField.setAccessible(true); - final JobPartTrackerImpl clientJobPartTrackerImpl = (JobPartTrackerImpl) clientJobPartTrackerField.get(jobPartTrackerDecorator); - - final Field clientTackersField = clientJobPartTrackerImpl.getClass().getDeclaredField("trackers"); - clientTackersField.setAccessible(true); - final Map clientTrackers = (Map) clientTackersField.get(clientJobPartTrackerImpl); - final ObjectPartTrackerImpl clientObjectPartTrackerImpl = (ObjectPartTrackerImpl) clientTrackers.get(FILE_NAMES[0]); - - final Field clientObjectCompletedListenersField = clientObjectPartTrackerImpl.getClass().getDeclaredField("objectCompletedListeners"); - clientObjectCompletedListenersField.setAccessible(true); - final Set clientObjectCompletedListeners = (Set) clientObjectCompletedListenersField.get(clientObjectPartTrackerImpl); - - assertEquals(0, clientObjectCompletedListeners.size()); - - // Data transfer listeners - job.removeDataTransferredListener(dataTransferredListener); - final Field clientDataTransferListenersField = clientObjectPartTrackerImpl.getClass().getDeclaredField("dataTransferredListeners"); - clientDataTransferListenersField.setAccessible(true); - final Set clientDataTransferredListeners = (Set) clientDataTransferListenersField.get(clientObjectPartTrackerImpl); - - assertEquals(0, clientDataTransferredListeners.size()); - } - - @Test - public void testReadObjectCompletionEventsPopulateCorrectJobPartTracker() - throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, NoSuchFieldException, ClassNotFoundException - { - putBigFileIntoBlackPearl(); - - final IntValue intValue = new IntValue(); - - final ObjectCompletedListener objectCompletedListener = new ObjectCompletedListener() { - private int numCompletedObjects = 0; - - @Override - public void objectCompleted(final String name) { - intValue.increment(); - assertTrue(bookTitles.contains(name)); - assertEquals(1, ++numCompletedObjects); - } - }; - - // This is used just to make sure the callback handler ends up in the right place. - final DataTransferredListener dataTransferredListener = new DataTransferredListener() { - @Override - public void dataTransferred(final long size) { - - } - }; - - final JobImplJobPartDecoratorPair jobImplJobPartDecoratorPair = createJobPartDecoratorAndEnsureObjectPartTrackersPopulated( - objectCompletedListener, - dataTransferredListener, - AccessMode.ForReading); - - // trigger the callback - - jobImplJobPartDecoratorPair.getJobPartTrackerDecorator().completePart(FILE_NAMES[0], new ObjectPart(0, bookSize)); - - assertEquals(1, intValue.getValue()); - } - - private void putBigFileIntoBlackPearl() throws IOException, URISyntaxException { - final int maxNumBlockAllocationRetries = 1; - final int maxNumObjectTransferAttempts = 3; - final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client, - maxNumBlockAllocationRetries, - maxNumObjectTransferAttempts); - - final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects); - writeJob.transfer(new FileObjectPutter(dirPath)); - } - - @Test - public void testReadObjectCompletionFiresInternalHandlersFirst() - throws NoSuchMethodException, IOException, ClassNotFoundException, URISyntaxException, IllegalAccessException, InvocationTargetException, NoSuchFieldException - { - putBigFileIntoBlackPearl(); - - final String tempPathPrefix = null; - final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); - - final IntValue intValue = new IntValue(); - - final ObjectCompletedListener objectCompletedListener = new ObjectCompletedListener() { - private int numCompletedObjects = 0; - - @Override - public void objectCompleted(final String name) { - intValue.increment(); - assertTrue(bookTitles.contains(name)); - assertEquals(1, ++numCompletedObjects); - } - }; - - final DataTransferredListener dataTransferredListener = new DataTransferredListener() { - @Override - public void dataTransferred(final long size) { - - } - }; - - try { - final JobImplJobPartDecoratorPair jobImplJobPartDecoratorPair = createJobPartDecoratorAndEnsureObjectPartTrackersPopulated( - objectCompletedListener, - dataTransferredListener, - AccessMode.ForReading); - - final JobPartDecoratorInterceptor jobPartDecoratorInterceptor = new JobPartDecoratorInterceptor(jobImplJobPartDecoratorPair); - jobPartDecoratorInterceptor.transfer(new FileObjectGetter(tempDirectory)); - assertTrue(jobPartDecoratorInterceptor.bothHandlersFired()); - } finally { - FileUtils.deleteDirectory(tempDirectory.toFile()); - } - } -} diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/DataIntegrity_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/DataIntegrity_Test.java index 85b3be4f0..c041a3949 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/DataIntegrity_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/DataIntegrity_Test.java @@ -173,6 +173,8 @@ public void autoChecksumming() throws IOException { final SingleChecksumListener listener = new SingleChecksumListener(); + job.attachChecksumListener(listener); + job.removeChecksumListener(listener); job.attachChecksumListener(listener); job.transfer(new ResourceObjectPutter("books/")); diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index dc4c47469..1b75e99fb 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -18,27 +18,64 @@ import com.google.common.collect.Lists; import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.Ds3ClientImpl; -import com.spectralogic.ds3client.IntValue; -import com.spectralogic.ds3client.commands.DeleteObjectRequest; import com.spectralogic.ds3client.commands.GetObjectRequest; import com.spectralogic.ds3client.commands.GetObjectResponse; import com.spectralogic.ds3client.commands.PutObjectRequest; +import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request; +import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Response; import com.spectralogic.ds3client.commands.spectrads3.GetJobSpectraS3Request; import com.spectralogic.ds3client.commands.spectrads3.GetJobSpectraS3Response; -import com.spectralogic.ds3client.helpers.*; +import com.spectralogic.ds3client.helpers.ChecksumListener; +import com.spectralogic.ds3client.helpers.DataTransferredListener; +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; +import com.spectralogic.ds3client.helpers.FailureEventListener; +import com.spectralogic.ds3client.helpers.FileObjectGetter; +import com.spectralogic.ds3client.helpers.FileObjectPutter; + +import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.MetadataReceivedListener; +import com.spectralogic.ds3client.helpers.ObjectCompletedListener; + +import com.spectralogic.ds3client.helpers.UnrecoverableIOException; +import com.spectralogic.ds3client.helpers.WaitingForChunksListener; import com.spectralogic.ds3client.helpers.events.FailureEvent; +import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner; import com.spectralogic.ds3client.helpers.options.ReadJobOptions; -import com.spectralogic.ds3client.integration.test.helpers.*; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryDelayBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ClientDefinedChunkAttemptRetryDelayBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.GetSequentialBlobStrategy; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.MaxChunkAttemptsRetryBehavior; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.SequentialFileWriterChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.MaxNumObjectTransferAttemptsDecorator; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferMethod; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferRetryDecorator; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; +import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers; +import com.spectralogic.ds3client.integration.test.helpers.ABMTestHelper; +import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShim; +import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory; +import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds; +import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil; +import com.spectralogic.ds3client.models.BulkObject; import com.spectralogic.ds3client.models.ChecksumType; -import com.spectralogic.ds3client.models.Contents; +import com.spectralogic.ds3client.models.MasterObjectList; import com.spectralogic.ds3client.models.Priority; import com.spectralogic.ds3client.models.bulk.Ds3Object; import com.spectralogic.ds3client.models.bulk.PartialDs3Object; import com.spectralogic.ds3client.models.common.Range; +import com.spectralogic.ds3client.networking.Metadata; import com.spectralogic.ds3client.utils.Platform; import com.spectralogic.ds3client.utils.ResourceUtils; import org.apache.commons.io.FileUtils; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; @@ -54,6 +91,7 @@ import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.nio.channels.WritableByteChannel; +import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -61,14 +99,21 @@ import java.security.AccessControlException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static com.spectralogic.ds3client.integration.Util.RESOURCE_BASE_NAME; import static com.spectralogic.ds3client.integration.Util.deleteAllContents; +import static com.spectralogic.ds3client.integration.Util.deleteBucketContents; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.*; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class GetJobManagement_Test { @@ -106,12 +151,24 @@ public static void teardown() throws IOException { private static void setupBucket(final UUID dataPolicy) { try { HELPERS.ensureBucketExists(BUCKET_NAME, dataPolicy); - putBeowulf(); } catch (final Exception e) { LOG.error("Setting up test environment failed: " + e.getMessage()); } } + @Before + public void setupForEachTest() throws Exception { + LOG.info("Setting up before test."); + putBigFiles(); + putBeowulf(); + } + + @After + public void teardownForEachtest() throws IOException { + LOG.info("Tearing down after test."); + deleteBucketContents(client, BUCKET_NAME); + } + private static void putBeowulf() throws Exception { final String book1 = "beowulf.txt"; final Path objPath1 = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + book1); @@ -152,8 +209,6 @@ public void createReadJob() throws IOException, InterruptedException, URISyntaxE @Test public void createReadJobWithBigFile() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { - putBigFiles(); - final String tempPathPrefix = null; final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); @@ -175,6 +230,52 @@ public void createReadJobWithBigFile() throws IOException, URISyntaxException, N final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj)); + final AtomicBoolean dataTransferredEventReceived = new AtomicBoolean(false); + final AtomicBoolean objectCompletedEventReceived = new AtomicBoolean(false); + final AtomicBoolean checksumEventReceived = new AtomicBoolean(false); + final AtomicBoolean metadataEventReceived = new AtomicBoolean(false); + final AtomicBoolean waitingForChunksEventReceived = new AtomicBoolean(false); + final AtomicBoolean failureEventReceived = new AtomicBoolean(false); + + readJob.attachDataTransferredListener(new DataTransferredListener() { + @Override + public void dataTransferred(final long size) { + dataTransferredEventReceived.set(true); + assertEquals(bookSize, size); + } + }); + readJob.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + objectCompletedEventReceived.set(true); + } + }); + readJob.attachChecksumListener(new ChecksumListener() { + @Override + public void value(final BulkObject obj, final ChecksumType.Type type, final String checksum) { + checksumEventReceived.set(true); + assertEquals("0feqCQBgdtmmgGs9pB/Huw==", checksum); + } + }); + readJob.attachMetadataReceivedListener(new MetadataReceivedListener() { + @Override + public void metadataReceived(final String filename, final Metadata metadata) { + metadataEventReceived.set(true); + } + }); + readJob.attachWaitingForChunksListener(new WaitingForChunksListener() { + @Override + public void waiting(final int secondsToWait) { + waitingForChunksEventReceived.set(true); + } + }); + readJob.attachFailureEventListener(new FailureEventListener() { + @Override + public void onFailure(final FailureEvent failureEvent) { + failureEventReceived.set(true); + } + }); + final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); @@ -185,15 +286,21 @@ public void createReadJobWithBigFile() throws IOException, URISyntaxException, N final File originalFile = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME).toFile(); final File fileCopiedFromBP = Paths.get(tempDirectory.toString(), FILE_NAME).toFile(); assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP)); + + assertTrue(dataTransferredEventReceived.get()); + assertTrue(objectCompletedEventReceived.get()); + assertTrue(checksumEventReceived.get()); + assertTrue(metadataEventReceived.get()); + assertFalse(waitingForChunksEventReceived.get()); + assertFalse(failureEventReceived.get()); } finally { FileUtils.deleteDirectory(tempDirectory.toFile()); - deleteBigFileFromBlackPearlBucket(); } } - private void putBigFiles() throws IOException, URISyntaxException { + private static void putBigFiles() throws IOException, URISyntaxException { final String DIR_NAME = "largeFiles/"; - final String[] FILE_NAMES = new String[] { "lesmis-copies.txt", "GreatExpectations.txt" }; + final String[] FILE_NAMES = new String[] {"lesmis.txt", "lesmis-copies.txt", "GreatExpectations.txt" }; final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); @@ -218,21 +325,8 @@ private void putBigFiles() throws IOException, URISyntaxException { writeJob.transfer(new FileObjectPutter(dirPath)); } - private void deleteBigFileFromBlackPearlBucket() throws IOException { - final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); - - final Iterable objects = helpers.listObjects(BUCKET_NAME); - for (final Contents contents : objects) { - if (contents.getKey().equals("lesmis-copies.txt") || contents.getKey().equals("GreatExpectations.txt")) { - client.deleteObject(new DeleteObjectRequest(BUCKET_NAME, contents.getKey())); - } - } - } - - @Test(expected = RuntimeException.class) + @Test(expected = AccessDeniedException.class) public void testReadRetrybugFixWithUnwritableDirectory() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InterruptedException { - putBigFiles(); - final String tempPathPrefix = null; final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix); @@ -285,7 +379,6 @@ public void testReadRetrybugFixWithUnwritableDirectory() throws IOException, URI Runtime.getRuntime().exec("chmod +w " + tempDirectoryName).waitFor(); } FileUtils.deleteDirectory(tempDirectoryPath.toFile()); - deleteBigFileFromBlackPearlBucket(); } } @@ -333,8 +426,6 @@ private void enableWritePermissionForRoot(final String fileOrDirName) { @Test(expected = AccessControlException.class) public void testReadRetrybugWhenChannelThrowsAccessException() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { - putBigFiles(); - final String tempPathPrefix = null; final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix); @@ -369,14 +460,11 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { }); } finally { FileUtils.deleteDirectory(tempDirectoryPath.toFile()); - deleteBigFileFromBlackPearlBucket(); } } @Test public void testReadRetryBugWhenDiskIsFull() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { - putBigFiles(); - final String tempPathPrefix = null; final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix); @@ -410,7 +498,6 @@ public void testReadRetryBugWhenDiskIsFull() throws IOException, URISyntaxExcept } } finally { FileUtils.deleteDirectory(tempDirectoryPath.toFile()); - deleteBigFileFromBlackPearlBucket(); } } @@ -486,6 +573,18 @@ public void createReadJobWithPriorityOption() throws IOException, assertThat(jobSpectraS3Response.getMasterObjectListResult().getPriority(), is(Priority.LOW)); } + @Test + public void testCreatingStreamedReadJobWithPriorityOption() throws IOException, + InterruptedException, URISyntaxException { + + final Ds3ClientHelpers.Job readJob = HELPERS.startReadJobUsingStreamedBehavior(BUCKET_NAME, Lists.newArrayList( + new Ds3Object("beowulf.txt", 10)), ReadJobOptions.create().withPriority(Priority.LOW)); + final GetJobSpectraS3Response jobSpectraS3Response = client + .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); + + assertThat(jobSpectraS3Response.getMasterObjectListResult().getPriority(), is(Priority.LOW)); + } + @Test public void createReadJobWithNameOption() throws IOException, URISyntaxException, InterruptedException { @@ -498,6 +597,18 @@ public void createReadJobWithNameOption() throws IOException, assertThat(jobSpectraS3Response.getMasterObjectListResult().getName(), is("test_job")); } + @Test + public void testCreatingStreamedReadJobWithNameOption() throws IOException, + URISyntaxException, InterruptedException { + + final Ds3ClientHelpers.Job readJob = HELPERS.startReadJobUsingStreamedBehavior(BUCKET_NAME, Lists.newArrayList( + new Ds3Object("beowulf.txt", 10)), ReadJobOptions.create().withName("test_job")); + final GetJobSpectraS3Response jobSpectraS3Response = client + .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); + + assertThat(jobSpectraS3Response.getMasterObjectListResult().getName(), is("test_job")); + } + @Test public void createReadJobWithNameAndPriorityOptions() throws IOException, URISyntaxException, InterruptedException { @@ -513,9 +624,21 @@ public void createReadJobWithNameAndPriorityOptions() throws IOException, } @Test - public void testPartialRetriesWithInjectedFailures() throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException, URISyntaxException { - putBigFiles(); + public void testCreatingStreamedReadJobWithNameAndPriorityOption() throws IOException, + URISyntaxException, InterruptedException { + final Ds3ClientHelpers.Job readJob = HELPERS.startReadJobUsingStreamedBehavior(BUCKET_NAME, Lists.newArrayList( + new Ds3Object("beowulf.txt", 10)), ReadJobOptions.create() + .withName("test_job").withPriority(Priority.LOW)); + final GetJobSpectraS3Response jobSpectraS3Response = client + .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); + + assertThat(jobSpectraS3Response.getMasterObjectListResult().getName(), is("test_job")); + assertThat(jobSpectraS3Response.getMasterObjectListResult().getPriority(), is(Priority.LOW)); + } + + @Test + public void testPartialRetriesWithInjectedFailures() throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException, URISyntaxException { final String tempPathPrefix = null; final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); @@ -542,7 +665,7 @@ public void testPartialRetriesWithInjectedFailures() throws NoSuchMethodExceptio maxNumObjectTransferAttempts); final Ds3ClientHelpers.Job job = ds3ClientHelpers.startReadJob(BUCKET_NAME, filesToGet); - final IntValue intValue = new IntValue(); + final AtomicInteger intValue = new AtomicInteger(); job.attachObjectCompletedListener(new ObjectCompletedListener() { int numPartsCompleted = 0; @@ -550,7 +673,7 @@ public void testPartialRetriesWithInjectedFailures() throws NoSuchMethodExceptio @Override public void objectCompleted(final String name) { assertEquals(1, ++numPartsCompleted); - intValue.increment(); + intValue.incrementAndGet(); } }); @@ -563,11 +686,11 @@ public void dataTransferred(final long size) { job.transfer(new FileObjectGetter(tempDirectory)); - assertEquals(1, intValue.getValue()); + assertEquals(1, intValue.get()); try (final InputStream originalFileStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(DIR_NAME + FILE_NAME)) { final byte[] first300000Bytes = new byte[300000 - offsetIntoFirstRange]; - originalFileStream.skip(10); + originalFileStream.skip(offsetIntoFirstRange); int numBytesRead = originalFileStream.read(first300000Bytes, 0, 300000 - offsetIntoFirstRange); assertThat(numBytesRead, is(300000 -offsetIntoFirstRange )); @@ -583,7 +706,6 @@ public void dataTransferred(final long size) { } } finally { FileUtils.deleteDirectory(tempDirectory.toFile()); - deleteBigFileFromBlackPearlBucket(); } } @@ -592,18 +714,16 @@ public void dataTransferred(final long size) { public void testFiringFailureHandlerWhenGettingChunks() throws URISyntaxException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, IOException { - putBigFiles(); - final String tempPathPrefix = null; final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); try { - final IntValue numFailuresRecorded = new IntValue(); + final AtomicInteger numFailuresRecorded = new AtomicInteger(); final FailureEventListener failureEventListener = new FailureEventListener() { @Override public void onFailure(final FailureEvent failureEvent) { - numFailuresRecorded.increment(); + numFailuresRecorded.incrementAndGet(); assertEquals(FailureEvent.FailureActivity.GettingObject, failureEvent.doingWhat()); } }; @@ -615,11 +735,10 @@ public void onFailure(final FailureEvent failureEvent) { try { readJob.transfer(new FileObjectGetter(tempDirectory)); } catch (final IOException e) { - assertEquals(1, numFailuresRecorded.getValue()); + assertEquals(1, numFailuresRecorded.get()); } } finally { FileUtils.deleteDirectory(tempDirectory.toFile()); - deleteBigFileFromBlackPearlBucket(); } } @@ -650,18 +769,16 @@ private Ds3ClientHelpers.Job createReadJobWithObjectsReadyToTransfer(final Ds3Cl public void testFiringFailureHandlerWhenGettingObject() throws URISyntaxException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, IOException { - putBigFiles(); - final String tempPathPrefix = null; final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); try { - final IntValue numFailuresRecorded = new IntValue(); + final AtomicInteger numFailuresRecorded = new AtomicInteger(); final FailureEventListener failureEventListener = new FailureEventListener() { @Override public void onFailure(final FailureEvent failureEvent) { - numFailuresRecorded.increment(); + numFailuresRecorded.incrementAndGet(); assertEquals(FailureEvent.FailureActivity.GettingObject, failureEvent.doingWhat()); } }; @@ -673,11 +790,582 @@ public void onFailure(final FailureEvent failureEvent) { try { readJob.transfer(new FileObjectGetter(tempDirectory)); } catch (final IOException e) { - assertEquals(1, numFailuresRecorded.getValue()); + assertEquals(1, numFailuresRecorded.get()); + } + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + + @Test + public void testCreatingReadJobWithStreamedBehavior() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + doReadJobWithJobStarter(new ReadJobStarter() { + @Override + public Ds3ClientHelpers.Job startReadJob(final Ds3ClientHelpers ds3ClientHelpers, final String bucketName, final Iterable objectsToread) + throws IOException + { + return ds3ClientHelpers.startReadJobUsingStreamedBehavior(BUCKET_NAME, objectsToread); + } + }); + } + + private interface ReadJobStarter { + Ds3ClientHelpers.Job startReadJob(final Ds3ClientHelpers ds3ClientHelpers, final String bucketName, Iterable objectsToread) throws IOException; + } + + private void doReadJobWithJobStarter(final ReadJobStarter readJobStarter) throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + try { + final String DIR_NAME = "largeFiles/"; + final String FILE_NAME = "lesmis.txt"; + + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize); + + final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl)client); + + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 3; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final Ds3ClientHelpers.Job readJob = readJobStarter.startReadJob(ds3ClientHelpers, BUCKET_NAME, Arrays.asList(obj)); + + final AtomicBoolean dataTransferredEventReceived = new AtomicBoolean(false); + final AtomicBoolean objectCompletedEventReceived = new AtomicBoolean(false); + final AtomicBoolean checksumEventReceived = new AtomicBoolean(false); + final AtomicBoolean metadataEventReceived = new AtomicBoolean(false); + final AtomicBoolean waitingForChunksEventReceived = new AtomicBoolean(false); + final AtomicBoolean failureEventReceived = new AtomicBoolean(false); + + readJob.attachDataTransferredListener(new DataTransferredListener() { + @Override + public void dataTransferred(final long size) { + dataTransferredEventReceived.set(true); + assertEquals(bookSize, size); + } + }); + readJob.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + objectCompletedEventReceived.set(true); + } + }); + readJob.attachChecksumListener(new ChecksumListener() { + @Override + public void value(final BulkObject obj, final ChecksumType.Type type, final String checksum) { + checksumEventReceived.set(true); + assertEquals("69+JXWeZuzl2HFTM6Lbo8A==", checksum); + } + }); + readJob.attachMetadataReceivedListener(new MetadataReceivedListener() { + @Override + public void metadataReceived(final String filename, final Metadata metadata) { + metadataEventReceived.set(true); + } + }); + readJob.attachWaitingForChunksListener(new WaitingForChunksListener() { + @Override + public void waiting(final int secondsToWait) { + waitingForChunksEventReceived.set(true); + } + }); + readJob.attachFailureEventListener(new FailureEventListener() { + @Override + public void onFailure(final FailureEvent failureEvent) { + failureEventReceived.set(true); + } + }); + + final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim + .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); + + readJob.transfer(new FileObjectGetter(tempDirectory)); + + final File originalFile = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME).toFile(); + final File fileCopiedFromBP = Paths.get(tempDirectory.toString(), FILE_NAME).toFile(); + assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP)); + + assertTrue(dataTransferredEventReceived.get()); + assertTrue(objectCompletedEventReceived.get()); + assertTrue(checksumEventReceived.get()); + assertTrue(metadataEventReceived.get()); + assertFalse(waitingForChunksEventReceived.get()); + assertFalse(failureEventReceived.get()); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + + @Test + public void testCreatingReadJobWithRandomAccessBehavior() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + doReadJobWithJobStarter(new ReadJobStarter() { + @Override + public Ds3ClientHelpers.Job startReadJob(final Ds3ClientHelpers ds3ClientHelpers, final String bucketName, final Iterable objectsToread) + throws IOException + { + return ds3ClientHelpers.startReadJobUsingRandomAccessBehavior(BUCKET_NAME, objectsToread); } + }); + } + + @Test + public void testStartReadAllJobUsingStreamedBehavior() throws IOException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + try { + final AtomicInteger numFailuresRecorded = new AtomicInteger(0); + + final FailureEventListener failureEventListener = new FailureEventListener() { + @Override + public void onFailure(final FailureEvent failureEvent) { + numFailuresRecorded.incrementAndGet(); + assertEquals(FailureEvent.FailureActivity.GettingObject, failureEvent.doingWhat()); + } + }; + + final Ds3ClientHelpers.Job readJob = HELPERS.startReadAllJobUsingStreamedBehavior(BUCKET_NAME); + readJob.attachFailureEventListener(failureEventListener); + readJob.transfer(new FileObjectGetter(tempDirectory)); + + final Collection filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false); + + final List filesWeExpectToBeInTempDirectory = Arrays.asList("beowulf.txt", "lesmis.txt", "lesmis-copies.txt", "GreatExpectations.txt"); + + for (final File fileInTempDirectory : filesInTempDirectory) { + assertTrue(filesWeExpectToBeInTempDirectory.contains(fileInTempDirectory.getName())); + } + + assertEquals(0, numFailuresRecorded.get()); } finally { FileUtils.deleteDirectory(tempDirectory.toFile()); - deleteBigFileFromBlackPearlBucket(); } } + + @Test + public void testStartReadAllJobUsingRandomAccessBehavior() throws IOException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + try { + final AtomicInteger numFailuresRecorded = new AtomicInteger(0); + + final FailureEventListener failureEventListener = new FailureEventListener() { + @Override + public void onFailure(final FailureEvent failureEvent) { + numFailuresRecorded.incrementAndGet(); + assertEquals(FailureEvent.FailureActivity.GettingObject, failureEvent.doingWhat()); + } + }; + + final Ds3ClientHelpers.Job readJob = HELPERS.startReadAllJobUsingRandomAccessBehavior(BUCKET_NAME); + readJob.attachFailureEventListener(failureEventListener); + readJob.transfer(new FileObjectGetter(tempDirectory)); + + final Collection filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false); + + final List filesWeExpectToBeInTempDirectory = Arrays.asList("beowulf.txt", "lesmis.txt", "lesmis-copies.txt", "GreatExpectations.txt"); + + for (final File fileInTempDirectory : filesInTempDirectory) { + assertTrue(filesWeExpectToBeInTempDirectory.contains(fileInTempDirectory.getName())); + } + + assertEquals(0, numFailuresRecorded.get()); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + + @Test + public void testGetJobUsingTransferStrategy() throws IOException, InterruptedException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final String fileName = "beowulf.txt"; + + try { + final List objects = Lists.newArrayList(new Ds3Object(fileName)); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(BUCKET_NAME, objects); + + final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client.getBulkJobSpectraS3(getBulkJobSpectraS3Request); + + final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList(); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(masterObjectList) + .withChannelBuilder(new FileObjectGetter(tempDirectory)) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), + PartialObjectHelpers.getPartialObjectsRanges(objects))); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy(); + + transferStrategy.transfer(); + + final Collection filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false); + + for (final File file : filesInTempDirectory) { + assertEquals(fileName, file.getName()); + } + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + + @Test + public void testGetJobWithUserSuppliedBlobStrategy() throws IOException, InterruptedException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final String fileName = "beowulf.txt"; + + try { + final List objects = Lists.newArrayList(new Ds3Object(fileName)); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(BUCKET_NAME, objects); + + final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client.getBulkJobSpectraS3(getBulkJobSpectraS3Request); + + final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList(); + + final EventDispatcher eventDispatcher = new EventDispatcherImpl(new SameThreadEventRunner()); + + final AtomicInteger numChunkAllocationAttempts = new AtomicInteger(0); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(masterObjectList) + .withChannelBuilder(new FileObjectGetter(tempDirectory)) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), + PartialObjectHelpers.getPartialObjectsRanges(objects))) + .withBlobStrategy(new UserSuppliedPutBlobStrategy(client, + masterObjectList, + eventDispatcher, + new MaxChunkAttemptsRetryBehavior(5), + new ClientDefinedChunkAttemptRetryDelayBehavior(1, eventDispatcher), + new Monitorable() { + @Override + public void monitor() { + numChunkAllocationAttempts.getAndIncrement(); + } + })); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy(); + + transferStrategy.transfer(); + + final Collection filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false); + + for (final File file : filesInTempDirectory) { + assertEquals(fileName, file.getName()); + } + + assertEquals(1, numChunkAllocationAttempts.get()); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + + private interface Monitorable { + void monitor(); + } + + private class UserSuppliedPutBlobStrategy implements BlobStrategy { + private final BlobStrategy wrappedBlobStrategy; + private final Monitorable monitorable; + + private UserSuppliedPutBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher, + final ChunkAttemptRetryBehavior retryBehavior, + final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior, + final Monitorable monitorable) + { + this.monitorable = monitorable; + + wrappedBlobStrategy = new GetSequentialBlobStrategy(client, masterObjectList, eventDispatcher, + retryBehavior, chunkAttemptRetryDelayBehavior); + } + + @Override + public Iterable getWork() throws IOException, InterruptedException { + monitorable.monitor(); + + return wrappedBlobStrategy.getWork(); + } + } + + @Test + public void testGetJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException { + testGetJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() { + @Override + public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) { + return transferStrategyBuilder; + } + }); + } + + private void testGetJobWithUserSuppliedChannelStrategy(final TransferStrategyBuilderModifiable transferStrategyBuilderModifiable) throws IOException, InterruptedException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final String fileName = "beowulf.txt"; + + try { + final List objects = Lists.newArrayList(new Ds3Object(fileName)); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(BUCKET_NAME, objects); + + final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client.getBulkJobSpectraS3(getBulkJobSpectraS3Request); + + final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList(); + + final AtomicInteger numTimesChannelOpened = new AtomicInteger(0); + final AtomicInteger numTimesChannelClosed = new AtomicInteger(0); + + TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(masterObjectList) + .withChannelBuilder(new FileObjectGetter(tempDirectory)) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), + PartialObjectHelpers.getPartialObjectsRanges(objects))) + .withChannelStrategy(new UserSuppliedPutChannelStrategy(new FileObjectGetter(tempDirectory), + new ChannelMonitorable() { + @Override + public void acquired() { + numTimesChannelOpened.getAndIncrement(); + } + + @Override + public void released() { + numTimesChannelClosed.getAndIncrement(); + } + })); + + transferStrategyBuilder = transferStrategyBuilderModifiable.modify(transferStrategyBuilder); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy(); + + transferStrategy.transfer(); + + final Collection filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false); + + for (final File file : filesInTempDirectory) { + assertEquals(fileName, file.getName()); + } + + assertEquals(1, numTimesChannelOpened.get()); + assertEquals(1, numTimesChannelClosed.get()); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + + private interface TransferStrategyBuilderModifiable { + TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder); + } + + private interface ChannelMonitorable { + void acquired(); + void released(); + } + + private class UserSuppliedPutChannelStrategy implements ChannelStrategy { + final ChannelMonitorable channelMonitorable; + final ChannelStrategy wrappedPutStrategy; + + private UserSuppliedPutChannelStrategy(final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder, + final ChannelMonitorable channelMonitorable) + { + this.channelMonitorable = channelMonitorable; + wrappedPutStrategy = new SequentialFileWriterChannelStrategy(objectChannelBuilder); + } + + @Override + public SeekableByteChannel acquireChannelForBlob(final BulkObject blob) throws IOException { + channelMonitorable.acquired(); + return wrappedPutStrategy.acquireChannelForBlob(blob); + } + + @Override + public void releaseChannelForBlob(final SeekableByteChannel seekableByteChannel, final BulkObject blob) throws IOException { + channelMonitorable.released(); + wrappedPutStrategy.releaseChannelForBlob(seekableByteChannel, blob); + } + } + + @Test + public void testStreamedGetJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException { + testGetJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() { + @Override + public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) { + return transferStrategyBuilder.usingStreamedTransferBehavior(); + } + }); + } + + @Test + public void testRandomAccessGetJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException { + testGetJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() { + @Override + public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) { + return transferStrategyBuilder.usingRandomAccessTransferBehavior(); + } + }); + } + + @Test + public void testGetJobUserSuppliedTransferRetryDecorator() throws IOException, InterruptedException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final String fileName = "beowulf.txt"; + + try { + final List objects = Lists.newArrayList(new Ds3Object(fileName)); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(BUCKET_NAME, objects); + + final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client.getBulkJobSpectraS3(getBulkJobSpectraS3Request); + + final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList(); + + final AtomicInteger numTimesTransferCalled = new AtomicInteger(0); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(masterObjectList) + .withChannelBuilder(new FileObjectGetter(tempDirectory)) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), + PartialObjectHelpers.getPartialObjectsRanges(objects))) + .withTransferRetryDecorator(new UserSuppliedTransferRetryDecorator( + new Monitorable() { + @Override + public void monitor() { + numTimesTransferCalled.getAndIncrement(); + } + } + )); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy(); + + transferStrategy.transfer(); + + final Collection filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false); + + for (final File file : filesInTempDirectory) { + assertEquals(fileName, file.getName()); + } + + assertEquals(1, numTimesTransferCalled.get()); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + + private class UserSuppliedTransferRetryDecorator implements TransferRetryDecorator { + private final TransferRetryDecorator transferRetryDecorator; + private final Monitorable monitorable; + + private UserSuppliedTransferRetryDecorator(final Monitorable monitorable) { + this.transferRetryDecorator = new MaxNumObjectTransferAttemptsDecorator(5); + this.monitorable = monitorable; + } + + @Override + public TransferMethod wrap(final TransferMethod transferMethod) { + transferRetryDecorator.wrap(transferMethod); + return this; + } + + @Override + public void transferJobPart(final JobPart jobPart) throws IOException { + monitorable.monitor(); + transferRetryDecorator.transferJobPart(jobPart); + } + } + + @Test + public void testGetJobUserSuppliedChunkAttemptRetryBehavior() throws IOException, InterruptedException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final String fileName = "beowulf.txt"; + + try { + final List objects = Lists.newArrayList(new Ds3Object(fileName)); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(BUCKET_NAME, objects); + + final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client.getBulkJobSpectraS3(getBulkJobSpectraS3Request); + + final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList(); + + final AtomicInteger numTimesInvokeCalled = new AtomicInteger(0); + final AtomicInteger numTimesResetCalled = new AtomicInteger(0); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(masterObjectList) + .withChannelBuilder(new FileObjectGetter(tempDirectory)) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), + PartialObjectHelpers.getPartialObjectsRanges(objects))) + .withChunkAttemptRetryBehavior(new UserSuppliedChunkAttemptRetryBehavior( + new ChunkAttemptRetryBehaviorMonitorable() { + @Override + public void invoke() { + numTimesInvokeCalled.getAndIncrement(); + } + + @Override + public void reset() { + numTimesResetCalled.getAndIncrement(); + } + } + )); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy(); + + transferStrategy.transfer(); + + final Collection filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false); + + for (final File file : filesInTempDirectory) { + assertEquals(fileName, file.getName()); + } + + assertEquals(0, numTimesInvokeCalled.get()); + assertEquals(1, numTimesResetCalled.get()); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + + private class UserSuppliedChunkAttemptRetryBehavior implements ChunkAttemptRetryBehavior { + private final ChunkAttemptRetryBehaviorMonitorable chunkAttemptRetryBehaviorMonitorable; + private final ChunkAttemptRetryBehavior wrappedChunkAttemptRetryBehavior; + + private UserSuppliedChunkAttemptRetryBehavior(final ChunkAttemptRetryBehaviorMonitorable chunkAttemptRetryBehaviorMonitorable) { + this.chunkAttemptRetryBehaviorMonitorable = chunkAttemptRetryBehaviorMonitorable; + wrappedChunkAttemptRetryBehavior = new MaxChunkAttemptsRetryBehavior(5); + } + + @Override + public void invoke() throws IOException { + chunkAttemptRetryBehaviorMonitorable.invoke(); + wrappedChunkAttemptRetryBehavior.invoke(); + } + + @Override + public void reset() { + chunkAttemptRetryBehaviorMonitorable.reset(); + wrappedChunkAttemptRetryBehavior.reset(); + } + } + + private interface ChunkAttemptRetryBehaviorMonitorable { + void invoke(); + void reset(); + } } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index e8f00744a..10a77cb29 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -20,18 +20,35 @@ import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.Ds3ClientBuilder; import com.spectralogic.ds3client.Ds3ClientImpl; -import com.spectralogic.ds3client.IntValue; import com.spectralogic.ds3client.commands.*; import com.spectralogic.ds3client.commands.spectrads3.*; import com.spectralogic.ds3client.commands.spectrads3.notifications.*; import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException; +import com.spectralogic.ds3client.helpers.ChecksumFunction; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; import com.spectralogic.ds3client.helpers.FailureEventListener; import com.spectralogic.ds3client.helpers.FileObjectGetter; import com.spectralogic.ds3client.helpers.FileObjectPutter; +import com.spectralogic.ds3client.helpers.JobPart; import com.spectralogic.ds3client.helpers.ObjectCompletedListener; import com.spectralogic.ds3client.helpers.events.FailureEvent; +import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner; import com.spectralogic.ds3client.helpers.options.WriteJobOptions; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryDelayBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ClientDefinedChunkAttemptRetryDelayBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.MaxChunkAttemptsRetryBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.PutSequentialBlobStrategy; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.SequentialFileReaderChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.MaxNumObjectTransferAttemptsDecorator; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferMethod; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferRetryDecorator; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; import com.spectralogic.ds3client.integration.test.helpers.ABMTestHelper; import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory; import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds; @@ -46,15 +63,21 @@ import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory.ClientFailureType; +import com.spectralogic.ds3client.utils.hashing.ChecksumUtils; +import com.spectralogic.ds3client.utils.hashing.Hasher; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.*; +import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; import java.nio.ByteBuffer; +import java.nio.channels.ByteChannel; +import java.nio.channels.Channels; import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; @@ -63,6 +86,7 @@ import java.util.Date; import java.util.List; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShim; @@ -72,7 +96,13 @@ import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; +import com.spectralogic.ds3client.helpers.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class PutJobManagement_Test { + private static final Logger LOG = LoggerFactory.getLogger(PutJobManagement_Test.class); private static final Ds3Client client = Util.fromEnv(); private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client); @@ -539,6 +569,52 @@ public void aggregateTwoJobs() throws IOException { } } + @Test + public void testPutJobOptionsWithStreamingBehavior() throws IOException { + + try { + final WriteJobOptions writeJobOptions = WriteJobOptions.create().withAggregating(); + + final Ds3ClientHelpers.Job jobOne = HELPERS.startWriteJobUsingStreamedBehavior(BUCKET_NAME, + Lists.newArrayList(new Ds3Object("test", 2)), + writeJobOptions); + final UUID jobOneId = jobOne.getJobId(); + + final Ds3ClientHelpers.Job jobTwo = HELPERS.startWriteJobUsingStreamedBehavior(BUCKET_NAME, + Lists.newArrayList(new Ds3Object("test2", 2)), + writeJobOptions); + final UUID jobTwoId = jobTwo.getJobId(); + + assertThat(jobOneId, is(jobTwoId)); + + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + @Test + public void testPutJobOptionsWithRandomAccessBehavior() throws IOException { + + try { + final WriteJobOptions writeJobOptions = WriteJobOptions.create().withAggregating(); + + final Ds3ClientHelpers.Job jobOne = HELPERS.startWriteJobUsingRandomAccessBehavior(BUCKET_NAME, + Lists.newArrayList(new Ds3Object("test", 2)), + writeJobOptions); + final UUID jobOneId = jobOne.getJobId(); + + final Ds3ClientHelpers.Job jobTwo = HELPERS.startWriteJobUsingRandomAccessBehavior(BUCKET_NAME, + Lists.newArrayList(new Ds3Object("test2", 2)), + writeJobOptions); + final UUID jobTwoId = jobTwo.getJobId(); + + assertThat(jobOneId, is(jobTwoId)); + + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + @Test public void allocateJobChunk() throws IOException { @@ -823,18 +899,22 @@ public void putMultiPartUploadPart() throws IOException { @Test public void testWriteJobWithRetries() throws Exception { final int maxNumObjectTransferAttempts = 3; + final boolean computeChecksumWithUserSuppliedFunction = false; + transferAndCheckFileContent(maxNumObjectTransferAttempts, new ObjectTransferExceptionHandler() { @Override public boolean handleException(final Throwable t) { + t.printStackTrace(); fail("Got unexpected exception: " + t.getMessage()); return false; } - }); + }, computeChecksumWithUserSuppliedFunction); } private void transferAndCheckFileContent(final int maxNumObjectTransferAttempts, - final ObjectTransferExceptionHandler objectTransferExceptionHandler) + final ObjectTransferExceptionHandler objectTransferExceptionHandler, + final boolean computeChecksumWithUserSuppliedFunction) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException, URISyntaxException { final String userAgent = "Agent Provocateur"; @@ -851,7 +931,7 @@ private void transferAndCheckFileContent(final int maxNumObjectTransferAttempts, try { final String DIR_NAME = "largeFiles/"; - final String[] FILE_NAMES = new String[]{"lesmis.txt"}; + final String[] FILE_NAMES = new String[]{"lesmis-copies.txt"}; final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); @@ -871,17 +951,89 @@ private void transferAndCheckFileContent(final int maxNumObjectTransferAttempts, maxNumBlockAllocationRetries, maxNumObjectTransferAttempts); - final IntValue intValue = new IntValue(); + final AtomicInteger numTimesObjectCompletedEventCalled = new AtomicInteger(0); + final AtomicInteger numTimesDataTransferredEventCalled = new AtomicInteger(0); + final AtomicBoolean dataTransferredEventReceived = new AtomicBoolean(false); + final AtomicBoolean objectCompletedEventReceived = new AtomicBoolean(false); + final AtomicBoolean checksumEventReceived = new AtomicBoolean(false); + final AtomicBoolean waitingForChunksEventReceived = new AtomicBoolean(false); + final AtomicBoolean failureEventReceived = new AtomicBoolean(false); + final AtomicBoolean checksumFunctionCalled = new AtomicBoolean(false); + + final WriteJobOptions writeJobOptions = WriteJobOptions.create() + .withChecksumType(ChecksumType.Type.MD5); + + final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects, writeJobOptions); + + if (computeChecksumWithUserSuppliedFunction) { + writeJob.withChecksum(new ChecksumFunction() { + @Override + public String compute(final BulkObject obj, final ByteChannel channel) { + checksumFunctionCalled.set(true); + try { + final InputStream dataStream = new BufferedInputStream(Channels.newInputStream(channel)); + + dataStream.mark(Integer.MAX_VALUE); + + final Hasher hasher = ChecksumUtils.getHasher(ChecksumType.Type.MD5); + + final String checksum = ChecksumUtils.hashInputStream(hasher, dataStream); + + LOG.info("Computed checksum for blob: {}", checksum); + + dataStream.reset(); + + return checksum; + } catch (final IOException e) { + LOG.error("Error calculating checksum for: " + obj.getName(), e); + fail("Error calculating checksum for: " + obj.getName()); + } + + return null; + } + }); + } + + writeJob.attachFailureEventListener(new FailureEventListener() { + @Override + public void onFailure(final FailureEvent failureEvent) { + failureEventReceived.set(true); + } + }); + + writeJob.attachChecksumListener(new ChecksumListener() { + @Override + public void value(final BulkObject obj, final ChecksumType.Type type, final String checksum) { + checksumEventReceived.set(true); + assertEquals("0feqCQBgdtmmgGs9pB/Huw==", checksum); + } + }); - final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects); writeJob.attachObjectCompletedListener(new ObjectCompletedListener() { private int numCompletedObjects = 0; @Override public void objectCompleted(final String name) { + objectCompletedEventReceived.set(true); assertTrue(bookTitles.contains(name)); assertEquals(1, ++numCompletedObjects); - intValue.increment(); + numTimesObjectCompletedEventCalled.getAndIncrement(); + } + }); + + writeJob.attachDataTransferredListener(new DataTransferredListener() { + @Override + public void dataTransferred(final long size) { + dataTransferredEventReceived.set(true); + assertEquals(objects.get(0).getSize(), size); + numTimesDataTransferredEventCalled.getAndIncrement(); + } + }); + + writeJob.attachWaitingForChunksListener(new WaitingForChunksListener() { + @Override + public void waiting(final int secondsToWait) { + waitingForChunksEventReceived.set(true); } }); @@ -897,7 +1049,19 @@ public void objectCompleted(final String name) { return; } - assertEquals(1, intValue.getValue()); + if (computeChecksumWithUserSuppliedFunction) { + assertTrue(checksumFunctionCalled.get()); + } else { + assertFalse(checksumFunctionCalled.get()); + } + assertTrue(dataTransferredEventReceived.get()); + assertTrue(objectCompletedEventReceived.get()); + assertTrue(checksumEventReceived.get()); + assertFalse(waitingForChunksEventReceived.get()); + assertFalse(failureEventReceived.get()); + + assertEquals(1, numTimesObjectCompletedEventCalled.get()); + assertEquals(1, numTimesDataTransferredEventCalled.get()); final GetBucketResponse request = ds3ClientShim.getBucket(new GetBucketRequest(BUCKET_NAME)); final ListBucketResult result = request.getListBucketResult(); @@ -936,9 +1100,27 @@ private interface ObjectTransferExceptionHandler { boolean handleException(final Throwable t); } + @Test + public void testWriteJobWithRetriesAndUserDefinedChecksum() throws Exception { + final int maxNumObjectTransferAttempts = 3; + final boolean computeChecksumWithUserSuppliedFunction = true; + + transferAndCheckFileContent(maxNumObjectTransferAttempts, + new ObjectTransferExceptionHandler() { + @Override + public boolean handleException(final Throwable t) { + t.printStackTrace(); + fail("Got unexpected exception: " + t.getMessage()); + return false; + } + }, computeChecksumWithUserSuppliedFunction); + } + @Test public void testWriteJobWithRetriesThrowsDs3NoMoreRetriesException() throws Exception { final int maxNumObjectTransferAttempts = 1; + final boolean computeChecksumWithUserSuppliedFunction = false; + transferAndCheckFileContent(maxNumObjectTransferAttempts, new ObjectTransferExceptionHandler() { @Override @@ -950,7 +1132,7 @@ public boolean handleException(final Throwable t) { return false; } - }); + }, computeChecksumWithUserSuppliedFunction); } @Test @@ -960,7 +1142,7 @@ public void testFiringOnFailureEventWithFailedChunkAllocation() final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); try { - final IntValue numFailureEventsFired = new IntValue(); + final AtomicInteger numFailureEventsFired = new AtomicInteger(0); final int maxNumObjectTransferAttempts = 1; final Ds3ClientHelpers.Job writeJob = createWriteJobWithObjectsReadyToTransfer(maxNumObjectTransferAttempts, @@ -969,7 +1151,7 @@ public void testFiringOnFailureEventWithFailedChunkAllocation() final FailureEventListener failureEventListener = new FailureEventListener() { @Override public void onFailure(final FailureEvent failureEvent) { - numFailureEventsFired.increment(); + numFailureEventsFired.getAndIncrement(); assertEquals(FailureEvent.FailureActivity.PuttingObject, failureEvent.doingWhat()); } }; @@ -979,7 +1161,7 @@ public void onFailure(final FailureEvent failureEvent) { try { writeJob.transfer(new FileObjectPutter(tempDirectory)); } catch (final Ds3NoMoreRetriesException e) { - assertEquals(1, numFailureEventsFired.getValue()); + assertEquals(1, numFailureEventsFired.get()); } } finally { FileUtils.deleteDirectory(tempDirectory.toFile()); @@ -1016,6 +1198,50 @@ Ds3ClientHelpers.Job createWriteJobWithObjectsReadyToTransfer(final int maxNumOb return writeJob; } + @Test + public void testFiringWaitingForChunksEventWithFailedChunkAllocation() + throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + try { + final AtomicInteger numFailureEventsFired = new AtomicInteger(0); + final AtomicInteger numWaitingForChunksEventsFired = new AtomicInteger(0); + + final int maxNumObjectTransferAttempts = 3; + final Ds3ClientHelpers.Job writeJob = createWriteJobWithObjectsReadyToTransfer(maxNumObjectTransferAttempts, + ClientFailureType.ChunkAllocation); + + final FailureEventListener failureEventListener = new FailureEventListener() { + @Override + public void onFailure(final FailureEvent failureEvent) { + numFailureEventsFired.getAndIncrement(); + assertEquals(FailureEvent.FailureActivity.PuttingObject, failureEvent.doingWhat()); + } + }; + + writeJob.attachFailureEventListener(failureEventListener); + + writeJob.attachWaitingForChunksListener(new WaitingForChunksListener() { + @Override + public void waiting(final int secondsToWait) { + numWaitingForChunksEventsFired.getAndIncrement(); + } + }); + + try { + writeJob.transfer(new FileObjectPutter(tempDirectory)); + } catch (final Ds3NoMoreRetriesException e) { + assertEquals(1, numFailureEventsFired.get()); + } + + assertEquals(maxNumObjectTransferAttempts, numWaitingForChunksEventsFired.get()); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + deleteAllContents(client, BUCKET_NAME); + } + } + @Test public void testFiringOnFailureEventWithFailedPutObject() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { @@ -1023,7 +1249,7 @@ public void testFiringOnFailureEventWithFailedPutObject() final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); try { - final IntValue numFailureEventsFired = new IntValue(); + final AtomicInteger numFailureEventsFired = new AtomicInteger(); final int maxNumObjectTransferAttempts = 1; final Ds3ClientHelpers.Job writeJob = createWriteJobWithObjectsReadyToTransfer(maxNumObjectTransferAttempts, @@ -1032,7 +1258,7 @@ public void testFiringOnFailureEventWithFailedPutObject() final FailureEventListener failureEventListener = new FailureEventListener() { @Override public void onFailure(final FailureEvent failureEvent) { - numFailureEventsFired.increment(); + numFailureEventsFired.incrementAndGet(); assertEquals(FailureEvent.FailureActivity.PuttingObject, failureEvent.doingWhat()); } }; @@ -1041,8 +1267,8 @@ public void onFailure(final FailureEvent failureEvent) { try { writeJob.transfer(new FileObjectPutter(tempDirectory)); - } catch (final RuntimeException e) { - assertEquals(1, numFailureEventsFired.getValue()); + } catch (final IOException | RuntimeException e) { + assertEquals(1, numFailureEventsFired.get()); } } finally { FileUtils.deleteDirectory(tempDirectory.toFile()); @@ -1050,6 +1276,494 @@ public void onFailure(final FailureEvent failureEvent) { } } + @Test + public void testPutJobUsingTransferStrategy() throws IOException, InterruptedException, URISyntaxException { + final String DIR_NAME = "books/"; + final String[] FILE_NAMES = new String[]{"beowulf.txt"}; + + try { + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final List bookTitles = new ArrayList<>(); + final List objectsToWrite = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objectsToWrite.add(obj); + } + + final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(objectsToWrite)); + final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = client.putBulkJobSpectraS3(request); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(putBulkJobSpectraS3Response.getMasterObjectList()) + .withChannelBuilder(new FileObjectPutter(dirPath)); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makePutTransferStrategy(); + + transferStrategy.transfer(); + + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client); + final Iterable bucketContentsIterable = ds3ClientHelpers.listObjects(BUCKET_NAME); + + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(FILE_NAMES[0], bucketContents.getKey()); + } + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + @Test + public void testPutJobWithUserSuppliedBlobStrategy() throws IOException, InterruptedException, URISyntaxException { + final String DIR_NAME = "books/"; + final String[] FILE_NAMES = new String[]{"beowulf.txt"}; + + try { + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final List bookTitles = new ArrayList<>(); + final List objectsToWrite = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objectsToWrite.add(obj); + } + + final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(objectsToWrite)); + final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = client.putBulkJobSpectraS3(request); + + final MasterObjectList masterObjectList = putBulkJobSpectraS3Response.getMasterObjectList(); + + final EventDispatcher eventDispatcher = new EventDispatcherImpl(new SameThreadEventRunner()); + + final AtomicInteger numChunkAllocationAttempts = new AtomicInteger(0); + + final BlobStrategy blobStrategy = new UserSuppliedPutBlobStrategy(client, + masterObjectList, + eventDispatcher, + new MaxChunkAttemptsRetryBehavior(5), + new ClientDefinedChunkAttemptRetryDelayBehavior(1, eventDispatcher), + new Monitorable() { + @Override + public void monitor() { + numChunkAllocationAttempts.incrementAndGet(); + } + }); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(masterObjectList) + .withChannelBuilder(new FileObjectPutter(dirPath)) + .withBlobStrategy(blobStrategy); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makePutTransferStrategy(); + transferStrategy.transfer(); + + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client); + final Iterable bucketContentsIterable = ds3ClientHelpers.listObjects(BUCKET_NAME); + + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(FILE_NAMES[0], bucketContents.getKey()); + } + + assertEquals(1, numChunkAllocationAttempts.get()); + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + private interface Monitorable { + void monitor(); + } + + private class UserSuppliedPutBlobStrategy implements BlobStrategy { + private final BlobStrategy wrappedBlobStrategy; + private final Monitorable monitorable; + + private UserSuppliedPutBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher, + final ChunkAttemptRetryBehavior retryBehavior, + final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior, + final Monitorable monitorable) + { + this.monitorable = monitorable; + + wrappedBlobStrategy = new PutSequentialBlobStrategy(client, masterObjectList, eventDispatcher, retryBehavior, chunkAttemptRetryDelayBehavior); + } + + @Override + public Iterable getWork() throws IOException, InterruptedException { + monitorable.monitor(); + + return wrappedBlobStrategy.getWork(); + } + } + + @Test + public void testPutJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException, URISyntaxException { + testPutJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() { + @Override + public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) { + return transferStrategyBuilder; + } + }); + } + + private void testPutJobWithUserSuppliedChannelStrategy(final TransferStrategyBuilderModifiable strategyBuilderModifiable) throws IOException, InterruptedException, URISyntaxException { + final String DIR_NAME = "books/"; + final String[] FILE_NAMES = new String[]{"beowulf.txt"}; + + try { + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final List bookTitles = new ArrayList<>(); + final List objectsToWrite = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objectsToWrite.add(obj); + } + + final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(objectsToWrite)); + final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = client.putBulkJobSpectraS3(request); + + final MasterObjectList masterObjectList = putBulkJobSpectraS3Response.getMasterObjectList(); + + final AtomicInteger numTimesChannelOpened = new AtomicInteger(0); + final AtomicInteger numTimesChannelClosed = new AtomicInteger(0); + + final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder = new FileObjectPutter(dirPath); + final ChannelStrategy channelStrategy = new UserSuppliedPutChannelStrategy(objectChannelBuilder, + new ChannelMonitorable() { + @Override + public void acquired() { + numTimesChannelOpened.incrementAndGet(); + } + + @Override + public void released() { + numTimesChannelClosed.incrementAndGet(); + } + }); + + TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(masterObjectList) + .withChannelStrategy(channelStrategy); + + transferStrategyBuilder = strategyBuilderModifiable.modify(transferStrategyBuilder); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makePutTransferStrategy(); + transferStrategy.transfer(); + + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client); + final Iterable bucketContentsIterable = ds3ClientHelpers.listObjects(BUCKET_NAME); + + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(FILE_NAMES[0], bucketContents.getKey()); + } + + assertEquals(1, numTimesChannelOpened.get()); + assertEquals(1, numTimesChannelClosed.get()); + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + private interface TransferStrategyBuilderModifiable { + TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder); + } + + private interface ChannelMonitorable { + void acquired(); + void released(); + } + + private class UserSuppliedPutChannelStrategy implements ChannelStrategy { + final ChannelMonitorable channelMonitorable; + final ChannelStrategy wrappedPutStrategy; + + + private UserSuppliedPutChannelStrategy(final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder, + final ChannelMonitorable channelMonitorable) + { + this.channelMonitorable = channelMonitorable; + wrappedPutStrategy = new SequentialFileReaderChannelStrategy(objectChannelBuilder); + } + + @Override + public SeekableByteChannel acquireChannelForBlob(final BulkObject blob) throws IOException { + channelMonitorable.acquired(); + return wrappedPutStrategy.acquireChannelForBlob(blob); + } + + @Override + public void releaseChannelForBlob(final SeekableByteChannel seekableByteChannel, final BulkObject blob) throws IOException { + channelMonitorable.released(); + wrappedPutStrategy.releaseChannelForBlob(seekableByteChannel, blob); + } + } + + @Test + public void testStreamingPutJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException, URISyntaxException { + testPutJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() { + @Override + public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) { + return transferStrategyBuilder.usingStreamedTransferBehavior(); + } + }); + } + + @Test + public void testRandomAccessPutJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException, URISyntaxException { + testPutJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() { + @Override + public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) { + return transferStrategyBuilder.usingRandomAccessTransferBehavior(); + } + }); + } + + @Test + public void testPutJobWithUserSuppliedTransferRetryDecorator() throws IOException, InterruptedException, URISyntaxException { + final String DIR_NAME = "books/"; + final String[] FILE_NAMES = new String[]{"beowulf.txt"}; + + try { + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final List bookTitles = new ArrayList<>(); + final List objectsToWrite = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objectsToWrite.add(obj); + } + + final AtomicInteger numTimesTransferCalled = new AtomicInteger(0); + + final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(objectsToWrite)); + final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = client.putBulkJobSpectraS3(request); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(putBulkJobSpectraS3Response.getMasterObjectList()) + .withChannelBuilder(new FileObjectPutter(dirPath)) + .withTransferRetryDecorator(new UserSuppliedTransferRetryDecorator(new Monitorable() { + @Override + public void monitor() { + numTimesTransferCalled.getAndIncrement(); + } + })); + + + final TransferStrategy transferStrategy = transferStrategyBuilder.makePutTransferStrategy(); + + transferStrategy.transfer(); + + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client); + final Iterable bucketContentsIterable = ds3ClientHelpers.listObjects(BUCKET_NAME); + + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(FILE_NAMES[0], bucketContents.getKey()); + } + + assertEquals(1, numTimesTransferCalled.get()); + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + private class UserSuppliedTransferRetryDecorator implements TransferRetryDecorator { + private final TransferRetryDecorator transferRetryDecorator; + private final Monitorable monitorable; + + private UserSuppliedTransferRetryDecorator(final Monitorable monitorable) { + this.transferRetryDecorator = new MaxNumObjectTransferAttemptsDecorator(5); + this.monitorable = monitorable; + } + + @Override + public TransferMethod wrap(final TransferMethod transferMethod) { + transferRetryDecorator.wrap(transferMethod); + return this; + } + + @Override + public void transferJobPart(final JobPart jobPart) throws IOException { + monitorable.monitor(); + transferRetryDecorator.transferJobPart(jobPart); + } + } + + @Test + public void testPutJobWithUserSuppliedChunkAttemptRetryBehavior() throws IOException, InterruptedException, URISyntaxException { + final String DIR_NAME = "books/"; + final String[] FILE_NAMES = new String[]{"beowulf.txt"}; + + try { + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final List bookTitles = new ArrayList<>(); + final List objectsToWrite = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objectsToWrite.add(obj); + } + + final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(objectsToWrite)); + final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = client.putBulkJobSpectraS3(request); + + final AtomicInteger numTimesInvokeCalled = new AtomicInteger(0); + final AtomicInteger numTimesResetCalled = new AtomicInteger(0); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(putBulkJobSpectraS3Response.getMasterObjectList()) + .withChannelBuilder(new FileObjectPutter(dirPath)) + .withChunkAttemptRetryBehavior( + new UserSuppliedChunkAttemptRetryBehavior(new ChunkAttemptRetryBehaviorMonitorable() { + @Override + public void invoke() { + numTimesInvokeCalled.getAndIncrement(); + } + + @Override + public void reset() { + numTimesResetCalled.getAndIncrement(); + } + })); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makePutTransferStrategy(); + + transferStrategy.transfer(); + + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client); + final Iterable bucketContentsIterable = ds3ClientHelpers.listObjects(BUCKET_NAME); + + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(FILE_NAMES[0], bucketContents.getKey()); + } + + assertEquals(0, numTimesInvokeCalled.get()); + assertEquals(1, numTimesResetCalled.get()); + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + private class UserSuppliedChunkAttemptRetryBehavior implements ChunkAttemptRetryBehavior { + private final ChunkAttemptRetryBehaviorMonitorable chunkAttemptRetryBehaviorMonitorable; + private final ChunkAttemptRetryBehavior wrappedChunkAttemptRetryBehavior; + + private UserSuppliedChunkAttemptRetryBehavior(final ChunkAttemptRetryBehaviorMonitorable chunkAttemptRetryBehaviorMonitorable) { + this.chunkAttemptRetryBehaviorMonitorable = chunkAttemptRetryBehaviorMonitorable; + wrappedChunkAttemptRetryBehavior = new MaxChunkAttemptsRetryBehavior(5); + } + + @Override + public void invoke() throws IOException { + chunkAttemptRetryBehaviorMonitorable.invoke(); + wrappedChunkAttemptRetryBehavior.invoke(); + } + + @Override + public void reset() { + chunkAttemptRetryBehaviorMonitorable.reset(); + wrappedChunkAttemptRetryBehavior.reset(); + } + } + + private interface ChunkAttemptRetryBehaviorMonitorable { + void invoke(); + void reset(); + } + + @Test + public void testPutJobUsingStreamedTransferStrategy() throws IOException, URISyntaxException { + final String DIR_NAME = "books/"; + final String[] FILE_NAMES = new String[]{"beowulf.txt"}; + + try { + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final List bookTitles = new ArrayList<>(); + final List objectsToWrite = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objectsToWrite.add(obj); + } + + final Ds3ClientHelpers.Job writeJob = HELPERS.startWriteJobUsingStreamedBehavior(BUCKET_NAME, objectsToWrite); + writeJob.transfer(new FileObjectPutter(dirPath)); + + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client); + final Iterable bucketContentsIterable = ds3ClientHelpers.listObjects(BUCKET_NAME); + + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(FILE_NAMES[0], bucketContents.getKey()); + } + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + @Test + public void testPutJobUsingRandomAccessTransferStrategy() throws IOException, URISyntaxException { + final String DIR_NAME = "books/"; + final String[] FILE_NAMES = new String[]{"beowulf.txt"}; + + try { + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final List bookTitles = new ArrayList<>(); + final List objectsToWrite = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objectsToWrite.add(obj); + } + + final Ds3ClientHelpers.Job writeJob = HELPERS.startWriteJobUsingRandomAccessBehavior(BUCKET_NAME, objectsToWrite); + writeJob.transfer(new FileObjectPutter(dirPath)); + + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client); + final Iterable bucketContentsIterable = ds3ClientHelpers.listObjects(BUCKET_NAME); + + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(FILE_NAMES[0], bucketContents.getKey()); + } + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + @Test public void testThatMetadataAccessDoesNotTerminateTransfer() throws IOException, URISyntaxException, InterruptedException { final String tempPathPrefix = null; @@ -1105,6 +1819,7 @@ public void onFailure(final FailureEvent failureEvent) { } } + /* @Test public void testPutting15000Files() throws IOException, URISyntaxException { final String tempPathPrefix = null; @@ -1165,4 +1880,5 @@ public void testPutting15000Files() throws IOException, URISyntaxException { deleteAllContents(client, BUCKET_NAME); } } + */ } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java index aa0e5aa26..5df2ef496 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java @@ -35,6 +35,7 @@ import com.spectralogic.ds3client.networking.Metadata; import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel; import com.spectralogic.ds3client.utils.ResourceUtils; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -42,6 +43,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.nio.ByteBuffer; @@ -50,6 +52,8 @@ import java.nio.charset.Charset; import java.nio.file.*; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -503,6 +507,72 @@ public void testRecoverWriteJob() throws IOException, JobRecoveryException, URIS } } + @Test + public void testRecoverWriteJobUsingStreamedBehavior() throws IOException, JobRecoveryException, URISyntaxException { + runWriteRecoveryTest(new RecoveryJobFactory() { + @Override + public Ds3ClientHelpers.Job makeRecoveryJob(final Ds3ClientHelpers ds3ClientHelpers, final UUID jobId) throws JobRecoveryException, IOException { + return ds3ClientHelpers.recoverWriteJobUsingStreamedBehavior(jobId); + } + }); + } + + private void runWriteRecoveryTest(final RecoveryJobFactory recoveryJobFactory) throws IOException, JobRecoveryException, URISyntaxException { + final String bucketName = "test_recover_write_job_bucket"; + final String book1 = "beowulf.txt"; + final String book2 = "ulysses.txt"; + + try { + HELPERS.ensureBucketExists(bucketName, envDataPolicyId); + + final Path objPath1 = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + book1); + final Path objPath2 = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + book2); + final Ds3Object obj1 = new Ds3Object(book1, Files.size(objPath1)); + final Ds3Object obj2 = new Ds3Object(book2, Files.size(objPath2)); + + final Map blobCollection = new HashMap<>(); + blobCollection.put(book1, obj1); + blobCollection.put(book2, obj2); + + final Ds3ClientHelpers.Job job = Ds3ClientHelpers.wrap(client).startWriteJob(bucketName, Lists.newArrayList(obj1, obj2)); + + final PutObjectResponse putResponse1 = client.putObject(new PutObjectRequest( + job.getBucketName(), + book1, + new ResourceObjectPutter(RESOURCE_BASE_NAME).buildChannel(book1), + job.getJobId().toString(), + 0, + Files.size(objPath1))); + assertThat(putResponse1, is(notNullValue())); + + // Interuption... + final Ds3ClientHelpers.Job recoverJob = recoveryJobFactory.makeRecoveryJob(HELPERS, job.getJobId()); + + recoverJob.transfer(new ResourceObjectPutter(RESOURCE_BASE_NAME)); + + final Iterable bucketContentsIterable = HELPERS.listObjects(bucketName); + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(blobCollection.get(bucketContents.getKey()).getSize(), bucketContents.getSize()); + } + } finally { + deleteAllContents(client, bucketName); + } + } + + private interface RecoveryJobFactory { + Ds3ClientHelpers.Job makeRecoveryJob(final Ds3ClientHelpers ds3ClientHelpers, final UUID jobId) throws JobRecoveryException, IOException; + } + + @Test + public void testRecoverWriteJobUsingRandomAccessBehavior() throws IOException, JobRecoveryException, URISyntaxException { + runWriteRecoveryTest(new RecoveryJobFactory() { + @Override + public Ds3ClientHelpers.Job makeRecoveryJob(final Ds3ClientHelpers ds3ClientHelpers, final UUID jobId) throws JobRecoveryException, IOException { + return ds3ClientHelpers.recoverWriteJobUsingRandomAccessBehavior(jobId); + } + }); + } + @Test (expected = JobRecoveryNotActiveException.class) public void testRecoverWriteJobCanceledJob() throws IOException, URISyntaxException, JobRecoveryException { final String bucketName = "test_canceled_recover_write_job_bucket"; @@ -726,6 +796,86 @@ public void testRecoverReadJob() throws IOException, JobRecoveryException, URISy } } + @Test + public void testRecoverReadJobUsingStreamedBehavior() throws IOException, JobRecoveryException, URISyntaxException { + runReadRecoveryJob(new RecoveryJobFactory() { + @Override + public Ds3ClientHelpers.Job makeRecoveryJob(final Ds3ClientHelpers ds3ClientHelpers, final UUID jobId) throws JobRecoveryException, IOException { + return ds3ClientHelpers.recoverReadJobsingStreamedBehavior(jobId); + } + }); + } + + private void runReadRecoveryJob(final RecoveryJobFactory recoveryJobFactory) throws IOException, JobRecoveryException, URISyntaxException { + final String bucketName = "test_recover_read_job_bucket"; + final String book1 = "beowulf.txt"; + final String book2 = "ulysses.txt"; + final Path objPath1 = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + book1); + final Path objPath2 = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + book2); + final Ds3Object obj1 = new Ds3Object(book1, Files.size(objPath1)); + final Ds3Object obj2 = new Ds3Object(book2, Files.size(objPath2)); + + final Path dirPath = FileSystems.getDefault().getPath("output"); + if (!Files.exists(dirPath)) { + Files.createDirectory(dirPath); + } + + try { + HELPERS.ensureBucketExists(bucketName, envDataPolicyId); + + final Ds3ClientHelpers.Job putJob = HELPERS.startWriteJob(bucketName, Lists.newArrayList(obj1, obj2)); + putJob.transfer(new ResourceObjectPutter(RESOURCE_BASE_NAME)); + + final FileChannel channel1 = FileChannel.open( + dirPath.resolve(book1), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(bucketName, Lists.newArrayList(obj1, obj2)); + final GetObjectResponse readResponse1 = client.getObject( + new GetObjectRequest( + bucketName, + book1, + channel1, + readJob.getJobId().toString(), + 0)); + + assertThat(readResponse1, is(notNullValue())); + + // Interruption... + final Ds3ClientHelpers.Job recoverJob = recoveryJobFactory.makeRecoveryJob(HELPERS, readJob.getJobId()); + + recoverJob.transfer(new FileObjectGetter(dirPath)); + + final Map blobCollection = new HashMap<>(); + blobCollection.put(book1, obj1); + blobCollection.put(book2, obj2); + + final Collection filesInTempDirectory = FileUtils.listFiles(dirPath.toFile(), null, false); + for (final File fileInTempDirectory : filesInTempDirectory) { + assertEquals(blobCollection.get(fileInTempDirectory.getName()).getSize(), fileInTempDirectory.length()); + } + } finally { + deleteAllContents(client, bucketName); + for( final Path tempFile : Files.newDirectoryStream(dirPath) ){ + Files.delete(tempFile); + } + Files.delete(dirPath); + } + } + + @Test + public void testRecoverReadJobUsingRandomAccessBehavior() throws IOException, JobRecoveryException, URISyntaxException { + runReadRecoveryJob(new RecoveryJobFactory() { + @Override + public Ds3ClientHelpers.Job makeRecoveryJob(final Ds3ClientHelpers ds3ClientHelpers, final UUID jobId) throws JobRecoveryException, IOException { + return ds3ClientHelpers.recoverReadJobUsingRandomAccessBehavior(jobId); + } + }); + } + @Test (expected = JobRecoveryNotActiveException.class) public void testRecoverReadJobCanceledJob() throws IOException, JobRecoveryException, URISyntaxException { final String bucketName = "test_canceled_recover_read_job_bucket"; @@ -1328,6 +1478,41 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { } } + @Test + public void testQuestionMarkInQueryParam() throws IOException { + final String bucketName = "TestQuestionMarkInQueryParam"; + final String objectName = "Test?Question?Mark"; + try { + HELPERS.ensureBucketExists(bucketName, envDataPolicyId); + + final List objs = Lists.newArrayList(new Ds3Object(objectName, 10)); + + final Ds3ClientHelpers.Job job = HELPERS.startWriteJob(bucketName, objs); + + job.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() { + @Override + public SeekableByteChannel buildChannel(final String key) throws IOException { + + final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(124345, 10)); + final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData); + + final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(10); + channel.write(randomBuffer); + + return channel; + } + }); + + final GetObjectsDetailsSpectraS3Response getObjectsSpectraS3Response = client + .getObjectsDetailsSpectraS3(new GetObjectsDetailsSpectraS3Request().withName(objectName)); + + assertThat(getObjectsSpectraS3Response.getS3ObjectListResult().getS3Objects().size(), is(1)); + + } finally { + deleteAllContents(client, bucketName); + } + } + @Test public void testPercentInQueryParam() throws IOException { final String bucketName = "TestPercentInQueryParam"; diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShim.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShim.java index 968aff108..bdecd0970 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShim.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShim.java @@ -36,7 +36,9 @@ public class Ds3ClientShim extends Ds3ClientImpl { private static Method getNetClientMethod = null; - int numRetries = 0; + private final int maxNumRetries; + + private int numRetries = 0; static { try { @@ -48,17 +50,27 @@ public class Ds3ClientShim extends Ds3ClientImpl { getNetClientMethod.setAccessible(true); } - public Ds3ClientShim(final NetworkClient netClient) { + public Ds3ClientShim(final NetworkClient netClient, final int maxNumRetries) { super(netClient); + + this.maxNumRetries = maxNumRetries; + } + + public Ds3ClientShim(final NetworkClient netClient) { + this(netClient, 1); } public Ds3ClientShim(final Ds3ClientImpl ds3ClientImpl) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { - this((NetworkClient)getNetClientMethod.invoke(ds3ClientImpl)); + this((NetworkClient)getNetClientMethod.invoke(ds3ClientImpl), 1); + } + + public Ds3ClientShim(final Ds3ClientImpl ds3ClientImpl, final int maxNumRetries) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + this((NetworkClient)getNetClientMethod.invoke(ds3ClientImpl), maxNumRetries); } @Override public PutObjectResponse putObject(final PutObjectRequest request) throws IOException { - if (numRetries++ >= 1) { + if (numRetries++ >= maxNumRetries) { return super.putObject(request); } @@ -67,7 +79,7 @@ public PutObjectResponse putObject(final PutObjectRequest request) throws IOExce @Override public GetObjectResponse getObject(final GetObjectRequest request) throws IOException { - if (numRetries++ >= 1) { + if (numRetries++ >= maxNumRetries) { return super.getObject(request); } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShimWithFailedGetObject.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShimWithFailedGetObject.java index e9ed0d142..f2c9d3a61 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShimWithFailedGetObject.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShimWithFailedGetObject.java @@ -41,7 +41,6 @@ public GetObjectResponse getObject(final GetObjectRequest request) throws IOExce @Override public Ds3Client newForNode(final JobNode node) { - final ConnectionDetails newConnectionDetails; try { final Ds3Client newClient = super.newForNode(node); return new Ds3ClientShimWithFailedGetObject((Ds3ClientImpl)newClient); diff --git a/ds3-sdk-samples/src/main/java/com/spectralogic/ds3client/samples/BulkPutWithMetadata.java b/ds3-sdk-samples/src/main/java/com/spectralogic/ds3client/samples/BulkPutWithMetadata.java index 6d1c7fe6a..17c7b7296 100644 --- a/ds3-sdk-samples/src/main/java/com/spectralogic/ds3client/samples/BulkPutWithMetadata.java +++ b/ds3-sdk-samples/src/main/java/com/spectralogic/ds3client/samples/BulkPutWithMetadata.java @@ -57,7 +57,7 @@ public static void main(final String[] args) throws IOException { public Map getMetadataValue(final String objectName) { // Return a map of the metadata that you want assigned to the request object final Map metadata = new HashMap<>(); - metadata.put("file-format", "txt"); + metadata.put("File-Format", "txt"); return metadata; } }); diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/AutoCloseableCache.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/AutoCloseableCache.java deleted file mode 100644 index fff3b9ed3..000000000 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/AutoCloseableCache.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * ****************************************************************************** - * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.helpers; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -class AutoCloseableCache implements AutoCloseable { - private final ValueBuilder valueBuilder; - private final Set closedKeys = new HashSet<>(); - private Map values = new HashMap<>(); - - public interface ValueBuilder { - Value get(final Key key); - } - - public AutoCloseableCache(final ValueBuilder valueBuilder) { - this.valueBuilder = valueBuilder; - } - - public synchronized Value get(final Key key) { - if (this.values == null) { - throw new IllegalStateException("Cache already closed."); - } - if (this.closedKeys.contains(key)) { - throw new IllegalStateException("Cache has already closed the requested key."); - } - Value value = this.values.get(key); - if (value == null) { - value = this.valueBuilder.get(key); - this.values.put(key, value); - } - return value; - } - - public synchronized void close(final Key key) throws Exception { - if (this.values == null) { - throw new IllegalStateException("Cache already closed."); - } - final Value value = this.values.remove(key); - if (value != null) { - value.close(); - } - this.closedKeys.add(key); - } - - @Override - public synchronized void close() throws Exception { - if (this.values != null) { - for (final Value value : this.values.values()) { - value.close(); - } - this.values = null; - } - } -} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferrer.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferrer.java deleted file mode 100644 index d8afd0d95..000000000 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferrer.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * ****************************************************************************** - * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.helpers; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.spectralogic.ds3client.Ds3Client; -import com.spectralogic.ds3client.models.BulkObject; -import com.spectralogic.ds3client.helpers.strategy.BlobStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; - -class ChunkTransferrer implements Closeable { - private final static Logger LOG = LoggerFactory.getLogger(ChunkTransferrer.class); - - private final ItemTransferrer itemTransferrer; - private final JobPartTracker partTracker; - private final ListeningExecutorService executor; - - public interface ItemTransferrer { - void transferItem(Ds3Client client, BulkObject ds3Object) throws IOException; - } - - public ChunkTransferrer( - final ItemTransferrer transferrer, - final JobPartTracker partTracker, - final int maxParallelRequests) { - this.itemTransferrer = transferrer; - this.partTracker = partTracker; - LOG.debug("Starting executor service"); - executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(maxParallelRequests)); - LOG.debug("Executor service started"); - } - - public void transferChunks( - final BlobStrategy blobStrategy) - throws IOException, InterruptedException { - final List> tasks = new ArrayList<>(); - - final Iterable work = blobStrategy.getWork(); - - for (final JobPart jobPart : work) { - final BulkObject blob = jobPart.getBulkObject(); - final ObjectPart part = new ObjectPart(blob.getOffset(), blob.getLength()); - - if (this.partTracker.containsPart(blob.getName(), part)) { - LOG.debug("Adding {} offset {} to executor for processing", blob.getName(), blob.getOffset()); - tasks.add(executor.submit(new Callable() { - @Override - public Object call() throws Exception { - LOG.debug("Processing {} offset {}", blob.getName(), blob.getOffset()); - ChunkTransferrer.this.itemTransferrer.transferItem(jobPart.getClient(), blob); - blobStrategy.blobCompleted(blob); - ChunkTransferrer.this.partTracker.completePart(blob.getName(), part); - return null; - } - })); - } - } - - executeWithExceptionHandling(tasks); - } - - @Override - public void close() throws IOException { - LOG.debug("Shutting down executor"); - executor.shutdown(); - } - - private static void executeWithExceptionHandling(final List> tasks) - throws IOException { - try { - // Block on the asynchronous result. - Futures.allAsList(tasks).get(); - } catch (final InterruptedException e) { - // This is a checked exception, but we don't want to expose that this is implemented with futures. - throw new RuntimeException(e); - } catch (final ExecutionException e) { - // The future throws a wrapper exception, but we want don't want to expose that this was implemented with futures. - final Throwable cause = e.getCause(); - - // Throw each of the advertised thrown exceptions. - if (cause instanceof IOException) { - throw (IOException)cause; - } - - // The rest we don't know about, so we'll just forward them. - if (cause instanceof RuntimeException) { - throw (RuntimeException)cause; - } else { - throw new RuntimeException(cause); - } - } - } -} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java index c029e1ce1..825bd64c2 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java @@ -171,6 +171,36 @@ public abstract Ds3ClientHelpers.Job startWriteJob(final String bucket, final It public abstract Ds3ClientHelpers.Job startWriteJob(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) throws IOException; + /** + * Performs a bulk put job creation request and returns an {@link WriteJobImpl} configured to perform a "streamed" + * transfer. "Streamed" means that channels and blobs are read or written sequentially. + * You would use a streaming strategy when it is important that the source or destination channel is read or + * written in a predictable, sequential order while a transfer is in progress. + * See {@link WriteJobImpl} for information on how to write the objects for the job. + * + * @throws IOException + */ + public abstract Ds3ClientHelpers.Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite) + throws IOException; + + public abstract Ds3ClientHelpers.Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) + throws IOException; + + /** + * Performs a bulk put job creation request and returns an {@link WriteJobImpl} configured to perform a "random access" + * transfer. "Random access" means that channels and blobs are read and written in no particular order. You would use + * a random access strategy when it is not important that a source or destination channel is read or written + * in a predictable order while a transfer is in progress. + * See {@link WriteJobImpl} for information on how to write the objects for the job. + * + * @throws IOException + */ + public abstract Ds3ClientHelpers.Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite) + throws IOException; + + public abstract Ds3ClientHelpers.Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) + throws IOException; + /** * Performs a bulk get job creation request and returns an {@link ReadJobImpl}. * See {@link ReadJobImpl} for information on how to read the objects for the job. @@ -186,43 +216,128 @@ public abstract Ds3ClientHelpers.Job startReadJob(final String bucket, final Ite * * @throws IOException */ - public abstract Ds3ClientHelpers.Job startReadJob( - final String bucket, - final Iterable objectsToRead, - final ReadJobOptions options) + public abstract Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException; /** - * Performs a bulk get job creation request for all of the objects in the given bucket and returns an {@link ReadJobImpl}. + * Performs a bulk get job creation request and returns an {@link ReadJobImpl} configured to perform a "streamed" + * transfer. "Streamed" means that channels and blobs are read or written sequentially. + * You would use a streaming strategy when it is important that the source or destination channel is read or + * written in a predictable, sequential order while a transfer is in progress. + * See {@link ReadJobImpl} for information on how to read the objects for the job. * * @throws IOException */ - public abstract Ds3ClientHelpers.Job startReadAllJob(final String bucket) + public abstract Ds3ClientHelpers.Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead) + throws IOException; + + public abstract Ds3ClientHelpers.Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException; /** - * Performs a bulk get job creation request for all of the objects in the given bucket and returns an {@link ReadJobImpl}. + * Performs a bulk get job creation request and returns an {@link ReadJobImpl} configured to perform a "random access" + * transfer. "Random access" means that channels and blobs are read and written in no particular order. You would use + * a random access strategy when it is not important that a source or destination channel is read or written + * in a predictable order while a transfer is in progress. + * See {@link ReadJobImpl} for information on how to read the objects for the job. * * @throws IOException */ - public abstract Ds3ClientHelpers.Job startReadAllJob(final String bucket, final ReadJobOptions options) + public abstract Ds3ClientHelpers.Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead) + throws IOException; + + public abstract Ds3ClientHelpers.Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException; /** - * Queries job information based on job id and returns a {@link ReadJobImpl} that can resume the job. + * Performs a bulk get job creation request for all of the objects in the given bucket and returns an {@link ReadJobImpl}. + * + * @throws IOException + */ + public abstract Ds3ClientHelpers.Job startReadAllJob(final String bucket) throws IOException; + + /** + * Performs a bulk get job creation request for all of the objects in the given bucket and returns an {@link ReadJobImpl}. + * + * @throws IOException + */ + public abstract Ds3ClientHelpers.Job startReadAllJob(final String bucket, final ReadJobOptions options) throws IOException; + + /** + * Performs a bulk get job creation request for all of the objects in the given bucket and returns an {@link ReadJobImpl} + * configured to perform a "streamed" transfer. "Streamed" means that channels and blobs are read or written sequentially. + * You would use a streaming strategy when it is important that the source or destination channel is read or + * written in a predictable, sequential order while a transfer is in progress. + * + * @throws IOException + */ + public abstract Ds3ClientHelpers.Job startReadAllJobUsingStreamedBehavior(final String bucket) throws IOException; + public abstract Ds3ClientHelpers.Job startReadAllJobUsingStreamedBehavior(final String bucket, final ReadJobOptions options) throws IOException; + + /** + * Performs a bulk get job creation request for all of the objects in the given bucket and returns an {@link ReadJobImpl} + * configured to perform a "random access" transfer. "Random access" means that channels and blobs are read and written in no particular order. You would use + * a random access strategy when it is not important that a source or destination channel is read or written + * in a predictable order while a transfer is in progress. + * + * @throws IOException + */ + public abstract Ds3ClientHelpers.Job startReadAllJobUsingRandomAccessBehavior(final String bucket) throws IOException; + public abstract Ds3ClientHelpers.Job startReadAllJobUsingRandomAccessBehavior(final String bucket, final ReadJobOptions options) throws IOException; + + /** + * Queries job information based on job id and returns a {@link WriteJobImpl} that can resume the job. + * @throws IOException + * @throws JobRecoveryException + */ + public abstract Ds3ClientHelpers.Job recoverWriteJob(final UUID jobId) throws IOException, JobRecoveryException; + + /** + * Queries job information based on job id and returns a {@link WriteJobImpl} that can resume the job configured to + * perform a "streamed" transfer. "Streamed" means that channels and blobs are read or written sequentially. + * You would use a streaming strategy when it is important that the source or destination channel is read or + * written in a predictable, sequential order while a transfer is in progress. * @throws IOException * @throws JobRecoveryException */ - public abstract Ds3ClientHelpers.Job recoverWriteJob(final UUID jobId) - throws IOException, JobRecoveryException; + public abstract Ds3ClientHelpers.Job recoverWriteJobUsingStreamedBehavior(final UUID jobId) throws IOException, JobRecoveryException; + + /** + * Queries job information based on job id and returns a {@link WriteJobImpl} that can resume the job configured to + * perform a "random access" transfer. "Random access" means that channels and blobs are read and written in no particular order. You would use + * a random access strategy when it is not important that a source or destination channel is read or written + * in a predictable order while a transfer is in progress. + * @throws IOException + * @throws JobRecoveryException + */ + public abstract Ds3ClientHelpers.Job recoverWriteJobUsingRandomAccessBehavior(final UUID jobId) throws IOException, JobRecoveryException; /** * Queries job information based on job id and returns a {@link WriteJobImpl} that can resume the job. * @throws IOException * @throws JobRecoveryException */ - public abstract Ds3ClientHelpers.Job recoverReadJob(final UUID jobId) - throws IOException, JobRecoveryException; + public abstract Ds3ClientHelpers.Job recoverReadJob(final UUID jobId) throws IOException, JobRecoveryException; + + /** + * Queries job information based on job id and returns a {@link WriteJobImpl} that can resume the job configured to + * performed a "streamed" transfer. "Streamed" means that channels and blobs are read or written sequentially. + * You would use a streaming strategy when it is important that the source or destination channel is read or + * written in a predictable, sequential order while a transfer is in progress. + * @throws IOException + * @throws JobRecoveryException + */ + public abstract Ds3ClientHelpers.Job recoverReadJobsingStreamedBehavior(final UUID jobId) throws IOException, JobRecoveryException; + + /** + * Queries job information based on job id and returns a {@link WriteJobImpl} that can resume the job configured to + * perform a "random access" transfer. "Random access" means that channels and blobs are read and written in no particular order. You would use + * a random access strategy when it is not important that a source or destination channel is read or written + * in a predictable order while a transfer is in progress. + * @throws IOException + * @throws JobRecoveryException + */ + public abstract Ds3ClientHelpers.Job recoverReadJobUsingRandomAccessBehavior(final UUID jobId) throws IOException, JobRecoveryException; /** * Ensures that a bucket exists. The the bucket does not exist, it will be created. diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java index 060b7fc2f..e3270ef1e 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java @@ -16,6 +16,7 @@ package com.spectralogic.ds3client.helpers; import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; @@ -28,13 +29,14 @@ import com.spectralogic.ds3client.commands.spectrads3.*; import com.spectralogic.ds3client.helpers.options.ReadJobOptions; import com.spectralogic.ds3client.helpers.options.WriteJobOptions; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers; import com.spectralogic.ds3client.models.*; -import com.spectralogic.ds3client.models.bulk.RequestType; import com.spectralogic.ds3client.models.common.Range; import com.spectralogic.ds3client.models.bulk.*; import com.spectralogic.ds3client.networking.FailedRequestException; -import com.spectralogic.ds3client.utils.Guard; import com.spectralogic.ds3client.utils.collections.LazyIterable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,82 +53,96 @@ import java.util.UUID; class Ds3ClientHelpersImpl extends Ds3ClientHelpers { - - public final static int DEFAULT_RETRY_DELAY = -1; - public final static int DEFAULT_RETRY_AFTER = -1; - public final static int DEFAULT_OBJECT_TRANSFER_ATTEMPTS = 5; - private final static Logger LOG = LoggerFactory.getLogger(Ds3ClientHelpersImpl.class); private final static int DEFAULT_LIST_OBJECTS_RETRIES = 5; private final Ds3Client client; - private final int retryAfter; - private final int retryDelay; - private final int objectTransferAttempts; + private final int maxChunkAttempts; + private final int secondsBetweenChunkAttempts; + private final int maxObjectTransferAttempts; private final EventRunner eventRunner; private final FileSystemHelper fileSystemHelper; public Ds3ClientHelpersImpl(final Ds3Client client) { - this(client, DEFAULT_RETRY_AFTER); + this(client, TransferStrategyBuilder.DEFAULT_CHUNK_ATTEMPT_RETRY_ATTEMPTS); } - public Ds3ClientHelpersImpl(final Ds3Client client, final int retryAfter) { - this(client, retryAfter, DEFAULT_OBJECT_TRANSFER_ATTEMPTS); + public Ds3ClientHelpersImpl(final Ds3Client client, final int maxChunkAttempts) { + this(client, maxChunkAttempts, TransferStrategyBuilder.DEFAULT_OBJECT_TRANSFER_ATTEMPTS); } - public Ds3ClientHelpersImpl(final Ds3Client client, final int retryAfter, final int objectTransferAttempts) { - this(client, retryAfter, objectTransferAttempts, DEFAULT_RETRY_DELAY); + public Ds3ClientHelpersImpl(final Ds3Client client, final int maxChunkAttempts, final int maxObjectTransferAttempts) { + this(client, maxChunkAttempts, maxObjectTransferAttempts, TransferStrategyBuilder.DEFAULT_CHUNK_ATTEMPT_RETRY_INTERVAL); } - public Ds3ClientHelpersImpl(final Ds3Client client, final int retryAfter, final int objectTransferAttempts, final int retryDelay) { - this(client, retryAfter, objectTransferAttempts, retryDelay, new SameThreadEventRunner()); + public Ds3ClientHelpersImpl(final Ds3Client client, final int maxChunkAttempts, final int maxObjectTransferAttempts, final int secondsBetweenChunkAttempts) { + this(client, maxChunkAttempts, maxObjectTransferAttempts, secondsBetweenChunkAttempts, new SameThreadEventRunner()); } public Ds3ClientHelpersImpl(final Ds3Client client, - final int retryAfter, - final int objectTransferAttempts, - final int retryDelay, + final int maxChunkAttempts, + final int maxObjectTransferAttempts, + final int secondsBetweenChunkAttempts, final EventRunner eventRunner) { - this(client, retryAfter, objectTransferAttempts, retryDelay, eventRunner, new FileSystemHelperImpl()); + this(client, maxChunkAttempts, maxObjectTransferAttempts, secondsBetweenChunkAttempts, eventRunner, new FileSystemHelperImpl()); } public Ds3ClientHelpersImpl(final Ds3Client client, - final int retryAfter, - final int objectTransferAttempts, - final int retryDelay, + final int maxChunkAttempts, + final int maxObjectTransferAttempts, + final int secondsBetweenChunkAttempts, final EventRunner eventRunner, final FileSystemHelper fileSystemHelper) { this.client = client; - this.retryAfter = retryAfter; - this.objectTransferAttempts = objectTransferAttempts; - this.retryDelay = retryDelay; + this.maxChunkAttempts = maxChunkAttempts; + this.maxObjectTransferAttempts = maxObjectTransferAttempts; + this.secondsBetweenChunkAttempts = secondsBetweenChunkAttempts; this.eventRunner = eventRunner; this.fileSystemHelper = fileSystemHelper; } @Override public Ds3ClientHelpers.Job startWriteJob(final String bucket, final Iterable objectsToWrite) - throws IOException { - return innerStartWriteJob(bucket, objectsToWrite, WriteJobOptions.create()); + throws IOException + { + return startWriteJob(bucket, objectsToWrite, WriteJobOptions.create()); } @Override public Ds3ClientHelpers.Job startWriteJob(final String bucket, - final Iterable objectsToWrite, - final WriteJobOptions options) - throws IOException { + final Iterable objectsToWrite, + final WriteJobOptions options) + throws IOException + { if (options == null) { - return innerStartWriteJob(bucket, objectsToWrite, WriteJobOptions.create()); + return innerStartWriteJob(bucket, objectsToWrite, WriteJobOptions.create(), makeTransferStrategyBuilder()); } - return innerStartWriteJob(bucket, objectsToWrite, options); + + return innerStartWriteJob(bucket, objectsToWrite, options, makeTransferStrategyBuilder()); + } + + private TransferStrategyBuilder makeTransferStrategyBuilder() { + final EventDispatcher eventDispatcher = new EventDispatcherImpl(eventRunner); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withNumChunkAttemptRetries(maxChunkAttempts) + .withNumTransferRetries(maxObjectTransferAttempts) + .withChunkRetryDelayInSeconds(secondsBetweenChunkAttempts) + .withEventRunner(eventRunner) + .withEventDispatcher(eventDispatcher); + + return transferStrategyBuilder; } private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket, - final Iterable objectsToWrite, - final WriteJobOptions options) - throws IOException { + final Iterable objectsToWrite, + final WriteJobOptions options, + final TransferStrategyBuilder transferStrategyBuilder) + throws IOException + { final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(bucket, Lists.newArrayList(objectsToWrite)) .withPriority(options.getPriority()) .withAggregating(options.isAggregating()) @@ -137,62 +153,139 @@ private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket, request.withMaxUploadSize(options.getMaxUploadSize()); } - final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = this.client.putBulkJobSpectraS3( - request); + final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = this.client.putBulkJobSpectraS3(request); + transferStrategyBuilder.withMasterObjectList(putBulkJobSpectraS3Response.getMasterObjectList()) + .withChecksumType(options.getChecksumType()); - return new WriteJobImpl( - this.client, - putBulkJobSpectraS3Response.getMasterObjectList(), - this.retryAfter, - options.getChecksumType(), - this.objectTransferAttempts, - this.retryDelay, - this.eventRunner); + return new WriteJobImpl(transferStrategyBuilder); } @Override - public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objectsToRead) - throws IOException { - return innerStartReadJob(bucket, objectsToRead, ReadJobOptions.create()); + public Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite) throws IOException { + return startWriteJobUsingStreamedBehavior(bucket, objectsToWrite, WriteJobOptions.create()); + } + + @Override + public Job startWriteJobUsingStreamedBehavior(final String bucket, + final Iterable objectsToWrite, + final WriteJobOptions options) + throws IOException + { + Preconditions.checkNotNull(options, "options may not be null."); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder(); + transferStrategyBuilder.usingStreamedTransferBehavior(); + + return innerStartWriteJob(bucket, objectsToWrite, options, transferStrategyBuilder); + } + + @Override + public Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite) throws IOException { + return startWriteJobUsingRandomAccessBehavior(bucket, objectsToWrite, WriteJobOptions.create()); + } + + @Override + public Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) throws IOException { + Preconditions.checkNotNull(options, "options may not be null."); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder(); + transferStrategyBuilder.usingRandomAccessTransferBehavior(); + + return innerStartWriteJob(bucket, objectsToWrite, options, transferStrategyBuilder); + } + + @Override + public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objectsToRead) throws IOException { + return startReadJob(bucket, objectsToRead, ReadJobOptions.create()); } @Override public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) - throws IOException { + throws IOException + { + final List objects = Lists.newArrayList(objectsToRead); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request; + if (options == null) { - return innerStartReadJob(bucket, objectsToRead, ReadJobOptions.create()); + getBulkJobSpectraS3Request = makeGetBulkJobSpectraS3Request(bucket, objects, ReadJobOptions.create()); + } else { + getBulkJobSpectraS3Request = makeGetBulkJobSpectraS3Request(bucket, objects, options); } - return innerStartReadJob(bucket, objectsToRead, options); + + getBulkJobSpectraS3Request.withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.NONE); + + return innerStartReadJob(objects, getBulkJobSpectraS3Request, makeTransferStrategyBuilder()); } - private Ds3ClientHelpers.Job innerStartReadJob(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) - throws IOException { - final List objects = Lists.newArrayList(objectsToRead); - final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(bucket, objects) - .withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.NONE) - .withPriority(options.getPriority()); - if (!Guard.isStringNullOrEmpty(options.getName())) { - getBulkJobSpectraS3Request.withName(options.getName()); - } + private GetBulkJobSpectraS3Request makeGetBulkJobSpectraS3Request(final String bucket, final List objects, final ReadJobOptions options) { + return new GetBulkJobSpectraS3Request(bucket, objects) + .withPriority(options.getPriority()) + .withName(options.getName()); + } + private Ds3ClientHelpers.Job innerStartReadJob(final List objects, + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request, + final TransferStrategyBuilder transferStrategyBuilder) + throws IOException + { final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = this.client.getBulkJobSpectraS3(getBulkJobSpectraS3Request); final ImmutableMultimap partialRanges = PartialObjectHelpers.getPartialObjectsRanges(objects); - return new ReadJobImpl( - this.client, - getBulkJobSpectraS3Response.getMasterObjectList(), - partialRanges, - this.objectTransferAttempts, - this.retryAfter, - this.retryDelay, - this.eventRunner); + final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList(); + + transferStrategyBuilder.withMasterObjectList(masterObjectList) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), partialRanges)); + + return new ReadJobImpl(transferStrategyBuilder); } @Override - public Ds3ClientHelpers.Job startReadAllJob(final String bucket) - throws IOException { + public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead) throws IOException { + return startReadJobUsingStreamedBehavior(bucket, objectsToRead, ReadJobOptions.create()); + } + + @Override + public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException { + Preconditions.checkNotNull(options, "options may not be null."); + + final List objects = Lists.newArrayList(objectsToRead); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = makeGetBulkJobSpectraS3Request(bucket, objects, options); + + getBulkJobSpectraS3Request.withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.IN_ORDER); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder(); + transferStrategyBuilder.usingStreamedTransferBehavior(); + + return innerStartReadJob(objects, getBulkJobSpectraS3Request, transferStrategyBuilder); + } + + @Override + public Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead) throws IOException { + return startReadJobUsingRandomAccessBehavior(bucket, objectsToRead, ReadJobOptions.create()); + } + + @Override + public Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException { + Preconditions.checkNotNull(options, "options may not be null."); + + final List objects = Lists.newArrayList(objectsToRead); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = makeGetBulkJobSpectraS3Request(bucket, objects, options); + + getBulkJobSpectraS3Request.withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.NONE); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder(); + transferStrategyBuilder.usingRandomAccessTransferBehavior(); + + return innerStartReadJob(objects, getBulkJobSpectraS3Request, transferStrategyBuilder); + } + + @Override + public Ds3ClientHelpers.Job startReadAllJob(final String bucket) throws IOException { return innerStartReadAllJob(bucket, ReadJobOptions.create()); } @@ -205,53 +298,48 @@ public Ds3ClientHelpers.Job startReadAllJob(final String bucket, final ReadJobOp return innerStartReadAllJob(bucket, options); } - private Ds3ClientHelpers.Job innerStartReadAllJob(final String bucket, final ReadJobOptions options) - throws IOException { - final Iterable contentsList = this.listObjects(bucket); + private Ds3ClientHelpers.Job innerStartReadAllJob(final String bucket, final ReadJobOptions options) throws IOException { + return this.startReadJob(bucket, makeBlobList(bucket), options); + } - final Iterable ds3Objects = this.toDs3Iterable(contentsList, FolderNameFilter.filter()); + private final Iterable makeBlobList(final String bucket) throws IOException { + final Iterable contentsList = listObjects(bucket); - return this.startReadJob(bucket, ds3Objects, options); + return toDs3Iterable(contentsList, FolderNameFilter.filter()); } - @Override - public Ds3ClientHelpers.Job recoverWriteJob(final UUID jobId) throws IOException, JobRecoveryException { - innerVerifyJobActive(jobId); - final ModifyJobSpectraS3Response jobResponse = this.client.modifyJobSpectraS3(new ModifyJobSpectraS3Request(jobId.toString())); - if (JobRequestType.PUT != jobResponse.getMasterObjectListResult().getRequestType()) { - throw new JobRecoveryTypeException( - RequestType.PUT.toString(), - jobResponse.getMasterObjectListResult().getRequestType().toString()); - } - // TODO Need to allow the user to pass in the checksumming information again - return new WriteJobImpl( - this.client, - jobResponse.getMasterObjectListResult(), - this.retryAfter, - ChecksumType.Type.NONE, - this.objectTransferAttempts, - this.retryDelay, - this.eventRunner); + public Ds3ClientHelpers.Job startReadAllJobUsingStreamedBehavior(final String bucket) throws IOException { + return startReadAllJobUsingStreamedBehavior(bucket, ReadJobOptions.create()); + } + + public Ds3ClientHelpers.Job startReadAllJobUsingStreamedBehavior(final String bucket, final ReadJobOptions options) throws IOException { + Preconditions.checkNotNull(options, "options may not be null."); + + return startReadJobUsingStreamedBehavior(bucket, makeBlobList(bucket), options); + } + + public Ds3ClientHelpers.Job startReadAllJobUsingRandomAccessBehavior(final String bucket) throws IOException { + return startReadAllJobUsingRandomAccessBehavior(bucket, ReadJobOptions.create()); + } + + public Ds3ClientHelpers.Job startReadAllJobUsingRandomAccessBehavior(final String bucket, final ReadJobOptions options) throws IOException { + Preconditions.checkNotNull(options, "options may not be null."); + + return startReadJobUsingRandomAccessBehavior(bucket, makeBlobList(bucket), options); } + // TODO Need to allow the user to pass in the checksumming information again @Override - //TODO add a partial object read recovery method. That method will require the list of partial objects. - public Ds3ClientHelpers.Job recoverReadJob(final UUID jobId) throws IOException, JobRecoveryException { + public Ds3ClientHelpers.Job recoverWriteJob(final UUID jobId) throws IOException, JobRecoveryException { innerVerifyJobActive(jobId); - final ModifyJobSpectraS3Response jobResponse = this.client.modifyJobSpectraS3(new ModifyJobSpectraS3Request(jobId.toString())); - if (JobRequestType.GET != jobResponse.getMasterObjectListResult().getRequestType()){ - throw new JobRecoveryTypeException( - RequestType.GET.toString(), - jobResponse.getMasterObjectListResult().getRequestType().toString() ); - } - return new ReadJobImpl( - this.client, - jobResponse.getMasterObjectListResult(), - ImmutableMultimap.of(), - this.objectTransferAttempts, - this.retryAfter, - this.retryDelay, - this.eventRunner); + + final ModifyJobSpectraS3Response jobResponse = modifyJobSpectraS3Response(jobId, JobRequestType.PUT); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() + .withMasterObjectList(jobResponse.getMasterObjectListResult()) + .withChecksumType(ChecksumType.Type.NONE); + + return new WriteJobImpl(transferStrategyBuilder); } /** @@ -271,6 +359,90 @@ private void innerVerifyJobActive(final UUID jobId) throws IOException, JobRecov } } + private ModifyJobSpectraS3Response modifyJobSpectraS3Response(final UUID jobId, final JobRequestType jobRequestType) throws IOException, JobRecoveryException { + final ModifyJobSpectraS3Response jobResponse = client.modifyJobSpectraS3(new ModifyJobSpectraS3Request(jobId.toString())); + + if (jobRequestType != jobResponse.getMasterObjectListResult().getRequestType()) { + throw new JobRecoveryTypeException(jobRequestType.toString(), jobResponse.getMasterObjectListResult().getRequestType().toString()); + } + + return jobResponse; + } + + public Ds3ClientHelpers.Job recoverWriteJobUsingStreamedBehavior(final UUID jobId) throws IOException, JobRecoveryException { + innerVerifyJobActive(jobId); + + final ModifyJobSpectraS3Response jobResponse = modifyJobSpectraS3Response(jobId, JobRequestType.PUT); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() + .withMasterObjectList(jobResponse.getMasterObjectListResult()) + .withChecksumType(ChecksumType.Type.NONE) + .usingStreamedTransferBehavior(); + + return new WriteJobImpl(transferStrategyBuilder); + } + + public Ds3ClientHelpers.Job recoverWriteJobUsingRandomAccessBehavior(final UUID jobId) throws IOException, JobRecoveryException { + innerVerifyJobActive(jobId); + + final ModifyJobSpectraS3Response jobResponse = modifyJobSpectraS3Response(jobId, JobRequestType.PUT); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() + .withMasterObjectList(jobResponse.getMasterObjectListResult()) + .withChecksumType(ChecksumType.Type.NONE) + .usingRandomAccessTransferBehavior(); + + return new WriteJobImpl(transferStrategyBuilder); + } + + @Override + //TODO add a partial object read recovery method. That method will require the list of partial objects. + public Ds3ClientHelpers.Job recoverReadJob(final UUID jobId) throws IOException, JobRecoveryException { + innerVerifyJobActive(jobId); + + final MasterObjectList masterObjectList = masterObjectListForGetJob(jobId); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() + .withMasterObjectList(masterObjectList) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())); + + return new ReadJobImpl(transferStrategyBuilder); + } + + private MasterObjectList masterObjectListForGetJob(final UUID jobId) throws IOException, JobRecoveryException { + final ModifyJobSpectraS3Response jobResponse = modifyJobSpectraS3Response(jobId, JobRequestType.GET); + + return jobResponse.getMasterObjectListResult(); + } + + @Override + public Job recoverReadJobsingStreamedBehavior(final UUID jobId) throws IOException, JobRecoveryException { + innerVerifyJobActive(jobId); + + final MasterObjectList masterObjectList = masterObjectListForGetJob(jobId); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() + .withMasterObjectList(masterObjectList) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())) + .usingStreamedTransferBehavior(); + + return new ReadJobImpl(transferStrategyBuilder); + } + + @Override + public Job recoverReadJobUsingRandomAccessBehavior(final UUID jobId) throws IOException, JobRecoveryException { + innerVerifyJobActive(jobId); + + final MasterObjectList masterObjectList = masterObjectListForGetJob(jobId); + + final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() + .withMasterObjectList(masterObjectList) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())) + .usingRandomAccessTransferBehavior(); + + return new ReadJobImpl(transferStrategyBuilder); + } + @Override public void ensureBucketExists(final String bucket) throws IOException { final HeadBucketResponse response = this.client.headBucket(new HeadBucketRequest(bucket)); diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java index a6973dc46..24e5c5ca3 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java @@ -48,8 +48,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { return FileChannel.open( objectPath, StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING + StandardOpenOption.CREATE ); } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java index 0fbe8e685..9ba6c57aa 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java @@ -15,73 +15,50 @@ package com.spectralogic.ds3client.helpers; -import com.google.common.collect.Sets; -import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.Job; -import com.spectralogic.ds3client.helpers.events.EventRunner; -import com.spectralogic.ds3client.helpers.events.FailureEvent; -import com.spectralogic.ds3client.models.BulkObject; -import com.spectralogic.ds3client.models.ChecksumType; -import com.spectralogic.ds3client.models.MasterObjectList; -import com.spectralogic.ds3client.models.Objects; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.List; -import java.util.Set; import java.util.UUID; -import static com.spectralogic.ds3client.helpers.ReadJobImpl.getAllBlobApiBeans; - abstract class JobImpl implements Job { private static final Logger LOG = LoggerFactory.getLogger(JobImpl.class); - protected final Ds3Client client; - protected final MasterObjectList masterObjectList; protected boolean running = false; - protected int maxParallelRequests = 10; - private final int objectTransferAttempts; - private final JobPartTrackerDecorator jobPartTracker; - private final EventRunner eventRunner; - private final Set failureEventListeners; - private final Set waitingForChunksListeners; - private final Set checksumListeners; - public JobImpl(final Ds3Client client, - final MasterObjectList masterObjectList, - final int objectTransferAttempts, - final EventRunner eventRunner) { - this.client = client; - this.masterObjectList = masterObjectList; - this.objectTransferAttempts = objectTransferAttempts; - this.eventRunner = eventRunner; - this.failureEventListeners = Sets.newIdentityHashSet(); - this.waitingForChunksListeners = Sets.newIdentityHashSet(); - this.checksumListeners = Sets.newIdentityHashSet(); - this.jobPartTracker = makeJobPartTracker(getChunks(masterObjectList), eventRunner); + private final TransferStrategyBuilder transferStrategyBuilder; + + public JobImpl(final TransferStrategyBuilder transferStrategyBuilder) { + this.transferStrategyBuilder = transferStrategyBuilder; } - + @Override public UUID getJobId() { - if (this.masterObjectList == null) { + try { + return transferStrategyBuilder.masterObjectList().getJobId(); + } catch (final Throwable t) { + LOG.warn("Could not get job id.", t); return null; } - return this.masterObjectList.getJobId(); } @Override public String getBucketName() { - if (this.masterObjectList == null) { + try { + return transferStrategyBuilder.masterObjectList().getBucketName(); + } catch (final Throwable t) { + LOG.warn("Could not get bucket name.", t); return null; } - return this.masterObjectList.getBucketName(); } - + @Override public Job withMaxParallelRequests(final int maxParallelRequests) { - this.maxParallelRequests = maxParallelRequests; + transferStrategyBuilder.withNumConcurrentTransferThreads(maxParallelRequests); return this; } @@ -89,202 +66,76 @@ protected void checkRunning() { if (running) throw new IllegalStateException("You cannot modify a job after calling transfer"); } - protected void transferItem( - final Ds3Client client, - final BulkObject ds3Object, - final ChunkTransferrer.ItemTransferrer itemTransferrer) - throws IOException - { - int objectTransfersAttempted = 0; - - while(true) { - try { - itemTransferrer.transferItem(client, ds3Object); - break; - } catch (final Throwable t) { - if (ExceptionClassifier.isUnrecoverableException(t) || ++objectTransfersAttempted >= objectTransferAttempts) { - throw t; - } - } - } - } - - protected EventRunner getEventRunner() { - return eventRunner; - } - @Override public void attachChecksumListener(final ChecksumListener listener) { checkRunning(); - this.checksumListeners.add(listener); + eventDispatcher().attachChecksumListener(listener); } @Override public void removeChecksumListener(final ChecksumListener listener) { checkRunning(); - this.checksumListeners.remove(listener); + eventDispatcher().removeChecksumListener(listener); } @Override public void attachWaitingForChunksListener(final WaitingForChunksListener listener) { checkRunning(); - this.waitingForChunksListeners.add(listener); + eventDispatcher().attachWaitingForChunksListener(listener); } @Override public void removeWaitingForChunksListener(final WaitingForChunksListener listener) { checkRunning(); - this.waitingForChunksListeners.remove(listener); + eventDispatcher().removeWaitingForChunksListener(listener); } @Override public void attachFailureEventListener(final FailureEventListener listener) { checkRunning(); - this.failureEventListeners.add(listener); + eventDispatcher().attachFailureEventListener(listener); } @Override public void removeFailureEventListener(final FailureEventListener listener) { checkRunning(); - this.failureEventListeners.remove(listener); + eventDispatcher().removeFailureEventListener(listener); } @Override public void attachDataTransferredListener(final DataTransferredListener listener) { checkRunning(); - getJobPartTracker().attachDataTransferredListener(listener); + eventDispatcher().attachDataTransferredListener(listener); } @Override public void removeDataTransferredListener(final DataTransferredListener listener) { checkRunning(); - getJobPartTracker().removeDataTransferredListener(listener); + eventDispatcher().removeDataTransferredListener(listener); } @Override public void attachObjectCompletedListener(final ObjectCompletedListener listener) { checkRunning(); - getJobPartTracker().attachClientObjectCompletedListener(listener); + eventDispatcher().attachObjectCompletedListener(listener); } @Override public void removeObjectCompletedListener(final ObjectCompletedListener listener) { checkRunning(); - getJobPartTracker().removeClientObjectCompletedListener(listener); - } - - protected void emitFailureEvent(final FailureEvent failureEvent) { - for (final FailureEventListener failureEventListener : failureEventListeners) { - eventRunner.emitEvent(new Runnable() { - @Override - public void run() { - failureEventListener.onFailure(failureEvent); - } - }); - } - } - - protected FailureEvent makeFailureEvent(final FailureEvent.FailureActivity failureActivity, - final Throwable causalException, - final Objects chunk) - { - return FailureEvent.builder() - .doingWhat(failureActivity) - .withCausalException(causalException) - .withObjectNamed(getLabelForChunk(chunk)) - .usingSystemWithEndpoint(client.getConnectionDetails().getEndpoint()) - .build(); - } - - protected String getLabelForChunk(final Objects chunk) { - try { - return chunk.getObjects().get(0).getName(); - } catch (final Throwable t) { - LOG.error("Failed to get label for chunk.", t); - } - - return "unnamed object"; + eventDispatcher().removeObjectCompletedListener(listener); } - protected void emitWaitingForChunksEvents(final int retryAfter) { - for (final WaitingForChunksListener waitingForChunksListener : waitingForChunksListeners) { - eventRunner.emitEvent(new Runnable() { - @Override - public void run() { - waitingForChunksListener.waiting(retryAfter); - } - }); - } - } - - protected void emitChecksumEvents(final BulkObject bulkObject, final ChecksumType.Type type, final String checksum) { - for (final ChecksumListener listener : checksumListeners) { - getEventRunner().emitEvent(new Runnable() { - @Override - public void run() { - listener.value(bulkObject, type, checksum); - } - }); - } + @Override + public void transfer(final Ds3ClientHelpers.ObjectChannelBuilder channelBuilder) throws IOException { + transferStrategyBuilder.withChannelBuilder(channelBuilder); } - protected abstract List getChunks(final MasterObjectList masterObjectList); - protected abstract JobPartTrackerDecorator makeJobPartTracker(final List chunks, final EventRunner eventRunner); - - protected JobPartTrackerDecorator getJobPartTracker() { - return jobPartTracker; + protected TransferStrategyBuilder transferStrategyBuilder() { + return transferStrategyBuilder; } - protected static class JobPartTrackerDecorator implements JobPartTracker { - private final JobPartTracker clientJobPartTracker; - private final JobPartTracker internalJobPartTracker; - - protected JobPartTrackerDecorator(final List chunks, final EventRunner eventRunner) { - clientJobPartTracker = JobPartTrackerFactory.buildPartTracker(getAllBlobApiBeans(chunks), eventRunner); - internalJobPartTracker = JobPartTrackerFactory.buildPartTracker(getAllBlobApiBeans(chunks), eventRunner); - } - - @Override - public void completePart(final String key, final ObjectPart objectPart) { - // It's important to fire the internal completions -- those we set up to close channels we - // have opened -- before firing client-registered events. The reason is that some clients - // rely upon this ordering to know that channels are closed when their event handlers run. - internalJobPartTracker.completePart(key, objectPart); - clientJobPartTracker.completePart(key, objectPart); - } - - @Override - public boolean containsPart(final String key, final ObjectPart objectPart) { - return internalJobPartTracker.containsPart(key, objectPart) || clientJobPartTracker.containsPart(key, objectPart); - } - - @Override - public JobPartTracker attachDataTransferredListener(final DataTransferredListener listener) { - return clientJobPartTracker.attachDataTransferredListener(listener); - } - - @Override - public JobPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) { - internalJobPartTracker.attachObjectCompletedListener(listener); - return this; - } - - @Override - public void removeDataTransferredListener(final DataTransferredListener listener) { - clientJobPartTracker.removeDataTransferredListener(listener); - } - - @Override - public void removeObjectCompletedListener(final ObjectCompletedListener listener) { - internalJobPartTracker.removeObjectCompletedListener(listener); - } - - protected void attachClientObjectCompletedListener(final ObjectCompletedListener listener) { - clientJobPartTracker.attachObjectCompletedListener(listener); - } - - protected void removeClientObjectCompletedListener(final ObjectCompletedListener listener) { - clientJobPartTracker.removeObjectCompletedListener(listener); - } + protected EventDispatcher eventDispatcher() { + return transferStrategyBuilder.eventDispatcher(); } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java index 794efb118..12ad17d0f 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java @@ -18,6 +18,9 @@ import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.models.BulkObject; +/** + * A class that associates a blob with the Black Pearl on which it resides. + */ public class JobPart { private final Ds3Client client; private final BulkObject bulkObject; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java index 01247d99e..79b40ef6c 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java @@ -26,7 +26,7 @@ import java.util.Collection; import java.util.HashMap; -final class JobPartTrackerFactory { +public final class JobPartTrackerFactory { public static JobPartTracker buildPartTracker(final Iterable objects, final EventRunner eventRunner) { final ArrayListMultimap multimap = ArrayListMultimap.create(); for (final BulkObject bulkObject : Preconditions.checkNotNull(objects)) { diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java index 9428fbdaf..3a802d954 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java @@ -15,39 +15,19 @@ package com.spectralogic.ds3client.helpers; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; -import com.spectralogic.ds3client.helpers.AutoCloseableCache.ValueBuilder; -import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; -import com.spectralogic.ds3client.helpers.channels.RangedSeekableByteChannel; -import com.spectralogic.ds3client.helpers.channels.WindowedChannelFactory; import com.spectralogic.ds3client.models.BulkObject; import com.spectralogic.ds3client.models.Objects; -import com.spectralogic.ds3client.models.common.Range; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.channels.SeekableByteChannel; import java.util.Collection; import java.util.HashSet; import java.util.concurrent.atomic.AtomicInteger; -class JobState implements AutoCloseable { - - private static final Logger LOG = LoggerFactory.getLogger(JobState.class); - +public class JobState { private final AtomicInteger objectsRemaining; - private final AutoCloseableCache channelCache; private final JobPartTracker partTracker; - public JobState( - final ObjectChannelBuilder channelBuilder, - final Collection filteredChunks, - final JobPartTracker partTracker, - final ImmutableMap> objectRanges) { + public JobState(final Collection filteredChunks,final JobPartTracker partTracker) { this.objectsRemaining = new AtomicInteger(getObjectCount(filteredChunks)); - this.channelCache = buildCache(channelBuilder, objectRanges); this.partTracker = partTracker.attachObjectCompletedListener(new ObjectCompletedListenerImpl()); } @@ -65,50 +45,13 @@ private static int getObjectCount(final Collection chunks) { return result.size(); } - private static AutoCloseableCache buildCache( - final ObjectChannelBuilder channelBuilder, - final ImmutableMap> objectRanges) { - return new AutoCloseableCache<>( - new ValueBuilder() { - @Override - public WindowedChannelFactory get(final String key) { - try { - LOG.debug("Opening channel for: {}", key); - return new WindowedChannelFactory(RangedSeekableByteChannel.wrap(channelBuilder.buildChannel(key), objectRanges.get(key), key)); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - } - ); - } - - @Override - public void close() throws Exception { - this.channelCache.close(); - } - - public JobPartTracker getPartTracker() { - return partTracker; - } - private final class ObjectCompletedListenerImpl implements ObjectCompletedListener { @Override public void objectCompleted(final String name) { JobState.this.objectsRemaining.decrementAndGet(); - try { - LOG.debug("Closing file: {}", name); - JobState.this.channelCache.close(name); - } catch (final Exception e) { - throw new RuntimeException(e); - } } } - public SeekableByteChannel getChannel(final String name, final long offset, final long length) { - return this.channelCache.get(name).get(offset, length); - } - @Override public String toString() { return "JobState{" + diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java index 940d14290..adb915cc4 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java @@ -15,71 +15,29 @@ package com.spectralogic.ds3client.helpers; -import com.google.common.collect.*; -import com.spectralogic.ds3client.Ds3Client; -import com.spectralogic.ds3client.commands.GetObjectRequest; -import com.spectralogic.ds3client.commands.GetObjectResponse; -import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException; -import com.spectralogic.ds3client.helpers.ChunkTransferrer.ItemTransferrer; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; -import com.spectralogic.ds3client.helpers.events.EventRunner; -import com.spectralogic.ds3client.helpers.events.FailureEvent; -import com.spectralogic.ds3client.helpers.strategy.BlobStrategy; -import com.spectralogic.ds3client.helpers.strategy.GetStreamerStrategy; -import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers; -import com.spectralogic.ds3client.models.*; -import com.spectralogic.ds3client.models.common.Range; -import com.spectralogic.ds3client.networking.Metadata; -import com.spectralogic.ds3client.utils.Guard; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.MetaDataReceivedObserver; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; import java.io.IOException; -import java.util.List; -import java.util.Set; class ReadJobImpl extends JobImpl { - private final ImmutableMap> blobToRanges; - private final Set metadataListeners; - - private final int retryAfter; // Negative retryAfter value represent infinity retries - private final int retryDelay; // Negative value represents default - - public ReadJobImpl( - final Ds3Client client, - final MasterObjectList masterObjectList, - final ImmutableMultimap objectRanges, - final int objectTransferAttempts, - final int retryAfter, - final int retryDelay, - final EventRunner eventRunner - ) { - super(client, masterObjectList, objectTransferAttempts, eventRunner); - - this.blobToRanges = PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), objectRanges); - this.metadataListeners = Sets.newIdentityHashSet(); - - this.retryAfter = retryAfter; - this.retryDelay = retryDelay; - } - - protected static ImmutableList getAllBlobApiBeans(final List jobWithChunksApiBeans) { - final ImmutableList.Builder builder = ImmutableList.builder(); - for (final Objects objects : jobWithChunksApiBeans) { - builder.addAll(objects.getObjects()); - } - return builder.build(); + public ReadJobImpl(final TransferStrategyBuilder transferStrategyBuilder) { + super(transferStrategyBuilder); } @Override public void attachMetadataReceivedListener(final MetadataReceivedListener listener) { checkRunning(); - this.metadataListeners.add(listener); + eventDispatcher().attachMetadataReceivedEventObserver(new MetaDataReceivedObserver(listener)); } @Override public void removeMetadataReceivedListener(final MetadataReceivedListener listener) { checkRunning(); - this.metadataListeners.remove(listener); + eventDispatcher().removeMetadataReceivedEventObserver(new MetaDataReceivedObserver(listener)); } @Override @@ -93,216 +51,14 @@ public Ds3ClientHelpers.Job withChecksum(final ChecksumFunction checksumFunction } @Override - public void transfer(final ObjectChannelBuilder channelBuilder) - throws IOException { - try { - running = true; - - final BlobStrategy strategy = new GetStreamerStrategy( - client, - this.masterObjectList, - retryAfter, - retryDelay, - new GetStreamerStrategy.ChunkEventHandler() { - @Override - public void emitWaitingForChunksEvents(final int secondsToDelay) { - ReadJobImpl.super.emitWaitingForChunksEvents(secondsToDelay); - } - }); - - try (final JobState jobState = new JobState( - channelBuilder, - this.masterObjectList.getObjects(), - getJobPartTracker(), blobToRanges)) { - try (final ChunkTransferrer chunkTransferrer = new ChunkTransferrer( - new GetObjectTransferrerRetryDecorator(jobState), - jobState.getPartTracker(), - this.maxParallelRequests - )) { - while (jobState.hasObjects()) { - chunkTransferrer.transferChunks(strategy); - } - } - } catch (final RuntimeException | IOException e) { - throw e; - } catch (final Exception e) { - throw new RuntimeException(e); - } - } catch (final Throwable t) { - emitFailureEvent(makeFailureEvent(FailureEvent.FailureActivity.GettingObject, t, masterObjectList.getObjects().get(0))); - throw t; - } - } - - @Override - protected List getChunks(final MasterObjectList masterObjectList) { - return masterObjectList.getObjects(); - } - - @Override - protected JobPartTrackerDecorator makeJobPartTracker(final List chunks, final EventRunner eventRunner) { - return new JobPartTrackerDecorator(chunks, eventRunner); - } - - private final class GetObjectTransferrerRetryDecorator implements ItemTransferrer { - private final ItemTransferrer getObjectTransferrer; - - private GetObjectTransferrerRetryDecorator(final JobState jobState) { - getObjectTransferrer = new GetObjectTransferrerNetworkFailureDecorator(jobState); - } - - @Override - public void transferItem(final Ds3Client client, final BulkObject ds3Object) throws IOException { - ReadJobImpl.this.transferItem(client, ds3Object, getObjectTransferrer); - } - } - - private final class GetObjectTransferrerNetworkFailureDecorator implements ItemTransferrer { - private final JobState jobState; - private Long numBytesToTransfer; - private ImmutableCollection ranges; - private Long destinationChannelOffset = 0L; - private ItemTransferrer itemTransferrer; - - private GetObjectTransferrerNetworkFailureDecorator(final JobState jobState) { - this.jobState = jobState; - itemTransferrer = new GetObjectTransferrer(jobState); - } - - @Override - public void transferItem(final Ds3Client client, final BulkObject ds3Object) throws IOException { - try { - itemTransferrer.transferItem(client, ds3Object); - } catch (final ContentLengthNotMatchException contentLengthNotMatchException) { - makeNewItemTransferrer(ds3Object, contentLengthNotMatchException.getTotalBytes()); - - emitContentLengthMismatchFailureEvent(ds3Object, contentLengthNotMatchException); - - throw new RecoverableIOException(contentLengthNotMatchException); - } - } - - private void makeNewItemTransferrer(final BulkObject ds3Object, final long numBytesTransferred) { - initializeRangesAndTransferSize(ds3Object); - updateRanges(numBytesTransferred); - destinationChannelOffset += numBytesTransferred; - - itemTransferrer = new GetPartialObjectTransferrer(jobState, ranges, destinationChannelOffset); - } - - private void initializeRangesAndTransferSize(final BulkObject ds3Object) { - if (ranges == null) { - ranges = getRangesForBlob(blobToRanges, ds3Object); - } - - if (ranges == null) { - final long numBytesTransferred = 0; - ranges = RangeHelper.replaceRange(ranges, numBytesTransferred, ds3Object.getLength()); - } - - if (numBytesToTransfer == null) { - numBytesToTransfer = RangeHelper.transferSizeForRanges(ranges); - } - } - - private void updateRanges(final long numBytesTransferred) { - ranges = RangeHelper.replaceRange(ranges, numBytesTransferred, numBytesToTransfer); - } - - private void emitContentLengthMismatchFailureEvent(final BulkObject ds3Object, final Throwable t) { - final FailureEvent failureEvent = FailureEvent.builder() - .doingWhat(FailureEvent.FailureActivity.GettingObject) - .usingSystemWithEndpoint(client.getConnectionDetails().getEndpoint()) - .withCausalException(t) - .withObjectNamed(ds3Object.getName()) - .build(); - emitFailureEvent(failureEvent); - } - } - - private final class GetPartialObjectTransferrer implements ItemTransferrer { - private final JobState jobState; - private final ImmutableCollection ranges; - private final long destinationChannelOffset; - - private GetPartialObjectTransferrer(final JobState jobState, - final ImmutableCollection ranges, - final long destinationChannelOffset) - { - this.jobState = jobState; - this.ranges = ranges; - this.destinationChannelOffset = destinationChannelOffset; - } - - @Override - public void transferItem(final Ds3Client client, final BulkObject ds3Object) throws IOException { - final GetObjectRequest getObjectRequest = new GetObjectRequest( - ReadJobImpl.this.masterObjectList.getBucketName(), - ds3Object.getName(), - jobState.getChannel(ds3Object.getName(), destinationChannelOffset, RangeHelper.transferSizeForRanges(ranges)), - ReadJobImpl.this.getJobId().toString(), - ds3Object.getOffset() - ); - - getObjectRequest.withByteRanges(ranges); - - final GetObjectResponse response = client.getObject(getObjectRequest); - final Metadata metadata = response.getMetadata(); - - emitChecksumEvents(ds3Object, response.getChecksumType(), response.getChecksum()); - sendMetadataEvents(ds3Object.getName(), metadata); - } - } - - private final class GetObjectTransferrer implements ItemTransferrer { - private final JobState jobState; - - private GetObjectTransferrer(final JobState jobState) { - this.jobState = jobState; - } - - @Override - public void transferItem(final Ds3Client client, final BulkObject ds3Object) - throws IOException { - - final ImmutableCollection ranges = getRangesForBlob(blobToRanges, ds3Object); - - final GetObjectRequest request = new GetObjectRequest( - ReadJobImpl.this.masterObjectList.getBucketName(), - ds3Object.getName(), - jobState.getChannel(ds3Object.getName(), ds3Object.getOffset(), ds3Object.getLength()), - ReadJobImpl.this.getJobId().toString(), - ds3Object.getOffset() - ); - - if (Guard.isNotNullAndNotEmpty(ranges)) { - request.withByteRanges(ranges); - } - - final GetObjectResponse response = client.getObject(request); - final Metadata metadata = response.getMetadata(); - - emitChecksumEvents(ds3Object, response.getChecksumType(), response.getChecksum()); - sendMetadataEvents(ds3Object.getName(), metadata); - } - } + public void transfer(final ObjectChannelBuilder channelBuilder) throws IOException { + super.transfer(channelBuilder); - private void sendMetadataEvents(final String objName , final Metadata metadata) { - for (final MetadataReceivedListener listener : this.metadataListeners) { - getEventRunner().emitEvent(new Runnable() { - @Override - public void run() { - listener.metadataReceived(objName, metadata); - } - }); - } + transfer(transferStrategyBuilder().makeGetTransferStrategy()); } - private static ImmutableCollection getRangesForBlob( - final ImmutableMap> blobToRanges, - final BulkObject ds3Object) { - final ImmutableMultimap ranges = blobToRanges.get(ds3Object.getName()); - if (ranges == null) return null; - return ranges.get(ds3Object); + public void transfer(final TransferStrategy transferStrategy) throws IOException { + running = true; + transferStrategy.transfer(); } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java index 287be1d8d..897d60d37 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java @@ -15,58 +15,18 @@ package com.spectralogic.ds3client.helpers; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; -import com.spectralogic.ds3client.Ds3Client; -import com.spectralogic.ds3client.commands.PutObjectRequest; -import com.spectralogic.ds3client.helpers.ChunkTransferrer.ItemTransferrer; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; -import com.spectralogic.ds3client.helpers.events.EventRunner; -import com.spectralogic.ds3client.helpers.events.FailureEvent; -import com.spectralogic.ds3client.helpers.strategy.BlobStrategy; -import com.spectralogic.ds3client.helpers.strategy.PutStreamerStrategy; -import com.spectralogic.ds3client.models.*; -import com.spectralogic.ds3client.models.Objects; -import com.spectralogic.ds3client.models.common.Range; -import com.spectralogic.ds3client.utils.Guard; -import com.spectralogic.ds3client.utils.SeekableByteChannelInputStream; -import com.spectralogic.ds3client.utils.hashing.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.SeekableByteChannel; -import java.util.*; - -import static com.spectralogic.ds3client.helpers.strategy.StrategyUtils.filterChunks; - -class WriteJobImpl extends JobImpl { - static private final Logger LOG = LoggerFactory.getLogger(WriteJobImpl.class); - - private final List filteredChunks; - private final ChecksumType.Type checksumType; - - private final int retryAfter; // Negative retryAfter value represent infinity retries - private final int retryDelay; //Negative value means use default +public class WriteJobImpl extends JobImpl { private MetadataAccess metadataAccess = null; private ChecksumFunction checksumFunction = null; - public WriteJobImpl( - final Ds3Client client, - final MasterObjectList masterObjectList, - final int retryAfter, - final ChecksumType.Type type, - final int objectTransferAttempts, - final int retryDelay, - final EventRunner eventRunner) { - super(client, masterObjectList, objectTransferAttempts, eventRunner); - this.filteredChunks = getChunks(masterObjectList); - this.retryAfter = retryAfter; - this.retryDelay = retryDelay; - - this.checksumType = type; + public WriteJobImpl(final TransferStrategyBuilder transferStrategyBuilder) { + super(transferStrategyBuilder); } @Override @@ -93,180 +53,18 @@ public Ds3ClientHelpers.Job withChecksum(final ChecksumFunction checksumFunction } @Override - public void transfer(final ObjectChannelBuilder channelBuilder) - throws IOException { - try { - running = true; - LOG.debug("Starting job transfer"); - if (this.masterObjectList == null || this.masterObjectList.getObjects() == null) { - LOG.info("There is nothing to transfer for job" - + (this.getJobId() == null ? "" : " " + this.getJobId().toString())); - return; - } - - final BlobStrategy blobStrategy = new PutStreamerStrategy(client, - this.masterObjectList, - retryAfter, - retryDelay, - new BlobStrategy.ChunkEventHandler() { - @Override - public void emitWaitingForChunksEvents(final int secondsToDelay) { - WriteJobImpl.super.emitWaitingForChunksEvents(secondsToDelay); - } - }); - - try (final JobState jobState = new JobState( - channelBuilder, - filteredChunks, - getJobPartTracker(), - ImmutableMap.>of())) { - try (final ChunkTransferrer chunkTransferrer = new ChunkTransferrer( - new PutObjectTransferrerRetryDecorator(jobState), - jobState.getPartTracker(), - this.maxParallelRequests - )) { - while (jobState.hasObjects()) { - chunkTransferrer.transferChunks(blobStrategy); - } - } - - } catch (final IOException | RuntimeException e) { - throw e; - } catch (final Exception e) { - throw new RuntimeException(e); - } - } catch (final Throwable t) { - emitFailureEvent(makeFailureEvent(FailureEvent.FailureActivity.PuttingObject, t, masterObjectList.getObjects().get(0))); - throw t; - } - } - - @Override - protected List getChunks(final MasterObjectList masterObjectList) { - if (masterObjectList == null || masterObjectList.getObjects() == null) { - LOG.info("Job has no data to transfer"); - return null; - } - - LOG.info("Ready to start transfer for job {} with {} chunks", masterObjectList.getJobId().toString(), masterObjectList.getObjects().size()); - return filterChunks(masterObjectList.getObjects()); - } - - @Override - protected JobPartTrackerDecorator makeJobPartTracker(final List chunks, final EventRunner eventRunner) { - return chunks == null ? null : new JobPartTrackerDecorator(chunks, eventRunner); - } + public void transfer(final ObjectChannelBuilder channelBuilder) throws IOException { + super.transfer(channelBuilder); - private final class PutObjectTransferrerRetryDecorator implements ItemTransferrer { - private final PutObjectTransferrer putObjectTransferrer; + transferStrategyBuilder().withChecksumFunction(checksumFunction); + transferStrategyBuilder().withMetadataAccess(metadataAccess); - private PutObjectTransferrerRetryDecorator(final JobState jobState) { - putObjectTransferrer = new PutObjectTransferrer(jobState); - } - - @Override - public void transferItem(final Ds3Client client, final BulkObject ds3Object) throws IOException { - WriteJobImpl.this.transferItem(client, ds3Object, putObjectTransferrer); - } + transfer(transferStrategyBuilder().makePutTransferStrategy()); } - private final class PutObjectTransferrer implements ItemTransferrer { - private final JobState jobState; - - private PutObjectTransferrer(final JobState jobState) { - this.jobState = jobState; - } - - @Override - public void transferItem(final Ds3Client client, final BulkObject ds3Object) - throws IOException { - client.putObject(createRequest(ds3Object)); - } - - private PutObjectRequest createRequest(final BulkObject ds3Object) throws IOException { - final SeekableByteChannel channel = jobState.getChannel(ds3Object.getName(), ds3Object.getOffset(), ds3Object.getLength()); - - final PutObjectRequest request = new PutObjectRequest( - WriteJobImpl.this.masterObjectList.getBucketName(), - ds3Object.getName(), - jobState.getChannel(ds3Object.getName(), ds3Object.getOffset(), ds3Object.getLength()), - WriteJobImpl.this.getJobId().toString(), - ds3Object.getOffset(), - ds3Object.getLength() - ); - - if (ds3Object.getOffset() == 0 && metadataAccess != null) { - final Map metadata = metadataAccess.getMetadataValue(ds3Object.getName()); - if (Guard.isMapNullOrEmpty(metadata)) return request; - final ImmutableMap immutableMetadata = ImmutableMap.copyOf(metadata); - for (final Map.Entry value : immutableMetadata.entrySet()) { - request.withMetaData(value.getKey(), value.getValue()); - } - } - - final String checksum = calculateChecksum(ds3Object, channel); - if (checksum != null) { - request.withChecksum(ChecksumType.value(checksum), WriteJobImpl.this.checksumType); - emitChecksumEvents(ds3Object, WriteJobImpl.this.checksumType, checksum); - } - - return request; - } - - private String calculateChecksum(final BulkObject ds3Object, final SeekableByteChannel channel) throws IOException { - if (WriteJobImpl.this.checksumType != ChecksumType.Type.NONE) { - if (WriteJobImpl.this.checksumFunction == null) { - LOG.info("Calculating {} checksum for blob: {}", WriteJobImpl.this.checksumType.toString(), ds3Object.toString()); - final SeekableByteChannelInputStream dataStream = new SeekableByteChannelInputStream(channel); - final Hasher hasher = getHasher(WriteJobImpl.this.checksumType); - final String checksum = hashInputStream(hasher, dataStream); - LOG.info("Computed checksum for blob: {}", checksum); - return checksum; - } else { - LOG.info("Getting checksum from user supplied callback for blob: {}", ds3Object.toString()); - final String checksum = WriteJobImpl.this.checksumFunction.compute(ds3Object, channel); - LOG.info("User supplied checksum is: {}", checksum); - return checksum; - } - } - return null; - } - - private static final int READ_BUFFER_SIZE = 10 * 1024 * 1024; - - private String hashInputStream(final Hasher digest, final InputStream stream) throws IOException { - final byte[] buffer = new byte[READ_BUFFER_SIZE]; - int bytesRead; - - while (true) { - bytesRead = stream.read(buffer); - - if (bytesRead < 0) { - break; - } - - digest.update(buffer, 0, bytesRead); - } - - return digest.digest(); - } - - private Hasher getHasher(final ChecksumType.Type checksumType) { - switch (checksumType) { - case MD5: - return new MD5Hasher(); - case SHA_256: - return new SHA256Hasher(); - case SHA_512: - return new SHA512Hasher(); - case CRC_32: - return new CRC32Hasher(); - case CRC_32C: - return new CRC32CHasher(); - default: - throw new RuntimeException("Unknown checksum type " + checksumType.toString()); - } - } + public void transfer(final TransferStrategy transferStrategy) throws IOException { + running = true; + transferStrategy.transfer(); } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java new file mode 100644 index 000000000..31d385062 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java @@ -0,0 +1,114 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.events; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.utils.Guard; + +public class FailureEvent { + private final FailureActivity doingWhat; + private final String objectName; + private final String endpoint; + private final Throwable causalException; + + private FailureEvent(final FailureActivity what, final String objectName, final String endpoint, final Throwable causalException) { + this.doingWhat = what; + this.objectName = objectName; + this.endpoint = endpoint; + this.causalException = causalException; + } + + public FailureActivity doingWhat() { + return doingWhat; + } + + public String withObjectNamed() { + return objectName; + } + + public String usingSystemWithEndpoint() { + return endpoint; + } + + public Throwable getCausalException() { + return causalException; + } + + @Override + public String toString() { + return "Failure " + doingWhat().getActivityText() + " with object named \"" + withObjectNamed() + "\" using system with endpoint " + usingSystemWithEndpoint(); + } + + public static FailureEvent.Builder builder() { + return new FailureEvent.Builder(); + } + + public enum FailureActivity { + PuttingObject("putting object"), + GettingObject("getting object"), + ComputingChecksum("computing checksum"), + RecordingMetadata("recording metadata"), + RestoringMetadata("restoring metadata"); + + private final String activityText; + + FailureActivity(final String activityText) { + this.activityText = activityText; + } + + public String getActivityText() { + return activityText; + } + } + + public static class Builder { + private FailureActivity doingWhat; + private String withObjectNamed; + private String usingSystemWithEndpoint; + private Throwable causalException; + + public Builder() { } + + public Builder doingWhat(final FailureActivity what) { + this.doingWhat = what; + return this; + } + + public Builder withObjectNamed(final String objectName) { + this.withObjectNamed = objectName; + return this; + } + + public Builder usingSystemWithEndpoint(final String endpoint) { + this.usingSystemWithEndpoint = endpoint; + return this; + } + + public Builder withCausalException(final Throwable causalException) { + this.causalException = causalException; + return this; + } + + public FailureEvent build() { + Preconditions.checkNotNull(doingWhat, "The failed activity may not be null or empty."); + Guard.throwOnNullOrEmptyString(withObjectNamed, "The name of the object involved in the activity may not be null or empty."); + Guard.throwOnNullOrEmptyString(usingSystemWithEndpoint, "The endpoint referenced in the activity may not be null or empty."); + Preconditions.checkNotNull(causalException, "The exception causing a failure may not be null."); + + return new FailureEvent(doingWhat, withObjectNamed, usingSystemWithEndpoint, causalException); + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/MetadataEvent.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/MetadataEvent.java new file mode 100644 index 000000000..cbe89fef2 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/MetadataEvent.java @@ -0,0 +1,36 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.events; + +import com.spectralogic.ds3client.networking.Metadata; + +public class MetadataEvent { + private final String objectName; + private final Metadata metadata; + + public MetadataEvent(final String objectName, final Metadata metadata) { + this.objectName = objectName; + this.metadata = metadata; + } + + public String getObjectName() { + return objectName; + } + + public Metadata getMetadata() { + return metadata; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/BlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/BlobStrategy.java deleted file mode 100644 index b20baefc2..000000000 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/BlobStrategy.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * ****************************************************************************** - * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.helpers.strategy; - -import com.spectralogic.ds3client.Ds3Client; -import com.spectralogic.ds3client.helpers.JobPart; -import com.spectralogic.ds3client.models.BulkObject; -import com.spectralogic.ds3client.models.MasterObjectList; - -import java.io.IOException; - -public abstract class BlobStrategy { - - private final Ds3Client client; - private final MasterObjectList masterObjectList; - private final int retryAfter; - private final int retryDelay; - private final BlobStrategy.ChunkEventHandler chunkEventHandler; - - public BlobStrategy(final Ds3Client client, final MasterObjectList masterObjectList, final int retryAfter, final int retryDelay, final BlobStrategy.ChunkEventHandler chunkEventHandler) { - this.client = client; - this.masterObjectList = masterObjectList; - this.retryAfter = retryAfter; - this.retryDelay = retryDelay; - this.chunkEventHandler = chunkEventHandler; - } - - public abstract Iterable getWork() throws IOException, InterruptedException; - public abstract void blobCompleted(final BulkObject bulkObject); - - public Ds3Client getClient() { - return client; - } - - public int getRetryAfter() { - return retryAfter; - } - - public int getRetryDelay() { - return retryDelay; - } - - public BlobStrategy.ChunkEventHandler getChunkEventHandler() { - return chunkEventHandler; - } - - public MasterObjectList getMasterObjectList() { - return masterObjectList; - } - - protected int computeDelay(final int retryAfterSeconds) { - final int retryDelay = getRetryDelay(); - if (retryDelay == -1) { - return retryAfterSeconds; - } else { - return retryDelay; - } - } - - public interface ChunkEventHandler { - void emitWaitingForChunksEvents(final int secondsToDelay); - } -} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java index 96f36d150..ab41b2127 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java @@ -15,12 +15,15 @@ package com.spectralogic.ds3client.helpers.strategy; +import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.models.BulkObject; import com.spectralogic.ds3client.models.JobNode; import com.spectralogic.ds3client.models.Objects; +import com.spectralogic.ds3client.models.common.Range; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,13 +71,13 @@ public static ImmutableMap buildNodeMap(final Iterable n /** * Filters out chunks that have already been completed. We will get the same chunk name back from the server, but it * will not have any objects in it, so we remove that from the list of objects that are returned. - * @param objectsList The list to be filtered + * @param chunks The list to be filtered * @return The filtered list */ - public static ImmutableList filterChunks(final Iterable objectsList) { + public static ImmutableList filterChunks(final Iterable chunks) { final ImmutableList.Builder builder = ImmutableList.builder(); - for (final Objects objects : objectsList) { - final Objects filteredChunk = filterChunk(objects); + for (final Objects chunk : chunks) { + final Objects filteredChunk = filterChunk(chunk); if (filteredChunk.getObjects().size() > 0) { builder.add(filteredChunk); } @@ -82,24 +85,33 @@ public static ImmutableList filterChunks(final Iterable object return builder.build(); } - private static Objects filterChunk(final Objects objects) { - final Objects newObjects = new Objects(); - newObjects.setChunkId(objects.getChunkId()); - newObjects.setChunkNumber(objects.getChunkNumber()); - newObjects.setNodeId(objects.getNodeId()); - newObjects.setObjects(filterObjects(objects.getObjects())); - return newObjects; + private static Objects filterChunk(final Objects chunk) { + final Objects newChunk = new Objects(); + newChunk.setChunkId(chunk.getChunkId()); + newChunk.setChunkNumber(chunk.getChunkNumber()); + newChunk.setNodeId(chunk.getNodeId()); + newChunk.setObjects(filterObjects(chunk.getObjects())); + return newChunk; } - private static ImmutableList filterObjects(final List list) { + private static ImmutableList filterObjects(final List blobs) { final ImmutableList.Builder builder = ImmutableList.builder(); - for (final BulkObject obj : list) { - if (!obj.getInCache()) { - builder.add(obj); + for (final BulkObject blob : blobs) { + if (!blob.getInCache()) { + builder.add(blob); } } return builder.build(); } + public static ImmutableCollection getRangesForBlob(final ImmutableMap> rangesForBlobs, + final BulkObject blob) { + final ImmutableMultimap rangesForBlob = rangesForBlobs.get(blob.getName()); + if (rangesForBlob != null) { + return rangesForBlob.get(blob); + } + + return null; + } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/AbstractBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/AbstractBlobStrategy.java new file mode 100644 index 000000000..450ba31dd --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/AbstractBlobStrategy.java @@ -0,0 +1,78 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.BlobTransferredEventObserver; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.UpdateStrategy; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.MasterObjectList; + +public abstract class AbstractBlobStrategy implements BlobStrategy { + + private final Ds3Client client; + private final MasterObjectList masterObjectList; + private final EventDispatcher eventDispatcher; + private final ChunkAttemptRetryBehavior retryBehavior; + private final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior; + + public AbstractBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher, + final ChunkAttemptRetryBehavior retryBehavior, + final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior) + { + this.client = client; + this.masterObjectList = masterObjectList; + this.eventDispatcher = eventDispatcher; + this.retryBehavior = retryBehavior; + this.chunkAttemptRetryDelayBehavior = chunkAttemptRetryDelayBehavior; + + eventDispatcher.attachBlobTransferredEventObserver(new BlobTransferredEventObserver(new UpdateStrategy() { + @Override + public void update(final BulkObject eventData) { + blobCompleted(eventData); + } + })); + } + + /** + * Emit an event when a blob is transferred. + * @param bulkObject The transferred blob. + */ + public abstract void blobCompleted(final BulkObject bulkObject); + + public Ds3Client client() { + return client; + } + + public EventDispatcher eventDispatcher() { + return eventDispatcher; + } + + public MasterObjectList masterObjectList() { + return masterObjectList; + } + + public ChunkAttemptRetryBehavior retryBehavior() { + return retryBehavior; + } + + public ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior() { + return chunkAttemptRetryDelayBehavior; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlackPearlChunkAttemptRetryDelayBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlackPearlChunkAttemptRetryDelayBehavior.java new file mode 100644 index 000000000..bd4c3aa8c --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlackPearlChunkAttemptRetryDelayBehavior.java @@ -0,0 +1,43 @@ +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An implementation of {@link ChunkAttemptRetryDelayBehavior} that uses the delay a Black Pearl returns + * in its payload when a chunk operation does not complete. + */ +public class BlackPearlChunkAttemptRetryDelayBehavior implements ChunkAttemptRetryDelayBehavior { + private static final Logger LOG = LoggerFactory.getLogger(BlackPearlChunkAttemptRetryDelayBehavior.class); + + private final EventDispatcher eventDispatcher; + + /** + * @param eventDispatcher An instance of {@link EventDispatcher} used to send {@link com.spectralogic.ds3client.helpers.WaitingForChunksListener} + * events. + */ + public BlackPearlChunkAttemptRetryDelayBehavior(final EventDispatcher eventDispatcher) { + Preconditions.checkNotNull(eventDispatcher, "eventDispatcher may not be null."); + + this.eventDispatcher = eventDispatcher; + } + + /** + * Delay for the amount of time a Black Pearl specifies in the payload it returns when a chunk operation does + * not complete. + * @param delayIntervalInSeconds The delay in seconds, between chunk operation retry attempts, that comes from a Black Pearl payload returned + * when a chunk operation does not complete. + * @throws InterruptedException + */ + @Override + public void delay(final int delayIntervalInSeconds) throws InterruptedException { + LOG.debug("Will retry allocate chunk call after {} seconds", delayIntervalInSeconds); + + eventDispatcher.emitWaitingForChunksEvents(delayIntervalInSeconds); + + Thread.sleep(delayIntervalInSeconds * 1000); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategy.java new file mode 100644 index 000000000..ec1ad0848 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategy.java @@ -0,0 +1,35 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; + +import java.io.IOException; + +/** + * An interface whose concrete implementation is used in chunk retrieval or allocation. When using streamed behavior, + * {@link TransferStrategyBuilder#usingStreamedTransferBehavior()}, blobs are retrieved sequentially, from the beginning + * of an object's range. + */ +public interface BlobStrategy { + /** + * @return An iterable listing the blobs and the client(s) those blobs reside on when doing a put or get. + * @throws IOException + * @throws InterruptedException + */ + Iterable getWork() throws IOException, InterruptedException; +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategyMaker.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategyMaker.java new file mode 100644 index 000000000..2dfde31e5 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategyMaker.java @@ -0,0 +1,30 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; +import com.spectralogic.ds3client.models.MasterObjectList; + +/** + * An interface used as an intermediate when building a {@link com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy} + * instance as part of getting a transfer strategy from a {@link com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder}. + */ +public interface BlobStrategyMaker { + BlobStrategy makeBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher); +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryBehavior.java new file mode 100644 index 000000000..b2be8250f --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryBehavior.java @@ -0,0 +1,35 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +import java.io.IOException; + +/** + * An interface whose concrete subclasses determine the retry behavior when a chunk + * operation does not complete. + */ +public interface ChunkAttemptRetryBehavior { + /** + * Perform a retry. + * @throws IOException + */ + void invoke() throws IOException; + + /** + * A chunk operation succeeded; reset the retry logic. + */ + void reset(); +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryDelayBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryDelayBehavior.java new file mode 100644 index 000000000..c70a886cc --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryDelayBehavior.java @@ -0,0 +1,29 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +/** + * An interface whose concrete subclasses determine the delay between retries when a + * chunk operation does not complete. + */ +public interface ChunkAttemptRetryDelayBehavior { + /** + * @param delayIntervalInSeconds The delay in seconds, between chunk operation retry attempts, that comes from a Black Pearl payload returned + * when a chunk operation does not complete. + * @throws InterruptedException + */ + void delay(final int delayIntervalInSeconds) throws InterruptedException; +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ClientDefinedChunkAttemptRetryDelayBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ClientDefinedChunkAttemptRetryDelayBehavior.java new file mode 100644 index 000000000..81d031d88 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ClientDefinedChunkAttemptRetryDelayBehavior.java @@ -0,0 +1,51 @@ +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An implementation of {@link ChunkAttemptRetryDelayBehavior} that uses the delay a client specifies, + * as opposed to using the delay a Black Pearl returns in its payload when a chunk operation does not complete. + */ +public class ClientDefinedChunkAttemptRetryDelayBehavior implements ChunkAttemptRetryDelayBehavior { + private static final Logger LOG = LoggerFactory.getLogger(ClientDefinedChunkAttemptRetryDelayBehavior.class); + + private final int clientDefinedDelayInSeconds; + private final EventDispatcher eventDispatcher; + + /** + * @param clientDefinedDelayInSeconds A user-specified delay in seconds between retry attempts when a chunk + * operation does not complete. + * @param eventDispatcher + */ + public ClientDefinedChunkAttemptRetryDelayBehavior(final int clientDefinedDelayInSeconds, + final EventDispatcher eventDispatcher) + { + Preconditions.checkState(clientDefinedDelayInSeconds >= 0, "clientDefinedDelayInSeconds must be >= 0"); + Preconditions.checkNotNull(eventDispatcher, "eventDispatcher may not be null"); + + this.clientDefinedDelayInSeconds = clientDefinedDelayInSeconds; + this.eventDispatcher = eventDispatcher; + } + + /** + * @param delayIntervalInSeconds The delay in seconds, between chunk operation retry attempts, that comes from a Black Pearl payload returned + * when a chunk operation does not complete. This {@link ChunkAttemptRetryDelayBehavior} + * subclass will use the {@code clientDefinedDelayInSeconds} as the delay, and not the + * delay the Black Pearl specifies. + * @throws InterruptedException + */ + @Override + public void delay(final int delayIntervalInSeconds) throws InterruptedException { + LOG.debug("Will retry allocate chunk call after {} seconds", clientDefinedDelayInSeconds); + + // Even though the BP is telling us how long to wait before trying to allocate a chunk again, + // we'll use the delay time the client has specified. + eventDispatcher.emitWaitingForChunksEvents(clientDefinedDelayInSeconds); + + Thread.sleep(clientDefinedDelayInSeconds * 1000); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ContinueForeverChunkAttemptsRetryBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ContinueForeverChunkAttemptsRetryBehavior.java new file mode 100644 index 000000000..b917b683e --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ContinueForeverChunkAttemptsRetryBehavior.java @@ -0,0 +1,34 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +import java.io.IOException; + +/** + * An implementation of {@link ChunkAttemptRetryBehavior} that will continue to retry an uncompleted + * chunk operation until that operation succeeds. + */ +public class ContinueForeverChunkAttemptsRetryBehavior implements ChunkAttemptRetryBehavior { + @Override + public void invoke() throws IOException { + // intentionally not implemented + } + + @Override + public void reset() { + // intentionally not implemented + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/GetStreamerStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java similarity index 79% rename from ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/GetStreamerStrategy.java rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java index 79141c35b..63742c4f9 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/GetStreamerStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java @@ -13,22 +13,19 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.helpers.strategy; +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; import com.google.common.base.Function; import com.google.common.base.Predicate; -import com.google.common.base.Predicates; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.commands.spectrads3.GetJobChunksReadyForClientProcessingSpectraS3Request; import com.spectralogic.ds3client.commands.spectrads3.GetJobChunksReadyForClientProcessingSpectraS3Response; -import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException; import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; import com.spectralogic.ds3client.models.BulkObject; -import com.spectralogic.ds3client.models.JobNode; import com.spectralogic.ds3client.models.MasterObjectList; import com.spectralogic.ds3client.models.Objects; @@ -38,16 +35,23 @@ import java.util.Set; import java.util.UUID; -public class GetStreamerStrategy extends BlobStrategy { - +/** + * A subclass of {@link BlobStrategy} used in get transfers. + */ +public class GetSequentialBlobStrategy extends AbstractBlobStrategy { private final Set processedChunks; private final Object lock = new Object(); private final Set activeBlobs = new HashSet<>(); private ImmutableList outstandingJobParts; - public GetStreamerStrategy(final Ds3Client client, final MasterObjectList masterObjectList, final int retryAfter, final int retryDelay, final BlobStrategy.ChunkEventHandler chunkEventHandler) { - super(client, masterObjectList, retryAfter, retryDelay, chunkEventHandler); + public GetSequentialBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher, + final ChunkAttemptRetryBehavior retryBehavior, + final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior) + { + super(client, masterObjectList, eventDispatcher, retryBehavior, chunkAttemptRetryDelayBehavior); this.processedChunks = new HashSet<>(); } @@ -56,8 +60,6 @@ public Iterable getWork() throws IOException, InterruptedException { final MasterObjectList available = getAvailable(); - final ImmutableMap jobNodes = StrategyUtils.buildNodeMap(available.getNodes()); - // filter any chunks that have been processed final FluentIterable chunks = FluentIterable.from(available.getObjects()); @@ -76,7 +78,7 @@ public Iterable apply(@Nullable final Objects objects) { @Nullable @Override public JobPart apply(@Nullable final BulkObject blob) { - return new JobPart(getClient(), blob); + return new JobPart(client(), blob); // TODO: When we get to the point where BP enables clustering, we'll want to be able to get the // client connection info correct for the server on which a chunk resides. StrategyUtils.getClient @@ -112,37 +114,21 @@ public boolean apply(@Nullable final JobPart input) { return nextWorkParts; } - private Iterable getNextIterable(final FluentIterable jobParts) { - if (outstandingJobParts == null) { - return jobParts; - } - return Iterables.concat(outstandingJobParts, jobParts); - } - - @Override - public void blobCompleted(final BulkObject bulkObject) { - synchronized (lock) { - activeBlobs.remove(bulkObject.getName()); - } - } - private MasterObjectList getAvailable() throws IOException, InterruptedException { - int retryAfterLeft = getRetryAfter(); do { final GetJobChunksReadyForClientProcessingSpectraS3Response availableJobChunks = - getClient().getJobChunksReadyForClientProcessingSpectraS3(new GetJobChunksReadyForClientProcessingSpectraS3Request(getMasterObjectList().getJobId().toString())); + client().getJobChunksReadyForClientProcessingSpectraS3(new GetJobChunksReadyForClientProcessingSpectraS3Request(masterObjectList().getJobId().toString())); + switch (availableJobChunks.getStatus()) { case AVAILABLE: { + retryBehavior().reset(); return availableJobChunks.getMasterObjectListResult(); } case RETRYLATER: { - if (retryAfterLeft == 0) { - throw new Ds3NoMoreRetriesException(getRetryAfter()); - } - retryAfterLeft--; - final int secondsToDelay = computeDelay(availableJobChunks.getRetryAfterSeconds()); - getChunkEventHandler().emitWaitingForChunksEvents(secondsToDelay); - Thread.sleep(secondsToDelay * 1000); + retryBehavior().invoke(); + + chunkAttemptRetryDelayBehavior().delay(availableJobChunks.getRetryAfterSeconds()); + continue; } default: @@ -151,4 +137,21 @@ private MasterObjectList getAvailable() throws IOException, InterruptedException } while(true); } + private Iterable getNextIterable(final FluentIterable jobParts) { + if (outstandingJobParts == null) { + return jobParts; + } + return Iterables.concat(outstandingJobParts, jobParts); + } + + /** + * Emit an event when a blob is transferred. + * @param bulkObject The transferred blob. + */ + @Override + public void blobCompleted(final BulkObject bulkObject) { + synchronized (lock) { + activeBlobs.remove(bulkObject.getName()); + } + } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/MaxChunkAttemptsRetryBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/MaxChunkAttemptsRetryBehavior.java new file mode 100644 index 000000000..c9d832597 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/MaxChunkAttemptsRetryBehavior.java @@ -0,0 +1,31 @@ +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; + +import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A subclass of {@link ChunkAttemptRetryBehavior} that will retry an uncompleted + * chunk operation at most {@code maxAttempts} before giving up. + */ +public class MaxChunkAttemptsRetryBehavior implements ChunkAttemptRetryBehavior { + private final int maxAttempts; + private final AtomicInteger currentAttempt = new AtomicInteger(0); + + public MaxChunkAttemptsRetryBehavior(final int maxAttempts) { + this.maxAttempts = maxAttempts; + } + + @Override + public void invoke() throws IOException { + if (currentAttempt.incrementAndGet() > maxAttempts) { + throw new Ds3NoMoreRetriesException(maxAttempts); + } + } + + @Override + public void reset() { + currentAttempt.set(0); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/PutStreamerStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java similarity index 71% rename from ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/PutStreamerStrategy.java rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java index 4a349b098..35312e02d 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/PutStreamerStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java @@ -13,7 +13,7 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.helpers.strategy; +package com.spectralogic.ds3client.helpers.strategy.blobstrategy; import com.google.common.base.Function; import com.google.common.base.Predicate; @@ -22,8 +22,8 @@ import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.commands.spectrads3.AllocateJobChunkSpectraS3Request; import com.spectralogic.ds3client.commands.spectrads3.AllocateJobChunkSpectraS3Response; -import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException; import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; import com.spectralogic.ds3client.models.BulkObject; import com.spectralogic.ds3client.models.JobNode; import com.spectralogic.ds3client.models.MasterObjectList; @@ -39,19 +39,22 @@ import static com.spectralogic.ds3client.helpers.strategy.StrategyUtils.buildNodeMap; import static com.spectralogic.ds3client.helpers.strategy.StrategyUtils.filterChunks; -public class PutStreamerStrategy extends BlobStrategy { - - private final static Logger LOG = LoggerFactory.getLogger(PutStreamerStrategy.class); +/** + * A subclass of {@link BlobStrategy} used in put transfers. + */ +public class PutSequentialBlobStrategy extends AbstractBlobStrategy { + private final static Logger LOG = LoggerFactory.getLogger(PutSequentialBlobStrategy.class); private final ImmutableMap uuidJobNodeImmutableMap; private final Iterator filteredChunkIterator; - private int retryAfterLeft; - - - - public PutStreamerStrategy(final Ds3Client client, final MasterObjectList masterObjectList, final int retryAfter, final int retryDelay, final ChunkEventHandler chunkEventHandler) { - super(client, masterObjectList, retryAfter, retryDelay, chunkEventHandler); + public PutSequentialBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher, + final ChunkAttemptRetryBehavior retryBehavior, + final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior) + { + super(client, masterObjectList, eventDispatcher, retryBehavior, chunkAttemptRetryDelayBehavior); this.filteredChunkIterator = filterChunks(masterObjectList.getObjects()).iterator(); this.uuidJobNodeImmutableMap = buildNodeMap(masterObjectList.getNodes()); } @@ -70,7 +73,7 @@ public boolean apply(@Nullable final BulkObject input) { @Nullable @Override public JobPart apply(@Nullable final BulkObject blob) { - return new JobPart(getClient(), blob); + return new JobPart(client(), blob); // TODO: When we get to the point where BP enables clustering, we'll want to be able to get the // client connection info correct for the server on which a chunk resides. StrategyUtils.getClient @@ -82,11 +85,6 @@ public JobPart apply(@Nullable final BulkObject blob) { }); } - @Override - public void blobCompleted(final BulkObject bulkObject) { - - } - private Objects allocateChunk(final Objects filtered) throws IOException { Objects chunk = null; while (chunk == null) { @@ -97,31 +95,31 @@ private Objects allocateChunk(final Objects filtered) throws IOException { private Objects tryAllocateChunk(final Objects filtered) throws IOException { final AllocateJobChunkSpectraS3Response response = - getClient().allocateJobChunkSpectraS3(new AllocateJobChunkSpectraS3Request(filtered.getChunkId().toString())); + client().allocateJobChunkSpectraS3(new AllocateJobChunkSpectraS3Request(filtered.getChunkId().toString())); LOG.info("AllocatedJobChunkResponse status: {}", response.getStatus().toString()); + switch (response.getStatus()) { - case ALLOCATED: - retryAfterLeft = getRetryAfter(); // Reset the number of retries to the initial value - return response.getObjectsResult(); - case RETRYLATER: - try { - if (getRetryAfter() != -1 && retryAfterLeft == 0) { - throw new Ds3NoMoreRetriesException(getRetryAfter()); + case ALLOCATED: + retryBehavior().reset(); + return response.getObjectsResult(); + case RETRYLATER: + retryBehavior().invoke(); + + try { + chunkAttemptRetryDelayBehavior().delay(response.getRetryAfterSeconds()); + } catch (final InterruptedException e) { + throw new RuntimeException(e); } - retryAfterLeft--; - - final int retryDelay = computeDelay(response.getRetryAfterSeconds()); - getChunkEventHandler().emitWaitingForChunksEvents(retryDelay); - LOG.debug("Will retry allocate chunk call after {} seconds", retryDelay); - Thread.sleep(retryDelay * 1000); return null; - } catch (final InterruptedException e) { - throw new RuntimeException(e); - } - default: - assert false : "This line of code should be impossible to hit."; return null; + default: + assert false : "This line of code should be impossible to hit."; return null; } } + + @Override + public void blobCompleted(final BulkObject bulkObject) { + + } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelPreparable.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelPreparable.java new file mode 100644 index 000000000..aaec19ebb --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelPreparable.java @@ -0,0 +1,36 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; + +import java.io.IOException; + +/** + * When initiating data movement in a put or get, the source or destination channel my need some sort of + * preparation. For instance, when initiating a get, the destination channel may need to be truncated + * prior to moving data. + */ +public interface ChannelPreparable { + /** + * Prepare a channel prior to moving data. + * @param channelName The channel's identifier. + * @param channelBuilder The instance of {@link com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder} + * used to create a channel. + * @throws IOException + */ + void prepareChannel(final String channelName, final Ds3ClientHelpers.ObjectChannelBuilder channelBuilder) throws IOException; +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelStrategy.java new file mode 100644 index 000000000..965d455d9 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelStrategy.java @@ -0,0 +1,46 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.spectralogic.ds3client.models.BulkObject; + +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; + +/** + * An interface whose concrete subclasses specify the behavior used to read or write a channel. + */ +public interface ChannelStrategy { + /** + * For a blob to be transferred, create a channel that will be the source for or destination + * of that transferred blob. + * @param blob The blob to be transferred. + * @return A {@link SeekableByteChannel} that will be the source for or destination + * of that transferred blob. + * @throws IOException + */ + SeekableByteChannel acquireChannelForBlob(final BulkObject blob) throws IOException; + + /** + * When a blob has been transferred, release the source or destination channel associated with that + * blob. + * @param seekableByteChannel A {@link SeekableByteChannel} that had been allocated as the source for or destination + * of a blob transfer. + * @param blob The blob {@code seekableByteChannel} was allocated to source or sink {@code blob}'s data. + * @throws IOException + */ + void releaseChannelForBlob(final SeekableByteChannel seekableByteChannel, final BulkObject blob) throws IOException; +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/NullChannelPreparable.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/NullChannelPreparable.java new file mode 100644 index 000000000..a3ac53908 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/NullChannelPreparable.java @@ -0,0 +1,40 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; + +import java.io.IOException; + +/** + * When initiating data movement in a put or get, the source or destination channel my need some sort of + * preparation. For instance, when initiating a get, the destination channel may need to be truncated + * prior to moving data. Use an instance of this class when the channel needs no preparation. Essentially, + * this class exists to provide a uniform way to create channels. + */ +public class NullChannelPreparable implements ChannelPreparable { + /** + * Prepare a channel prior to moving data. + * @param channelName The channel's identifier. + * @param channelBuilder The instance of {@link com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder} + * used to create a channel. + * @throws IOException + */ + @Override + public void prepareChannel(final String channelName, final Ds3ClientHelpers.ObjectChannelBuilder channelBuilder) throws IOException { + // Intentionally not implemented + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/RandomAccessChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/RandomAccessChannelStrategy.java new file mode 100644 index 000000000..87c4da13d --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/RandomAccessChannelStrategy.java @@ -0,0 +1,139 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; +import com.spectralogic.ds3client.helpers.channels.RangedSeekableByteChannel; +import com.spectralogic.ds3client.helpers.channels.WindowedSeekableByteChannel; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.common.Range; +import com.spectralogic.ds3client.utils.Guard; + +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; +import java.util.HashMap; +import java.util.Map; + +/** + * A subclass of {@link ChannelStrategy} that creates a channel per blob positioned at the offset + * correct for the current blob offset. + */ +public class RandomAccessChannelStrategy implements ChannelStrategy { + private final Object lock = new Object(); + + private final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder; + private final ImmutableMap> rangesForBlobs; + private final ChannelPreparable channelPreparer; + private final Map blobChannelMap; + + /** + * @param objectChannelBuilder An instance of {@link com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder}; + * usually an instance of {@link com.spectralogic.ds3client.helpers.FileObjectPutter} + * or {@link com.spectralogic.ds3client.helpers.FileObjectGetter}. + * @param rangesForBlobs A map that associates a blob's name with the {@link Range}(s) transfered as part of a get. This + * parameter may be null. + * @param channelPreparer An instance of {@link ChannelPreparable} used to prepare a channel prior to moving data, most likely + * either {@link TruncatingChannelPreparable} or {@link NullChannelPreparable}. + */ + public RandomAccessChannelStrategy(final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder, + final ImmutableMap> rangesForBlobs, + final ChannelPreparable channelPreparer) + { + Preconditions.checkNotNull(objectChannelBuilder, "objectChannelBuilder may not be null."); + Preconditions.checkNotNull(channelPreparer, "channelPreparer may not be null."); + + this.objectChannelBuilder = objectChannelBuilder; + this.rangesForBlobs = rangesForBlobs; + this.channelPreparer = channelPreparer; + blobChannelMap = new HashMap<>(); + } + + /** + * For a blob to be transferred, create a channel that will be the source for or destination + * of that transferred blob. + * @param blob The blob to be transferred. + * @return A {@link SeekableByteChannel} that will be the source for or destination + * of that transferred blob. + * @throws IOException + */ + @Override + public SeekableByteChannel acquireChannelForBlob(final BulkObject blob) throws IOException { + synchronized (lock) { + channelPreparer.prepareChannel(blob.getName(), objectChannelBuilder); + + SeekableByteChannel seekableByteChannel = blobChannelMap.get(blob); + + if (seekableByteChannel == null) { + seekableByteChannel = new RangedSeekableByteChannel(objectChannelBuilder.buildChannel(blob.getName()), + getRangesForABlob(blob), + blob.getName()); + blobChannelMap.put(blob, seekableByteChannel); + } + + return new WindowedSeekableByteChannel(seekableByteChannel, lock, blob.getOffset(), blob.getLength()); + } + } + + private ImmutableMultimap getRangesForABlob(final BulkObject blob) { + if (Guard.isMapNullOrEmpty(rangesForBlobs)) { + return ImmutableMultimap.of(); + } + + final ImmutableMultimap rangesForABlob = rangesForBlobs.get(blob.getName()); + + if (Guard.isMultiMapNullOrEmpty(rangesForABlob)) { + return ImmutableMultimap.of(); + } + + return rangesForABlob; + } + + /** + * When a blob has been transferred, release the source or destination channel associated with that + * blob. + * @param seekableByteChannel A {@link SeekableByteChannel} that had been allocated as the source for or destination + * of a blob transfer. + * @param blob The blob {@code seekableByteChannel} was allocated to source or sink {@code blob}'s data. + * @throws IOException + */ + @Override + public void releaseChannelForBlob(final SeekableByteChannel seekableByteChannel, final BulkObject blob) throws IOException { + synchronized (lock) { + final SeekableByteChannel cachedChannel = blobChannelMap.remove(blob); + + Throwable closingCachedChannelException = null; + + try { + cachedChannel.close(); + } catch (final Throwable t) { + closingCachedChannelException = t; + } + + try { + seekableByteChannel.close(); + } catch (final Throwable t) { + throw new IOException(t); + } + + if (closingCachedChannelException != null) { + throw new IOException(closingCachedChannelException); + } + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java new file mode 100644 index 000000000..1ed73bd5a --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java @@ -0,0 +1,95 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.google.common.base.Preconditions; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; + +/** + * An instance of {@link SeekableByteChannel} used to decorate another SeekableByteChannel in the + * situation where we re-use the same channel for more than 1 blob. This subclass prevents closing + * a channel when there are other blobs still referencing the shared channel. + */ +class SeekableByteChannelDecorator implements SeekableByteChannel { + private final SeekableByteChannel seekableByteChannel; + + SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel) { + Preconditions.checkNotNull(seekableByteChannel, "seekableByteChannel may not be null"); + this.seekableByteChannel = seekableByteChannel; + } + + SeekableByteChannel wrappedSeekableByteChannel() { + return seekableByteChannel; + } + + @Override + public int read(final ByteBuffer dst) throws IOException { + return seekableByteChannel.read(dst); + } + + @Override + public int write(final ByteBuffer src) throws IOException { + return seekableByteChannel.write(src); + } + + @Override + public long position() throws IOException { + return seekableByteChannel.position(); + } + + @Override + public SeekableByteChannel position(final long newPosition) throws IOException { + return seekableByteChannel.position(newPosition); + } + + @Override + public long size() throws IOException { + return seekableByteChannel.size(); + } + + @Override + public SeekableByteChannel truncate(final long size) throws IOException { + return seekableByteChannel.truncate(size); + } + + @Override + public boolean isOpen() { + return seekableByteChannel.isOpen(); + } + + @Override + public void close() throws IOException { + // Intentionally not implemented + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (!(o instanceof SeekableByteChannelDecorator)) return false; + + final SeekableByteChannelDecorator that = (SeekableByteChannelDecorator) o; + + return seekableByteChannel.equals(that.seekableByteChannel); + } + + @Override + public int hashCode() { + return seekableByteChannel.hashCode(); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java new file mode 100644 index 000000000..06e441c49 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java @@ -0,0 +1,118 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.SetMultimap; +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; +import com.spectralogic.ds3client.models.BulkObject; + +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; +import java.util.HashMap; +import java.util.Map; + +/** + * A subclass of {@link ChannelStrategy} used to provide "streamed" access to a channel, + * where streamed means that the channel will be accessed in sequentially increasing offsets. + * This class acts as an aggregator that references 1 underlying channel for any number of blobs that + * reference the channel. + */ +public class SequentialChannelStrategy implements ChannelStrategy { + private final Object lock = new Object(); + + private final SetMultimap blobNameOffsetMap = HashMultimap.create(); + private final Map blobNameChannelMap = new HashMap<>(); + + private final ChannelStrategy channelStrategyDelegate; + private final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder; + private final ChannelPreparable channelPreparer; + + /** + * @param channelStrategy The instance of {@link ChannelStrategy} that holds the 1 channel reference a blob needs + * to transfer data. + * @param objectChannelBuilder An instance of {@link com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder}; + * usually an instance of {@link com.spectralogic.ds3client.helpers.FileObjectPutter} + * or {@link com.spectralogic.ds3client.helpers.FileObjectGetter}. + * @param channelPreparer An instance of {@link ChannelPreparable} used to prepare a channel prior to moving data, most likely + * either {@link TruncatingChannelPreparable} or {@link NullChannelPreparable}. + */ + public SequentialChannelStrategy(final ChannelStrategy channelStrategy, + final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder, + final ChannelPreparable channelPreparer) + { + channelStrategyDelegate = channelStrategy; + this.objectChannelBuilder = objectChannelBuilder; + this.channelPreparer = channelPreparer; + } + + /** + * For a blob to be transferred, create a channel that will be the source for or destination + * of that transferred blob. + * @param blob The blob to be transferred. + * @return A {@link SeekableByteChannel} that will be the source for or destination + * of that transferred blob. + * @throws IOException + */ + @Override + public SeekableByteChannel acquireChannelForBlob(final BulkObject blob) throws IOException { + synchronized (lock) { + final String blobName = blob.getName(); + + blobNameOffsetMap.put(blobName, blob.getOffset()); + + final SeekableByteChannel seekableByteChannel = blobNameChannelMap.get(blobName); + + if (seekableByteChannel != null) { + return seekableByteChannel; + } + + return makeNewChannel(blob); + } + } + + private SeekableByteChannel makeNewChannel(final BulkObject blob) throws IOException { + channelPreparer.prepareChannel(blob.getName(), objectChannelBuilder); + + final SeekableByteChannel seekableByteChannel = channelStrategyDelegate.acquireChannelForBlob(blob); + final SeekableByteChannelDecorator seekableByteChannelDecorator = new SeekableByteChannelDecorator(seekableByteChannel); + + blobNameChannelMap.put(blob.getName(), seekableByteChannelDecorator); + return seekableByteChannelDecorator; + } + + /** + * When a blob has been transferred, release the source or destination channel associated with that + * blob. + * @param seekableByteChannel A {@link SeekableByteChannel} that had been allocated as the source for or destination + * of a blob transfer. + * @param blob The blob {@code seekableByteChannel} was allocated to source or sink {@code blob}'s data. + * @throws IOException + */ + @Override + public void releaseChannelForBlob(final SeekableByteChannel seekableByteChannel, final BulkObject blob) throws IOException { + synchronized (lock) { + final String blobName = blob.getName(); + + blobNameOffsetMap.remove(blobName, blob.getOffset()); + + if (blobNameOffsetMap.get(blobName).size() == 0) { + blobNameChannelMap.remove(blobName); + channelStrategyDelegate.releaseChannelForBlob(((SeekableByteChannelDecorator)seekableByteChannel).wrappedSeekableByteChannel(), blob); + } + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileReaderChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileReaderChannelStrategy.java new file mode 100644 index 000000000..e10a07d86 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileReaderChannelStrategy.java @@ -0,0 +1,68 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; +import com.spectralogic.ds3client.models.BulkObject; +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; + +/** + * A subclass of {@link ChannelStrategy} that holds the instance of a channel a blob will during + * a put operation. + */ +public class SequentialFileReaderChannelStrategy implements ChannelStrategy { + private final Object lock = new Object(); + private final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder; + + /** + * @param objectChannelBuilder An instance of {@link com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder}; + * usually an instance of {@link com.spectralogic.ds3client.helpers.FileObjectPutter}. + */ + public SequentialFileReaderChannelStrategy(final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder) { + this.objectChannelBuilder = objectChannelBuilder; + } + + /** + * For a blob to be transferred, create a channel that will be the source for or destination + * of that transferred blob. + * @param blob The blob to be transferred. + * @return A {@link SeekableByteChannel} that will be the source for or destination + * of that transferred blob. + * @throws IOException + */ + @Override + public SeekableByteChannel acquireChannelForBlob(final BulkObject blob) throws IOException { + synchronized (lock) { + return objectChannelBuilder.buildChannel(blob.getName()); + } + } + + /** + * When a blob has been transferred, release the source or destination channel associated with that + * blob. + * @param seekableByteChannel A {@link SeekableByteChannel} that had been allocated as the source for or destination + * of a blob transfer. + * @param blob The blob {@code seekableByteChannel} was allocated to source or sink {@code blob}'s data. + * @throws IOException + */ + @Override + public void releaseChannelForBlob(final SeekableByteChannel seekableByteChannel, final BulkObject blob) throws IOException { + synchronized (lock) { + seekableByteChannel.close(); + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileWriterChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileWriterChannelStrategy.java new file mode 100644 index 000000000..2122aa1f8 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileWriterChannelStrategy.java @@ -0,0 +1,69 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; +import com.spectralogic.ds3client.models.BulkObject; + +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; + +/** + * A subclass of {@link ChannelStrategy} that holds the instance of a channel a blob will during + * a get operation. + */ +public class SequentialFileWriterChannelStrategy implements ChannelStrategy { + private final Object lock = new Object(); + private final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder; + + /** + * @param objectChannelBuilder An instance of {@link com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder}; + * usually an instance of {@link com.spectralogic.ds3client.helpers.FileObjectGetter}. + */ + public SequentialFileWriterChannelStrategy(final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder) { + this.objectChannelBuilder = objectChannelBuilder; + } + + /** + * For a blob to be transferred, create a channel that will be the source for or destination + * of that transferred blob. + * @param blob The blob to be transferred. + * @return A {@link SeekableByteChannel} that will be the source for or destination + * of that transferred blob. + * @throws IOException + */ + @Override + public SeekableByteChannel acquireChannelForBlob(final BulkObject blob) throws IOException { + synchronized (lock) { + return objectChannelBuilder.buildChannel(blob.getName()); + } + } + + /** + * When a blob has been transferred, release the source or destination channel associated with that + * blob. + * @param seekableByteChannel A {@link SeekableByteChannel} that had been allocated as the source for or destination + * of a blob transfer. + * @param blob The blob {@code seekableByteChannel} was allocated to source or sink {@code blob}'s data. + * @throws IOException + */ + @Override + public void releaseChannelForBlob(final SeekableByteChannel seekableByteChannel, final BulkObject blob) throws IOException { + synchronized (lock) { + seekableByteChannel.close(); + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/TruncatingChannelPreparable.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/TruncatingChannelPreparable.java new file mode 100644 index 000000000..f589505f0 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/TruncatingChannelPreparable.java @@ -0,0 +1,46 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.channelstrategy; + +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * When initiating data movement in a put or get, the source or destination channel my need some sort of + * preparation. For instance, when initiating a get, the destination channel may need to be truncated + * prior to moving data. + */ +public class TruncatingChannelPreparable implements ChannelPreparable { + private final Set channelNames = new HashSet<>(); + + /** + * Prepare a channel prior to moving data. This will truncate a chennal prior to opening it the + * first time. + * @param channelName The channel's identifier. + * @param channelBuilder The instance of {@link com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder} + * used to create a channel. + * @throws IOException + */ + @Override + public void prepareChannel(final String channelName, final Ds3ClientHelpers.ObjectChannelBuilder channelBuilder) throws IOException { + if (channelNames.add(channelName)) { + channelBuilder.buildChannel(channelName).truncate(0).close(); + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractObserver.java new file mode 100644 index 000000000..8c03f5a74 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractObserver.java @@ -0,0 +1,36 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; + +/** + * An observer implementation that provides a default update method. + * @param The context data to send with an update. + */ +public abstract class AbstractObserver implements UpdateStrategy { + private final UpdateStrategy updateStrategy; + + public AbstractObserver(final UpdateStrategy updateStrategy) { + Preconditions.checkNotNull(updateStrategy, "updateStrategy may not be null."); + this.updateStrategy = updateStrategy; + } + + @Override + public void update(final T eventData) { + updateStrategy.update(eventData); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java new file mode 100644 index 000000000..a8648d340 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java @@ -0,0 +1,192 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request; +import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request; +import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.JobState; +import com.spectralogic.ds3client.helpers.events.FailureEvent; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy; +import com.spectralogic.ds3client.models.MasterObjectList; +import com.spectralogic.ds3client.models.Objects; + +import java.io.IOException; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An implementation of {@link TransferStrategy} that provides a default implementation {@code transfer} + * implementation. + */ +abstract class AbstractTransferStrategy implements TransferStrategy { + private static final Logger LOG = LoggerFactory.getLogger(AbstractTransferStrategy.class); + + private final BlobStrategy blobStrategy; + private final JobState jobState; + private final ListeningExecutorService executorService; + private final EventDispatcher eventDispatcher; + private final MasterObjectList masterObjectList; + private final FailureEvent.FailureActivity failureActivity; + + private TransferMethod transferMethod; + + /** + * @param blobStrategy Most likely one of {@link com.spectralogic.ds3client.helpers.strategy.blobstrategy.PutSequentialBlobStrategy} + * or {@link com.spectralogic.ds3client.helpers.strategy.blobstrategy.GetSequentialBlobStrategy}. + * @param jobState An instance of {@link JobState} that keeps track of the blobs we need to process. + * @param executorService An instance of {@link java.util.concurrent.Executor} used to run transfers. Streamed behavior + * uses a single-threaded executor, and random access uses a multi-threaded executor. + * @param eventDispatcher An instance of {@link EventDispatcher} used to emit events as transfers proceed. + * @param masterObjectList The {@link MasterObjectList} returned primarily retrieved from a call to {@link Ds3Client#putBulkJobSpectraS3(PutBulkJobSpectraS3Request)} + * or {@link Ds3Client#getBulkJobSpectraS3(GetBulkJobSpectraS3Request)}, used primarily + * to add contextual information to events. + * @param failureActivity Either {@link com.spectralogic.ds3client.helpers.events.FailureEvent.FailureActivity#PuttingObject} or + * {@link com.spectralogic.ds3client.helpers.events.FailureEvent.FailureActivity#PuttingObject}, used + * to add contextual information to events. + */ + public AbstractTransferStrategy(final BlobStrategy blobStrategy, + final JobState jobState, + final ListeningExecutorService executorService, + final EventDispatcher eventDispatcher, + final MasterObjectList masterObjectList, + final FailureEvent.FailureActivity failureActivity) + { + this.blobStrategy = blobStrategy; + this.jobState = jobState; + this.executorService = executorService; + this.eventDispatcher = eventDispatcher; + this.masterObjectList = masterObjectList; + this.failureActivity = failureActivity; + } + + /** + * Provide a delegate that moves each blob. + * @param transferMethod An instance of {@link TransferMethod} that moves data for a given blob. + * @return The instance of this class, with the intent that you can string together the transfer method composition + * and a call to {@code transfer}. + */ + public AbstractTransferStrategy withTransferMethod(final TransferMethod transferMethod) { + this.transferMethod = transferMethod; + return this; + } + + /** + * Perform data movement according to the properties you specify in {@link TransferStrategyBuilder}. + * @throws IOException + */ + @Override + public void transfer() throws IOException { + while (jobState.hasObjects()) { + transferJobParts(); + } + } + + private void transferJobParts() throws IOException { + try { + try { + final ImmutableList.Builder> transferTasksListBuilder = ImmutableList.builder(); + + final Iterable workQueue = blobStrategy.getWork(); + + for (final JobPart jobPart : workQueue) { + transferTasksListBuilder.add(executorService.submit(new Callable() { + @Override + public Void call() throws Exception { + transferMethod.transferJobPart(jobPart); + return null; + } + })); + } + + runTransferTasks(ImmutableList.copyOf(transferTasksListBuilder.build())); + } catch (final IOException | RuntimeException e) { + throw e; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } catch (final Throwable t) { + emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk())); + throw t; + } + } + + private void runTransferTasks(final Iterable> transferTasks) throws IOException { + try { + Futures.allAsList(transferTasks).get(); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } catch (final ExecutionException e) { + // The future throws a wrapper exception, but we want don't want to expose that this was implemented with futures. + final Throwable cause = e.getCause(); + + // Throw each of the advertised thrown exceptions. + if (cause instanceof IOException) { + throw (IOException)cause; + } + + // The rest we don't know about, so we'll just forward them. + if (cause instanceof RuntimeException) { + throw (RuntimeException)cause; + } else { + throw new RuntimeException(cause); + } + } + } + + private void emitFailureEvent(final FailureEvent failureEvent) { + eventDispatcher.emitFailureEvent(failureEvent); + } + + private FailureEvent makeFailureEvent(final FailureEvent.FailureActivity failureActivity, + final Throwable causalException, + final Objects chunk) + { + return new FailureEvent.Builder() + .doingWhat(failureActivity) + .withCausalException(causalException) + .withObjectNamed(labelForChunk(chunk)) + .usingSystemWithEndpoint(masterObjectList.getNodes().get(0).getEndPoint()) + .build(); + } + + private String labelForChunk(final Objects chunk) { + try { + return chunk.getObjects().get(0).getName(); + } catch (final Throwable t) { + LOG.error("Failed to get label for chunk.", t); + } + + return "unnamed object"; + } + + private Objects firstChunk() { + return masterObjectList.getObjects().get(0); + } + + @Override + public void close() throws IOException { + executorService.shutdown(); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/BlobTransferredEventObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/BlobTransferredEventObserver.java new file mode 100644 index 000000000..15e15fe28 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/BlobTransferredEventObserver.java @@ -0,0 +1,31 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.spectralogic.ds3client.models.BulkObject; + +/** + * This class provides the ability to emit an event when a blob transfer has completed. + */ +public class BlobTransferredEventObserver extends AbstractObserver { + /** + * @param updateStrategy The interface whose implementation determines how to handle emitting an event + * when a blob transfer completes. + */ + public BlobTransferredEventObserver(final UpdateStrategy updateStrategy) { + super(updateStrategy); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumEvent.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumEvent.java new file mode 100644 index 000000000..3b1aeadf7 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumEvent.java @@ -0,0 +1,46 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.ChecksumType; + +/** + * The event emitted when we have computed the checksum for a blob to be transferred to a Black Pearl. + */ +public class ChecksumEvent { + private final BulkObject blob; + private final ChecksumType.Type checksumType; + private final String checksum; + + public ChecksumEvent(final BulkObject blob, final ChecksumType.Type checksumType, final String checksum) { + this.blob = blob; + this.checksumType = checksumType; + this.checksum = checksum; + } + + public BulkObject getBlob() { + return blob; + } + + public ChecksumType.Type getChecksumType() { + return checksumType; + } + + public String getChecksum() { + return checksum; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java new file mode 100644 index 000000000..73f3bdf72 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java @@ -0,0 +1,51 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.ChecksumListener; + +/** + * This class provides the ability to emit an event when we have computed the checksum value for a blob to be + * transferred to a Black Pearl. + */ +public class ChecksumObserver extends AbstractObserver { + private ChecksumListener checksumListener; + + /** + * @param checksumListener An instance of {@link ChecksumListener} wrapped in an event updater. + */ + public ChecksumObserver(final ChecksumListener checksumListener) { + super(new UpdateStrategy() { + @Override + public void update(final ChecksumEvent eventData) { + checksumListener.value(eventData.getBlob(), eventData.getChecksumType(), eventData.getChecksum()); + } + }); + + Preconditions.checkNotNull(checksumListener, "checksumListener may not be null."); + + this.checksumListener = checksumListener; + } + + /** + * @param updateStrategy The interface whose implementation determines how to handle emitting an event + * when we compute the checksum for a blob. + */ + public ChecksumObserver(final UpdateStrategy updateStrategy) { + super(updateStrategy); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ContinueForeverTransferRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ContinueForeverTransferRetryDecorator.java new file mode 100644 index 000000000..64e200200 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ContinueForeverTransferRetryDecorator.java @@ -0,0 +1,46 @@ +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.ExceptionClassifier; +import com.spectralogic.ds3client.helpers.JobPart; + +import java.io.IOException; + +/** + * A {@link TransferRetryDecorator} that will continue to retry a blob transfer, until the transfer either + * succeeds or experiences an unrecoverable exception. + */ +public class ContinueForeverTransferRetryDecorator implements TransferRetryDecorator { + private TransferMethod transferMethodDelegate; + + /** + * @param jobPart An instance of {@link JobPart}, which tells us which Black Pearl is the source + * or destination for a blob transfer. + * @throws IOException + */ + @Override + public void transferJobPart(final JobPart jobPart) throws IOException { + while(true) { + try { + transferMethodDelegate.transferJobPart(jobPart); + break; + } catch (final Throwable t) { + if (ExceptionClassifier.isUnrecoverableException(t)) { + throw t; + } + } + } + } + + /** + * @param transferMethod An instance of {@link TransferMethod} used as a delegate when attempting a transfer. + * @return An instance of this class, with the intent to be able to chain a call to + * {@code transferJobPart}. + */ + @Override + public TransferMethod wrap(final TransferMethod transferMethod) { + Preconditions.checkNotNull(transferMethod, "transferMethod may not be null."); + transferMethodDelegate = transferMethod; + return this; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java new file mode 100644 index 000000000..a4b0c6976 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java @@ -0,0 +1,41 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.DataTransferredListener; +import org.slf4j.Logger; + +public class DataTransferredObserver extends AbstractObserver { + private DataTransferredListener dataTransferredListener; + + public DataTransferredObserver(final DataTransferredListener dataTransferredListener) { + super(new UpdateStrategy() { + @Override + public void update(final Long eventData) { + dataTransferredListener.dataTransferred(eventData); + } + }); + + Preconditions.checkNotNull(dataTransferredListener, "dataTransferredListener may not be null."); + + this.dataTransferredListener = dataTransferredListener; + } + + public DataTransferredObserver(final UpdateStrategy updateStrategy) { + super(updateStrategy); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java new file mode 100644 index 000000000..ee26f2abc --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java @@ -0,0 +1,138 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.spectralogic.ds3client.helpers.ChecksumListener; +import com.spectralogic.ds3client.helpers.DataTransferredListener; +import com.spectralogic.ds3client.helpers.FailureEventListener; +import com.spectralogic.ds3client.helpers.MetadataReceivedListener; +import com.spectralogic.ds3client.helpers.ObjectCompletedListener; +import com.spectralogic.ds3client.helpers.WaitingForChunksListener; +import com.spectralogic.ds3client.helpers.events.FailureEvent; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.ChecksumType; +import com.spectralogic.ds3client.networking.Metadata; + +/** + * An interface to provide the ability to register for and emit events. In prior versions of this SDK, + * clients needed to hold an instance of a listener implementation to later unregister its receipt of events. + * This interface allows for just keeping a reference to an observer to unregister. + */ +public interface EventDispatcher { + /** + * Attaches an event handler that is invoked when a blob is successfully + * transferred to Spectra S3. + */ + DataTransferredObserver attachDataTransferredObserver(final DataTransferredObserver dataTransferredObserver); + void removeDataTransferredObserver(final DataTransferredObserver dataTransferredObserver); + + /** + * Attaches an event handler that is invoked when a full object is + * successfully transferred to Spectra S3. + */ + ObjectCompletedObserver attachObjectCompletedObserver(final ObjectCompletedObserver objectCompletedObserver); + void removeObjectCompletedObserver(final ObjectCompletedObserver objectCompletedObserver); + + /** + * Attaches an event handler that is invoked when an object checksum is received. + */ + ChecksumObserver attachChecksumObserver(final ChecksumObserver checksumObserver); + void removeChecksumObserver(final ChecksumObserver checksumObserver); + + /** + * Attaches an event handler that will be invoked only when there are no chunks available + * for processing. + */ + WaitingForChunksObserver attachWaitingForChunksObserver(final WaitingForChunksObserver waitingForChunksObserver); + void removeWaitingForChunksObserver(final WaitingForChunksObserver waitingForChunksObserver); + + /** + * Attaches an event handler when an object transfer fails + */ + FailureEventObserver attachFailureEventObserver(final FailureEventObserver failureEventObserver); + void removeFailureEventObserver(final FailureEventObserver failureEventObserver); + + /** + * Attaches an event handler invoked when we receive metadata from an object get. + */ + MetaDataReceivedObserver attachMetadataReceivedEventObserver(final MetaDataReceivedObserver metaDataReceivedObserver); + void removeMetadataReceivedEventObserver(final MetaDataReceivedObserver metaDataReceivedObserver); + + /** + * Attaches an event invoked when a blob transfer completes. The primary use for this event is to remove + * a blob from the list of those to transfer when a blob transfer completes. + */ + BlobTransferredEventObserver attachBlobTransferredEventObserver(final BlobTransferredEventObserver blobTransferredEventObserver); + void removeBlobTransferredEventObserver(final BlobTransferredEventObserver blobTransferredEventObserver); + + /** + * Attaches an event handler that is invoked when a blob is successfully + * transferred to Spectra S3. + */ + void attachDataTransferredListener(final DataTransferredListener listener); + void removeDataTransferredListener(final DataTransferredListener listener); + + /** + * Attaches an event handler that is invoked when a full object is + * successfully transferred to Spectra S3. + */ + void attachObjectCompletedListener(final ObjectCompletedListener listener); + void removeObjectCompletedListener(final ObjectCompletedListener listener); + + /** + * Attaches an event handler that is invoked when metadata is received for + * an object. + */ + void attachMetadataReceivedListener(final MetadataReceivedListener listener); + void removeMetadataReceivedListener(final MetadataReceivedListener listener); + + /** + * Attaches an event handler that is invoked when an object checksum is received. + */ + void attachChecksumListener(final ChecksumListener listener); + void removeChecksumListener(final ChecksumListener listener); + + /** + * Attaches an event handler that will be invoked only when there are no chunks available + * for processing. + */ + void attachWaitingForChunksListener(final WaitingForChunksListener listener); + void removeWaitingForChunksListener(final WaitingForChunksListener listener); + + /** + * Attaches an event handler when an object transfer fails + */ + void attachFailureEventListener(final FailureEventListener listener); + void removeFailureEventListener(final FailureEventListener listener); + + void emitFailureEvent(final FailureEvent failureEvent); + void emitWaitingForChunksEvents(final int secondsToDelay); + void emitChecksumEvent(final BulkObject blob, final ChecksumType.Type type, final String checksum); + void emitDataTransferredEvent(final BulkObject blob); + void emitObjectCompletedEvent(final BulkObject blob); + void emitObjectCompletedEvent(final String blobName); + void emitMetaDataReceivedEvent(final String objectName, final Metadata metadata); + void emitBlobTransferredEvent(final BulkObject blob); + + /** + * Emit an event when a get transfers less content than was intended. This event is triggered by catching + * a {@link com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException}. + * @param ds3Object The blob whose content was only partially transferred. + * @param endpoint The host name or IP address of the Black Pearl involved in a transfer that only partially completed. + * @param t The {@link com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException}. + */ + void emitContentLengthMismatchFailureEvent(final BulkObject ds3Object, final String endpoint, final Throwable t); +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java new file mode 100644 index 000000000..905066ae0 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java @@ -0,0 +1,319 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.spectralogic.ds3client.helpers.ChecksumListener; +import com.spectralogic.ds3client.helpers.DataTransferredListener; +import com.spectralogic.ds3client.helpers.FailureEventListener; +import com.spectralogic.ds3client.helpers.MetadataReceivedListener; +import com.spectralogic.ds3client.helpers.ObjectCompletedListener; +import com.spectralogic.ds3client.helpers.WaitingForChunksListener; +import com.spectralogic.ds3client.helpers.events.EventRunner; +import com.spectralogic.ds3client.helpers.events.FailureEvent; +import com.spectralogic.ds3client.helpers.events.MetadataEvent; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.ChecksumType; +import com.spectralogic.ds3client.networking.Metadata; + +import java.util.Map; +import java.util.Set; + +public class EventDispatcherImpl implements EventDispatcher { + private final EventRunner eventRunner; + + private final Set dataTransferredObservers = Sets.newIdentityHashSet(); + private final Set objectCompletedObservers = Sets.newIdentityHashSet(); + private final Set checksumObservers = Sets.newIdentityHashSet(); + private final Set waitingForChunksObservers = Sets.newIdentityHashSet(); + private final Set failureEventObservers = Sets.newIdentityHashSet(); + private final Set metaDataReceivedObservers = Sets.newConcurrentHashSet(); + private final Set blobTransferredEventObservers = Sets.newIdentityHashSet(); + + private final Map dataTransferredListeners = Maps.newConcurrentMap(); + private final Map objectCompletedListeners = Maps.newIdentityHashMap(); + private final Map metadataReceivedListeners = Maps.newIdentityHashMap(); + private final Map checksumListeners = Maps.newIdentityHashMap(); + private final Map waitingForChunksListeners = Maps.newIdentityHashMap(); + private final Map failureEventListeners = Maps.newIdentityHashMap(); + + public EventDispatcherImpl(final EventRunner eventRunner) { + Preconditions.checkNotNull(eventRunner, "eventRunner must not be null."); + this.eventRunner = eventRunner; + } + + @Override + public DataTransferredObserver attachDataTransferredObserver(final DataTransferredObserver dataTransferredObserver) { + dataTransferredObservers.add(dataTransferredObserver); + return dataTransferredObserver; + } + + @Override + public void removeDataTransferredObserver(final DataTransferredObserver dataTransferredObserver) { + dataTransferredObservers.remove(dataTransferredObserver); + } + + @Override + public ObjectCompletedObserver attachObjectCompletedObserver(final ObjectCompletedObserver objectCompletedObserver) { + objectCompletedObservers.add(objectCompletedObserver); + return objectCompletedObserver; + } + + @Override + public void removeObjectCompletedObserver(final ObjectCompletedObserver objectCompletedObserver) { + objectCompletedObservers.remove(objectCompletedObserver); + } + + @Override + public ChecksumObserver attachChecksumObserver(final ChecksumObserver checksumObserver) { + checksumObservers.add(checksumObserver); + return checksumObserver; + } + + @Override + public void removeChecksumObserver(final ChecksumObserver checksumObserver) { + checksumObservers.remove(checksumObserver); + } + + @Override + public WaitingForChunksObserver attachWaitingForChunksObserver(final WaitingForChunksObserver waitingForChunksObserver) { + waitingForChunksObservers.add(waitingForChunksObserver); + return waitingForChunksObserver; + } + + @Override + public void removeWaitingForChunksObserver(final WaitingForChunksObserver waitingForChunksObserver) { + waitingForChunksObservers.remove(waitingForChunksObserver); + } + + @Override + public FailureEventObserver attachFailureEventObserver(final FailureEventObserver failureEventObserver) { + failureEventObservers.add(failureEventObserver); + return failureEventObserver; + } + + @Override + public void removeFailureEventObserver(final FailureEventObserver failureEventObserver) { + failureEventObservers.remove(failureEventObserver); + } + + @Override + public MetaDataReceivedObserver attachMetadataReceivedEventObserver(final MetaDataReceivedObserver metaDataReceivedObserver) { + metaDataReceivedObservers.add(metaDataReceivedObserver); + return metaDataReceivedObserver; + } + + @Override + public void removeMetadataReceivedEventObserver(final MetaDataReceivedObserver metaDataReceivedObserver) { + metaDataReceivedObservers.remove(metaDataReceivedObserver); + } + + @Override + public BlobTransferredEventObserver attachBlobTransferredEventObserver(final BlobTransferredEventObserver blobTransferredEventObserver) { + blobTransferredEventObservers.add(blobTransferredEventObserver); + return blobTransferredEventObserver; + } + + @Override + public void removeBlobTransferredEventObserver(final BlobTransferredEventObserver blobTransferredEventObserver) { + blobTransferredEventObservers.remove(blobTransferredEventObserver); + } + + + @Override + public void attachDataTransferredListener(final DataTransferredListener listener) { + DataTransferredObserver dataTransferredObserver = dataTransferredListeners.get(listener); + + if (dataTransferredObserver == null) { + dataTransferredObserver = attachDataTransferredObserver(new DataTransferredObserver(listener)); + dataTransferredListeners.put(listener, dataTransferredObserver); + } + } + + @Override + public void removeDataTransferredListener(final DataTransferredListener listener) { + final DataTransferredObserver dataTransferredObserver = dataTransferredListeners.get(listener); + + if (dataTransferredObserver != null) { + removeDataTransferredObserver(dataTransferredObserver); + dataTransferredListeners.remove(listener); + } + } + + @Override + public void attachObjectCompletedListener(final ObjectCompletedListener listener) { + ObjectCompletedObserver objectCompletedObserver = objectCompletedListeners.get(listener); + + if (objectCompletedObserver == null) { + objectCompletedObserver = attachObjectCompletedObserver(new ObjectCompletedObserver(listener)); + objectCompletedListeners.put(listener, objectCompletedObserver); + } + } + + @Override + public void removeObjectCompletedListener(final ObjectCompletedListener listener) { + final ObjectCompletedObserver objectCompletedObserver = objectCompletedListeners.get(listener); + + if (objectCompletedObserver != null) { + removeObjectCompletedObserver(objectCompletedObserver); + objectCompletedListeners.remove(listener); + } + } + + @Override + public void attachMetadataReceivedListener(final MetadataReceivedListener listener) { + MetaDataReceivedObserver metaDataReceivedObserver = metadataReceivedListeners.get(listener); + + if (metaDataReceivedObserver == null) { + metaDataReceivedObserver = attachMetadataReceivedEventObserver(new MetaDataReceivedObserver(listener)); + metadataReceivedListeners.put(listener, metaDataReceivedObserver); + } + } + + @Override + public void removeMetadataReceivedListener(final MetadataReceivedListener listener) { + final MetaDataReceivedObserver metaDataReceivedObserver = metadataReceivedListeners.get(listener); + + if (metaDataReceivedObserver != null) { + removeMetadataReceivedEventObserver(metaDataReceivedObserver); + metadataReceivedListeners.remove(listener); + } + } + + @Override + public void attachChecksumListener(final ChecksumListener listener) { + ChecksumObserver checksumObserver = checksumListeners.get(listener); + + if (checksumObserver == null) { + checksumObserver = attachChecksumObserver(new ChecksumObserver(listener)); + checksumListeners.put(listener, checksumObserver); + } + } + + @Override + public void removeChecksumListener(final ChecksumListener listener) { + final ChecksumObserver checksumObserver = checksumListeners.get(listener); + + if (checksumObserver != null) { + removeChecksumObserver(checksumObserver); + checksumListeners.remove(listener); + } + } + + @Override + public void attachWaitingForChunksListener(final WaitingForChunksListener listener) { + WaitingForChunksObserver waitingForChunksObserver = waitingForChunksListeners.get(listener); + + if (waitingForChunksObserver == null) { + waitingForChunksObserver = attachWaitingForChunksObserver(new WaitingForChunksObserver(listener)); + waitingForChunksListeners.put(listener, waitingForChunksObserver); + } + } + + @Override + public void removeWaitingForChunksListener(final WaitingForChunksListener listener) { + final WaitingForChunksObserver waitingForChunksObserver = waitingForChunksListeners.get(listener); + + if (waitingForChunksObserver != null) { + removeWaitingForChunksObserver(waitingForChunksObserver); + waitingForChunksListeners.remove(listener); + } + } + + @Override + public void attachFailureEventListener(final FailureEventListener listener) { + FailureEventObserver failureEventObserver = failureEventListeners.get(listener); + + if (failureEventObserver == null) { + failureEventObserver = attachFailureEventObserver(new FailureEventObserver(listener)); + failureEventListeners.put(listener, failureEventObserver); + } + } + + @Override + public void removeFailureEventListener(final FailureEventListener listener) { + final FailureEventObserver failureEventObserver = failureEventListeners.get(listener); + + if (failureEventObserver != null) { + removeFailureEventObserver(failureEventObserver); + failureEventListeners.remove(listener); + } + } + + @Override + public void emitFailureEvent(final FailureEvent failureEvent) { + emitEvents(failureEventObservers, failureEvent); + } + + private void emitEvents(final Set eventObservers, final T eventData) { + for (final Object eventObserver : eventObservers) { + eventRunner.emitEvent(new Runnable() { + @Override + public void run() { + ((Observer)eventObserver).update(eventData); + } + }); + } + } + + @Override + public void emitWaitingForChunksEvents(final int secondsToDelay) { + emitEvents(waitingForChunksObservers, secondsToDelay); + } + + @Override + public void emitChecksumEvent(final BulkObject blob, final ChecksumType.Type type, final String checksum) { + emitEvents(checksumObservers, new ChecksumEvent(blob, type, checksum)); + } + + @Override + public void emitDataTransferredEvent(final BulkObject blob) { + emitEvents(dataTransferredObservers, blob.getLength()); + } + + @Override + public void emitObjectCompletedEvent(final BulkObject blob) { + emitObjectCompletedEvent(blob.getName()); + } + + @Override + public void emitObjectCompletedEvent(final String blobName) { + emitEvents(objectCompletedObservers, blobName); + } + + @Override + public void emitMetaDataReceivedEvent(final String objectName, final Metadata metadata) { + emitEvents(metaDataReceivedObservers, new MetadataEvent(objectName, metadata)); + } + + @Override + public void emitBlobTransferredEvent(final BulkObject blob) { + emitEvents(blobTransferredEventObservers, blob); + } + + @Override + public void emitContentLengthMismatchFailureEvent(final BulkObject ds3Object, final String endpoint, final Throwable t) { + final FailureEvent failureEvent = FailureEvent.builder() + .doingWhat(FailureEvent.FailureActivity.GettingObject) + .usingSystemWithEndpoint(endpoint) + .withCausalException(t) + .withObjectNamed(ds3Object.getName()) + .build(); + emitFailureEvent(failureEvent); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java new file mode 100644 index 000000000..0a5c6aed2 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java @@ -0,0 +1,52 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.FailureEventListener; +import com.spectralogic.ds3client.helpers.events.FailureEvent; + +/** + * This class provides the ability to emit an event when we have encountered an exception + * during a blob transfer. + */ +public class FailureEventObserver extends AbstractObserver { + private FailureEventListener failureEventListener; + + /** + * @param failureEventListener An instance of {@link FailureEventListener} wrapped in an event updater. + */ + public FailureEventObserver(final FailureEventListener failureEventListener) { + super(new UpdateStrategy() { + @Override + public void update(final FailureEvent eventData) { + failureEventListener.onFailure(eventData); + } + }); + + Preconditions.checkNotNull(failureEventListener, "failureEventListener may not be null."); + + this.failureEventListener = failureEventListener; + } + + /** + * @param updateStrategy An interface whose implementation determines how to handle a failure + * event. + */ + public FailureEventObserver(final UpdateStrategy updateStrategy) { + super(updateStrategy); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java new file mode 100644 index 000000000..487d680b7 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java @@ -0,0 +1,171 @@ +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException; +import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.RecoverableIOException; +import com.spectralogic.ds3client.helpers.strategy.StrategyUtils; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.common.Range; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * An implementation of {@link TransferMethod} that wraps another instance of TransferMethod to provide the + * ability to resume a get operation that transfers less data than is needed for a blob transfer to complete. + */ +public class GetJobNetworkFailureRetryDecorator implements TransferMethod { + private final ChannelStrategy channelStrategy; + private final String bucketName; + private final String jobId; + private final EventDispatcher eventDispatcher; + private final ImmutableMap> rangesForBlobs; + + private final Map transferStateMap = new HashMap<>(); + + public GetJobNetworkFailureRetryDecorator(final ChannelStrategy channelStrategy, + final String bucketName, + final String jobId, + final EventDispatcher eventDispatcher, + final ImmutableMap> rangesForBlobs) + { + this.channelStrategy = channelStrategy; + this.bucketName = bucketName; + this.jobId = jobId; + this.eventDispatcher = eventDispatcher; + this.rangesForBlobs = rangesForBlobs; + } + + @Override + public void transferJobPart(final JobPart jobPart) throws IOException { + try { + getOrMakeTransferMethod(jobPart).transferJobPart(jobPart); + } catch (final ContentLengthNotMatchException contentLengthNotMatchException) { + makeTransferMethodForPartialRetry(jobPart, contentLengthNotMatchException.getTotalBytes()); + throw new RecoverableIOException(contentLengthNotMatchException); + } + } + + private synchronized TransferMethod getOrMakeTransferMethod(final JobPart jobPart) { + TransferState transferState = transferStateMap.get(jobPart); + + if (transferState != null) { + return transferState.getTransferMethod(); + } + + transferState = makeInitialTransferState(); + + transferStateMap.put(jobPart, transferState); + + return transferState.getTransferMethod(); + } + + private TransferState makeInitialTransferState() { + final ImmutableCollection ranges = null; + final Long numBytesToTransfer = null; + final long destinationChannelOffset = 0L; + + return new TransferState( + ranges, + numBytesToTransfer, + destinationChannelOffset, + new GetJobTransferMethod(channelStrategy, bucketName, jobId, eventDispatcher, rangesForBlobs) + ); + } + + private synchronized void makeTransferMethodForPartialRetry(final JobPart jobPart, final long numBytesTransferred) { + final TransferState existingTransferState = transferStateMap.get(jobPart); + + ImmutableCollection ranges = initializeRanges(jobPart.getBulkObject(), existingTransferState); + final Long numBytesToTransfer = initializeNumBytesToTransfer(existingTransferState, ranges); + ranges = updateRanges(ranges, numBytesTransferred, numBytesToTransfer); + final long destinationChannelOffset = existingTransferState.getDestinationChannelOffset() + numBytesTransferred; + + final TransferState newTransferState = new TransferState( + ranges, + numBytesToTransfer, + destinationChannelOffset, + new GetJobPartialBlobTransferMethod(channelStrategy, + bucketName, + jobId, + eventDispatcher, + ranges, + destinationChannelOffset) + ); + + transferStateMap.put(jobPart, newTransferState); + } + + private ImmutableCollection initializeRanges(final BulkObject blob, final TransferState existingTransferState) { + ImmutableCollection ranges = existingTransferState.getRanges(); + + if (ranges == null) { + ranges = StrategyUtils.getRangesForBlob(rangesForBlobs, blob); + } + + if (ranges == null) { + final long numBytesTransferred = 0; + ranges = updateRanges(ranges, numBytesTransferred, blob.getLength()); + } + + return ranges; + } + + private ImmutableCollection updateRanges(final ImmutableCollection ranges, + final long numBytesTransferred, + final Long numBytesToTransfer) + { + return RangeHelper.replaceRange(ranges, numBytesTransferred, numBytesToTransfer); + } + + private Long initializeNumBytesToTransfer(final TransferState existingTransferState, + final ImmutableCollection ranges) + { + Long numBytesToTransfer = existingTransferState.getNumBytesToTransfer(); + + if (numBytesToTransfer == null) { + numBytesToTransfer = RangeHelper.transferSizeForRanges(ranges); + } + + return numBytesToTransfer; + } + + private class TransferState { + private final ImmutableCollection ranges; + private final Long numBytesToTransfer; + private final Long destinationChannelOffset; + private final TransferMethod transferMethod; + + private TransferState(final ImmutableCollection ranges, + final Long numBytesToTransfer, + final Long destinationChannelOffset, + final TransferMethod transferMethod) + { + this.ranges = ranges; + this.numBytesToTransfer = numBytesToTransfer; + this.destinationChannelOffset = destinationChannelOffset; + this.transferMethod = transferMethod; + } + + public ImmutableCollection getRanges() { + return ranges; + } + + public Long getNumBytesToTransfer() { + return numBytesToTransfer; + } + + public Long getDestinationChannelOffset() { + return destinationChannelOffset; + } + + public TransferMethod getTransferMethod() { + return transferMethod; + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java new file mode 100644 index 000000000..262ff499d --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java @@ -0,0 +1,84 @@ +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableCollection; +import com.spectralogic.ds3client.commands.GetObjectRequest; +import com.spectralogic.ds3client.commands.GetObjectResponse; +import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.common.Range; + +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; + +/** + * When a blob transfer retrieves less data than is needed to compete the transfer, the {@link GetJobNetworkFailureRetryDecorator} + * that recognizes that situation will create an instance of this class. This class aggregates the data needed to + * transfer the remainder of blob data. + */ +public class GetJobPartialBlobTransferMethod implements TransferMethod { + private final ChannelStrategy channelStrategy; + private final String bucketName; + private final String jobId; + private final EventDispatcher eventDispatcher; + private final ImmutableCollection blobRanges; + private final long destinationChannelOffset; + + public GetJobPartialBlobTransferMethod(final ChannelStrategy channelStrategy, + final String bucketName, + final String jobId, + final EventDispatcher eventDispatcher, + final ImmutableCollection blobRanges, + final long destinationChannelOffset) + { + Preconditions.checkNotNull(blobRanges, "blobRanges may not be null."); + Preconditions.checkState(destinationChannelOffset >= 0, "destinationChannelOffset must be >= 0."); + + this.channelStrategy = channelStrategy; + this.bucketName = bucketName; + this.jobId = jobId; + this.eventDispatcher = eventDispatcher; + this.blobRanges = blobRanges; + this.destinationChannelOffset = destinationChannelOffset; + } + + /** + * @param jobPart An instance of {@link JobPart}, which tells us which Black Pearl is the source + * or destination for a blob transfer. + * @throws IOException + */ + @Override + public void transferJobPart(final JobPart jobPart) throws IOException { + final BulkObject blob = jobPart.getBulkObject(); + + final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(blob); + + seekableByteChannel.position(destinationChannelOffset); + + try { + final GetObjectResponse getObjectResponse = jobPart.getClient().getObject(makeGetObjectRequest(seekableByteChannel, blob)); + + eventDispatcher.emitChecksumEvent(blob, getObjectResponse.getChecksumType(), getObjectResponse.getChecksum()); + eventDispatcher.emitMetaDataReceivedEvent(blob.getName(), getObjectResponse.getMetadata()); + + eventDispatcher.emitBlobTransferredEvent(blob); + eventDispatcher.emitDataTransferredEvent(blob); + } finally { + channelStrategy.releaseChannelForBlob(seekableByteChannel, blob); + } + } + + private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekableByteChannel, final BulkObject blob) { + final GetObjectRequest getObjectRequest = new GetObjectRequest( + bucketName, + blob.getName(), + seekableByteChannel, + jobId, + blob.getOffset()); + + getObjectRequest.withByteRanges(blobRanges); + + return getObjectRequest; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java new file mode 100644 index 000000000..1e4bdb9d0 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java @@ -0,0 +1,97 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.spectralogic.ds3client.commands.GetObjectRequest; +import com.spectralogic.ds3client.commands.GetObjectResponse; +import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.strategy.StrategyUtils; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.common.Range; + +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; + +/** + * The HTTP GET implementation class that retrieves a blob from a Black Pearl. + */ +public class GetJobTransferMethod implements TransferMethod { + private final ChannelStrategy channelStrategy; + private final String bucketName; + private final String jobId; + private final EventDispatcher eventDispatcher; + + // This is a map of blob name to ranges for a blob of that name. + private final ImmutableMap> rangesForBlobs; + + public GetJobTransferMethod(final ChannelStrategy channelStrategy, + final String bucketName, + final String jobId, + final EventDispatcher eventDispatcher, + final ImmutableMap> rangesForBlobs) + { + this.channelStrategy = channelStrategy; + this.bucketName = bucketName; + this.jobId = jobId; + this.eventDispatcher = eventDispatcher; + this.rangesForBlobs = rangesForBlobs; + } + + /** + * @param jobPart An instance of {@link JobPart}, which tells us which Black Pearl is the source + * or destination for a blob transfer. + * @throws IOException + */ + @Override + public void transferJobPart(final JobPart jobPart) throws IOException { + final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(jobPart.getBulkObject()); + + final GetObjectResponse getObjectResponse = jobPart.getClient().getObject(makeGetObjectRequest(seekableByteChannel, jobPart)); + + final BulkObject blob = jobPart.getBulkObject(); + + channelStrategy.releaseChannelForBlob(seekableByteChannel, blob); + + eventDispatcher.emitChecksumEvent(blob, getObjectResponse.getChecksumType(), getObjectResponse.getChecksum()); + eventDispatcher.emitMetaDataReceivedEvent(blob.getName(), getObjectResponse.getMetadata()); + + eventDispatcher.emitBlobTransferredEvent(blob); + eventDispatcher.emitDataTransferredEvent(blob); + } + + private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekableByteChannel, final JobPart jobPart) { + final BulkObject blob = jobPart.getBulkObject(); + + final GetObjectRequest getObjectRequest = new GetObjectRequest( + bucketName, + blob.getName(), + seekableByteChannel, + jobId, + blob.getOffset()); + + final ImmutableCollection rangesForBlob = StrategyUtils.getRangesForBlob(rangesForBlobs, blob); + + if (rangesForBlob != null) { + getObjectRequest.withByteRanges(rangesForBlob); + } + + return getObjectRequest; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MaxNumObjectTransferAttemptsDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MaxNumObjectTransferAttemptsDecorator.java new file mode 100644 index 000000000..5fe828d77 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MaxNumObjectTransferAttemptsDecorator.java @@ -0,0 +1,68 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.ExceptionClassifier; +import com.spectralogic.ds3client.helpers.JobPart; + +import java.io.IOException; + +/** + * A {@link TransferRetryDecorator} that will continue to retry a blob transfer at most {@code maxNumObjectTransferAttempts} + * times, until the transfer either succeeds or experiences an unrecoverable exception. + */ +public class MaxNumObjectTransferAttemptsDecorator implements TransferRetryDecorator { + private final int maxNumObjectTransferAttempts; + private TransferMethod transferMethodDelegate; + + public MaxNumObjectTransferAttemptsDecorator(final int maxNumObjectTransferAttempts) { + this.maxNumObjectTransferAttempts = maxNumObjectTransferAttempts; + } + + /** + * @param transferMethod An instance of {@link TransferMethod} used as a delegate when attempting a transfer. + * @return An instance of this class, with the intent to be able to chain a call to + * {@code transferJobPart}. + */ + @Override + public TransferMethod wrap(final TransferMethod transferMethod) { + Preconditions.checkNotNull(transferMethod, "transferMethod may not be null."); + transferMethodDelegate = transferMethod; + return this; + } + + /** + * @param jobPart An instance of {@link JobPart}, which tells us which Black Pearl is the source + * or destination for a blob transfer. + * @throws IOException + */ + @Override + public void transferJobPart(final JobPart jobPart) throws IOException { + int objectTransfersAttempted = 0; + + while(true) { + try { + transferMethodDelegate.transferJobPart(jobPart); + break; + } catch (final Throwable t) { + if (ExceptionClassifier.isUnrecoverableException(t) || ++objectTransfersAttempted >= maxNumObjectTransferAttempts) { + throw t; + } + } + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MetaDataReceivedObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MetaDataReceivedObserver.java new file mode 100644 index 000000000..60d1229f5 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MetaDataReceivedObserver.java @@ -0,0 +1,71 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.MetadataReceivedListener; +import com.spectralogic.ds3client.helpers.events.MetadataEvent; + +/** + * This class provides the ability to emit an event when we receive metadata during a get operation. + */ +public class MetaDataReceivedObserver extends AbstractObserver { + private MetadataReceivedListener metadataReceivedListener; + + /** + * @param metadataReceivedListener An instance of {@link MetadataReceivedListener} wrapped in an event + * updater. + */ + public MetaDataReceivedObserver(final MetadataReceivedListener metadataReceivedListener) { + super(new UpdateStrategy() { + @Override + public void update(final MetadataEvent eventData) { + metadataReceivedListener.metadataReceived(eventData.getObjectName(), eventData.getMetadata()); + } + }); + + Preconditions.checkNotNull(metadataReceivedListener, "metadataReceivedListener may not be null."); + + this.metadataReceivedListener = metadataReceivedListener; + } + + /** + * @param updateStrategy An {@link UpdateStrategy} whose implementation determines how to emit a metadata + * event. + */ + public MetaDataReceivedObserver(final UpdateStrategy updateStrategy) { + super(updateStrategy); + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (!(o instanceof MetaDataReceivedObserver)) return false; + if (!super.equals(o)) return false; + + final MetaDataReceivedObserver that = (MetaDataReceivedObserver) o; + + return metadataReceivedListener != null ? metadataReceivedListener.equals(that.metadataReceivedListener) : that.metadataReceivedListener == null; + + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (metadataReceivedListener != null ? metadataReceivedListener.hashCode() : 0); + return result; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java new file mode 100644 index 000000000..49eb033e3 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java @@ -0,0 +1,44 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.util.concurrent.MoreExecutors; +import com.spectralogic.ds3client.helpers.JobState; +import com.spectralogic.ds3client.helpers.events.FailureEvent; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy; +import com.spectralogic.ds3client.models.MasterObjectList; + +import java.util.concurrent.Executors; + +/** + * An implementation of {@link TransferStrategy} that runs transfers in a multi-threaded executor. + */ +public class MultiThreadedTransferStrategy extends AbstractTransferStrategy { + public MultiThreadedTransferStrategy(final BlobStrategy blobStrategy, + final JobState jobState, + final int numConcurrentTransferThreads, + final EventDispatcher eventDispatcher, + final MasterObjectList masterObjectList, + final FailureEvent.FailureActivity failureActivity) + { + super(blobStrategy, + jobState, + MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numConcurrentTransferThreads)), + eventDispatcher, + masterObjectList, + failureActivity); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java new file mode 100644 index 000000000..61dee7966 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java @@ -0,0 +1,51 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.ObjectCompletedListener; + +/** + * An {@link Observer} that enables sending event when a DS3 object transfer completes. + */ +public class ObjectCompletedObserver extends AbstractObserver { + private ObjectCompletedListener objectCompletedListener; + + /** + * @param objectCompletedListener An {@link ObjectCompletedListener} wrapped in an event + * updater. + */ + public ObjectCompletedObserver(final ObjectCompletedListener objectCompletedListener) { + super(new UpdateStrategy() { + @Override + public void update(final String eventData) { + objectCompletedListener.objectCompleted(eventData); + } + }); + + Preconditions.checkNotNull(objectCompletedListener, "objectCompletedListener may not be null"); + + this.objectCompletedListener = objectCompletedListener; + } + + /** + * @param updateStrategy An {@link UpdateStrategy} whose implementation determines how to handle a DS3 object transfer + * completion. + */ + public ObjectCompletedObserver(final UpdateStrategy updateStrategy) { + super(updateStrategy); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/Observer.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/Observer.java new file mode 100644 index 000000000..2fc916811 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/Observer.java @@ -0,0 +1,24 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +/** + * An interface to allow for sending events. + * @param The context data to send with an event. + */ +public interface Observer { + void update(final T eventData); +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java new file mode 100644 index 000000000..a9bbe0c92 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java @@ -0,0 +1,134 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.collect.ImmutableMap; +import com.spectralogic.ds3client.commands.PutObjectRequest; +import com.spectralogic.ds3client.helpers.ChecksumFunction; +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; +import com.spectralogic.ds3client.helpers.JobPart; +import com.spectralogic.ds3client.helpers.MetadataAccess; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.ChecksumType; + +import java.io.IOException; +import java.nio.channels.ByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.util.Map; + +import com.spectralogic.ds3client.utils.Guard; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The HTTP PUT implementation class that retrieves a blob from a Black Pearl. + */ +public class PutJobTransferMethod implements TransferMethod { + private static final Logger LOG = LoggerFactory.getLogger(PutJobTransferMethod.class); + + private final ChannelStrategy channelStrategy; + private final String bucketName; + private final String jobId; + private final EventDispatcher eventDispatcher; + private final ChecksumFunction checksumFunction; + private final ChecksumType.Type checksumType; + private final MetadataAccess metadataAccess; + + public PutJobTransferMethod(final ChannelStrategy channelStrategy, + final String bucketName, + final String jobId, + final EventDispatcher eventDispatcher, + final ChecksumFunction checksumFunction, + final ChecksumType.Type checksumType, + final MetadataAccess metadataAccess) + { + this.channelStrategy = channelStrategy; + this.bucketName = bucketName; + this.jobId = jobId; + this.eventDispatcher = eventDispatcher; + this.checksumFunction = checksumFunction; + this.checksumType = checksumType; + this.metadataAccess = metadataAccess; + } + + /** + * @param jobPart An instance of {@link JobPart}, which tells us which Black Pearl is the source + * or destination for a blob transfer. + * @throws IOException + */ + @Override + public void transferJobPart(final JobPart jobPart) throws IOException { + final BulkObject blob = jobPart.getBulkObject(); + + final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(blob); + + jobPart.getClient().putObject(makePutObjectRequest(seekableByteChannel, jobPart)); + + channelStrategy.releaseChannelForBlob(seekableByteChannel, blob); + + eventDispatcher.emitBlobTransferredEvent(blob); + eventDispatcher.emitDataTransferredEvent(blob); + } + + private PutObjectRequest makePutObjectRequest(final SeekableByteChannel seekableByteChannel, final JobPart jobPart) { + final BulkObject blob = jobPart.getBulkObject(); + + final PutObjectRequest putObjectRequest = new PutObjectRequest( + bucketName, + blob.getName(), + seekableByteChannel, + jobId, + blob.getOffset(), + blob.getLength()); + + addCheckSum(blob, putObjectRequest); + addMetadata(blob, putObjectRequest); + + return putObjectRequest; + } + + private void addCheckSum(final BulkObject blob, final PutObjectRequest putObjectRequest) { + if (checksumFunction != null) { + final String checksum; + + try { + final ByteChannel byteChannel = channelStrategy.acquireChannelForBlob(blob); + checksum = checksumFunction.compute(blob, byteChannel); + + if (checksum != null) { + putObjectRequest.withChecksum(ChecksumType.value(checksum), checksumType); + eventDispatcher.emitChecksumEvent(blob, checksumType, checksum); + } + } catch (final IOException e) { + // TODO Emit a failure event here + LOG.info("Failure creating channel to calculate checksum.", e); + } + } + } + + private void addMetadata(final BulkObject blob, final PutObjectRequest putObjectRequest) { + if (blob.getOffset() == 0 && metadataAccess != null) { + final Map metadata = metadataAccess.getMetadataValue(blob.getName()); + if ( ! Guard.isMapNullOrEmpty(metadata)) { + final ImmutableMap immutableMetadata = ImmutableMap.copyOf(metadata); + for (final Map.Entry value : immutableMetadata.entrySet()) { + putObjectRequest.withMetaData(value.getKey(), value.getValue()); + } + } + } + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/RangeHelper.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java similarity index 74% rename from ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/RangeHelper.java rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java index 915a2576e..17f008171 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/RangeHelper.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java @@ -1,19 +1,4 @@ -/* - * ****************************************************************************** - * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.helpers; +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableCollection; @@ -22,6 +7,10 @@ import com.spectralogic.ds3client.models.common.Range; import com.spectralogic.ds3client.utils.Guard; +/** + * A class used in calculating the remaining data offset(s) and length(s) when we transfer less than an entire + * blob during a get. + */ final class RangeHelper { private RangeHelper() {} @@ -62,7 +51,7 @@ static ImmutableCollection replaceRange(final ImmutableCollection return newRangesbuilder.build(); } - static void addRemainingRanges(final UnmodifiableIterator existingRangesIterator, final ImmutableList.Builder newRangesbuilder) { + private static void addRemainingRanges(final UnmodifiableIterator existingRangesIterator, final ImmutableList.Builder newRangesbuilder) { while (existingRangesIterator.hasNext()) { newRangesbuilder.add(existingRangesIterator.next()); } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java new file mode 100644 index 000000000..a37debb2f --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java @@ -0,0 +1,43 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.util.concurrent.MoreExecutors; +import com.spectralogic.ds3client.helpers.JobState; +import com.spectralogic.ds3client.helpers.events.FailureEvent; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy; +import com.spectralogic.ds3client.models.MasterObjectList; + +import java.util.concurrent.Executors; + +/** + * An implementation of {@link TransferStrategy} that runs transfers in a single-threaded executor. + */ +public class SingleThreadedTransferStrategy extends AbstractTransferStrategy { + public SingleThreadedTransferStrategy(final BlobStrategy blobStrategy, + final JobState jobState, + final EventDispatcher eventDispatcher, + final MasterObjectList masterObjectList, + final FailureEvent.FailureActivity failureActivity) + { + super(blobStrategy, + jobState, + MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()), + eventDispatcher, + masterObjectList, + failureActivity); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethod.java new file mode 100644 index 000000000..603d9d441 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethod.java @@ -0,0 +1,32 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.spectralogic.ds3client.helpers.JobPart; + +import java.io.IOException; + +/** + * Define the contract used to put or get a blob. + */ +public interface TransferMethod { + /** + * @param jobPart An instance of {@link JobPart}, which tells us which Black Pearl is the source + * or destination for a blob transfer. + * @throws IOException + */ + void transferJobPart(final JobPart jobPart) throws IOException; +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethodMaker.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethodMaker.java new file mode 100644 index 000000000..b7a394969 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethodMaker.java @@ -0,0 +1,24 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +/** + * An interface whose implementation determines the specific {@link TransferMethod} subclass to create + * when building a {@link TransferStrategy} in {@link TransferStrategyBuilder}. + */ +interface TransferMethodMaker { + TransferMethod makeTransferMethod(); +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferRetryDecorator.java new file mode 100644 index 000000000..22ec734e7 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferRetryDecorator.java @@ -0,0 +1,20 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +public interface TransferRetryDecorator extends TransferMethod { + TransferMethod wrap(final TransferMethod transferMethod); +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java new file mode 100644 index 000000000..fd4088ae1 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java @@ -0,0 +1,30 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import java.io.Closeable; +import java.io.IOException; + +/** + * Define the contract for concrete classes that implement data movement. + */ +public interface TransferStrategy extends Closeable { + /** + * Perform data movement according to the properties you specify in {@link TransferStrategyBuilder}. + * @throws IOException + */ + void transfer() throws IOException; +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java new file mode 100644 index 000000000..51e437385 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java @@ -0,0 +1,941 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Iterables; +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.Ds3ClientBuilder; +import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request; +import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request; +import com.spectralogic.ds3client.helpers.ChecksumFunction; +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; +import com.spectralogic.ds3client.helpers.JobPartTracker; +import com.spectralogic.ds3client.helpers.JobPartTrackerFactory; +import com.spectralogic.ds3client.helpers.MetadataAccess; +import com.spectralogic.ds3client.helpers.ObjectCompletedListener; +import com.spectralogic.ds3client.helpers.ObjectPart; +import com.spectralogic.ds3client.helpers.events.EventRunner; +import com.spectralogic.ds3client.helpers.events.FailureEvent; +import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner; +import com.spectralogic.ds3client.helpers.strategy.StrategyUtils; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlackPearlChunkAttemptRetryDelayBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategyMaker; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryDelayBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ClientDefinedChunkAttemptRetryDelayBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ContinueForeverChunkAttemptsRetryBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.GetSequentialBlobStrategy; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.MaxChunkAttemptsRetryBehavior; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.PutSequentialBlobStrategy; +import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryBehavior; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.NullChannelPreparable; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.RandomAccessChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.SequentialChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.SequentialFileReaderChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.SequentialFileWriterChannelStrategy; +import com.spectralogic.ds3client.helpers.strategy.channelstrategy.TruncatingChannelPreparable; +import com.spectralogic.ds3client.models.BulkObject; +import com.spectralogic.ds3client.models.ChecksumType; +import com.spectralogic.ds3client.models.JobChunkClientProcessingOrderGuarantee; +import com.spectralogic.ds3client.models.MasterObjectList; +import com.spectralogic.ds3client.models.Objects; +import com.spectralogic.ds3client.models.common.Range; +import com.spectralogic.ds3client.utils.Guard; +import com.spectralogic.ds3client.utils.SeekableByteChannelInputStream; +import com.spectralogic.ds3client.utils.hashing.ChecksumUtils; +import com.spectralogic.ds3client.utils.hashing.Hasher; +import com.spectralogic.ds3client.helpers.JobState; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.ByteChannel; +import java.util.ArrayList; +import java.util.List; + +/** + * This class allows for building configurable behavior when putting objects to or getting objects + * from a Black Pearl. The intent is to provide the ability for transfers to be "streamed" or "random access". + * Streaming, as used in a transfer strategy, means that channels and blobs are read or written sequentially. + * You would use a streaming strategy when it is important that the source or destination channel is read or + * written in a predictable, sequential order while a transfer is in progress. + * Random access means that channels and blobs are read and written in no particular order. You would use + * a random access strategy when it is not important that a source or destination channel is read or written + * in a predictable order while a transfer is in progress. Once a transfer is complete, the destination channel + * and Black Pearl objects are ordered as was the original archived object. + * + * You specify that you want streamed behavior by calling {@link TransferStrategyBuilder#usingStreamedTransferBehavior()} + * and random access behavior by calling {@link TransferStrategyBuilder#usingRandomAccessTransferBehavior()}. If you + * specify neither, the transfer strategy used will behave as this SDK behaved before introducing the ability + * to specify transfer behavior. + * + * The 2 methods to call to get an instance of a {@link TransferStrategy}, which is the interface that performs data + * movement, are {@link TransferStrategyBuilder#makePutTransferStrategy()} and + * {@link TransferStrategyBuilder#makeGetTransferStrategy()}. Once you have a TransferStrategy + * interface instance, calling {@link TransferStrategy#transfer()} on the TransferStrategy interface initiates + * data movement. + * + * If using a {@link TransferStrategy} directly, as oppsed to going through a {@link Ds3ClientHelpers.Job}, the minimum + * needed as configuration items in a put transfer strategy builder are: + *
    + *
  • {@link TransferStrategyBuilder#withDs3Client(Ds3Client)}
    • + *
  • {@link TransferStrategyBuilder#withMasterObjectList(MasterObjectList)}
    • + *
  • {@link TransferStrategyBuilder#withChannelBuilder(Ds3ClientHelpers.ObjectChannelBuilder)}
    • + *
+ * + * When doing a get using a transfer strategy directly, you also need to specify: + *
    + *
  • {@link TransferStrategyBuilder#withRangesForBlobs(ImmutableMap)}
    • + *
+ */ +public final class TransferStrategyBuilder { + private static final Logger LOG = LoggerFactory.getLogger(TransferStrategyBuilder.class); + + private static final int DEFAULT_MAX_CONCURRENT_TRANSFER_THREADS = 10; + public static final int DEFAULT_CHUNK_ATTEMPT_RETRY_INTERVAL = -1; + public static final int DEFAULT_CHUNK_ATTEMPT_RETRY_ATTEMPTS = -1; + public static final int DEFAULT_OBJECT_TRANSFER_ATTEMPTS = 5; + + private BlobStrategy blobStrategy; + private ChannelStrategy channelStrategy; + private String bucketName; + private String jobId; + private TransferRetryDecorator transferRetryDecorator = new MaxNumObjectTransferAttemptsDecorator(DEFAULT_OBJECT_TRANSFER_ATTEMPTS); + private ChecksumFunction checksumFunction; + private ChecksumType.Type checksumType = ChecksumType.Type.NONE; + private EventRunner eventRunner = new SameThreadEventRunner(); + private EventDispatcher eventDispatcher = new EventDispatcherImpl(eventRunner); + private JobPartTracker jobPartTracker; + private JobState jobState; + private int numTransferRetries = DEFAULT_OBJECT_TRANSFER_ATTEMPTS; + private int numConcurrentTransferThreads = DEFAULT_MAX_CONCURRENT_TRANSFER_THREADS; + private int numChunkAttemptRetries = DEFAULT_CHUNK_ATTEMPT_RETRY_ATTEMPTS; + private int chunkRetryDelayInSeconds = DEFAULT_CHUNK_ATTEMPT_RETRY_INTERVAL; + private Ds3Client ds3Client; + private MasterObjectList masterObjectList; + private Ds3ClientHelpers.ObjectChannelBuilder channelBuilder; + private ImmutableMap> rangesForBlobs; + private MetadataAccess metadataAccess; + private ChunkAttemptRetryBehavior chunkAttemptRetryBehavior; + private ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior; + private TransferBehaviorType transferBehaviorType = TransferBehaviorType.OriginalSdkTransferBehavior; + private FailureEvent.FailureActivity failureActivity = FailureEvent.FailureActivity.PuttingObject; + + /** + * Use an instance of {@link BlobStrategy} you wish to create or retrieve blobs from a Black Pearl. There are + * 2 primary implementations: {@link PutSequentialBlobStrategy}; and {@link GetSequentialBlobStrategy}. Blob + * retrieval order is specified in the + * {@link com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request#withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee)} + * HTTP request. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish, + * for example: + * 
+     *     {@code
+     *     final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(objectsToWrite));
+     *     final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = client.putBulkJobSpectraS3(request);
+     *
+     *     final MasterObjectList masterObjectList = putBulkJobSpectraS3Response.getResult();
+     *
+     *     final EventDispatcher eventDispatcher = new EventDispatcherImpl(new SameThreadEventRunner());
+     *
+     *     final AtomicInteger numChunkAllocationAttempts = new AtomicInteger(0);
+     *
+     *     final BlobStrategy blobStrategy = new UserSuppliedPutBlobStrategy(client,
+     *                                                                       masterObjectList,
+     *                                                                       eventDispatcher,
+     *                                                                       new MaxChunkAttemptsRetryBehavior(5),
+     *                                                                       new ClientDefinedChunkAttemptRetryDelayBehavior(1, eventDispatcher),
+     *                                                                           new Monitorable() {
+     *                                                                               @Override
+     *                                                                                   public void monitor() {
+     *                                                                                       numChunkAllocationAttempts.incrementAndGet();
+     *                                                                                   }
+     *                                                                           });
+     *
+     *     final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
+     *         .withDs3Client(Ds3ClientBuilder.fromEnv().withHttps(false).build())
+     *         .withMasterObjectList(masterObjectList)
+     *         .withChannelBuilder(new FileObjectPutter(dirPath))
+     *         .withBlobStrategy(blobStrategy);
+     *
+     *     final TransferStrategy transferStrategy = transferStrategyBuilder.makePutTransferStrategy();
+     *     transferStrategy.transfer();
+     *
+ */ + public TransferStrategyBuilder withBlobStrategy(final BlobStrategy blobStrategy) { + this.blobStrategy = blobStrategy; + return this; + } + + /** + * Use an instance of {@link ChannelStrategy} you wish to manage the source or destination of a transfer. + * The 2 primary channel strategies are {@link RandomAccessChannelStrategy} and {@link SequentialChannelStrategy}. + * A channel strategy normally has a {@link Ds3ClientHelpers.ObjectChannelBuilder} that allocates the + * source or destination. Random access behavior creates a channel per blob; streamed access creates a single + * channel to reference all blobs associated with a DS3 object. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withChannelStrategy(final ChannelStrategy channelStrategy) { + this.channelStrategy = channelStrategy; + return this; + } + + /** + * The {@link Ds3ClientHelpers.ObjectChannelBuilder} you wish to use to allocate source or destination + * channels. This is used in a {@link ChannelStrategy} to manage streamed or random access behavior. + * This is the same interface used in {@link Ds3ClientHelpers.Job#transfer(Ds3ClientHelpers.ObjectChannelBuilder)}. + * The 2 primary implementations of this interface are {@link com.spectralogic.ds3client.helpers.FileObjectGetter} + * and {@link com.spectralogic.ds3client.helpers.FileObjectPutter}. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withChannelBuilder(final Ds3ClientHelpers.ObjectChannelBuilder channelBuilder) { + this.channelBuilder = channelBuilder; + return this; + } + + /** + * The {@link TransferRetryDecorator} to use when retrying a failed transfer attempt. If not specified directly, + * the builder will create a retry decorator based on the value specified in + * {@link TransferStrategyBuilder#withNumTransferRetries(int)}. The 2 primary transfer retry decorators + * are {@link MaxNumObjectTransferAttemptsDecorator} and {@link ContinueForeverTransferRetryDecorator} + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withTransferRetryDecorator(final TransferRetryDecorator transferRetryDecorator) { + this.transferRetryDecorator = transferRetryDecorator; + return this; + } + + /** + * The transfer function to use when putting an object to a Black Pearl when the value in + * {@link TransferStrategyBuilder#withChecksumType(ChecksumType.Type)} is not + * {@link ChecksumType.None}. This builder will make a checksum function of the correct type + * if you specify a checksum type other than {@link ChecksumType.None} but do not specify a + * checksum function. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withChecksumFunction(final ChecksumFunction checksumFunction) { + this.checksumFunction = checksumFunction; + return this; + } + + /** + * The {@link ChecksumType.None} you would like to compute and include in the payload sent to a + * Black Pearl when putting an object. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withChecksumType(final ChecksumType.Type checksumType) { + this.checksumType = checksumType; + return this; + } + + /** + * The {@link EventDispatcher} you would like to use when registering as an event observer and emitting + * events. The primary event dispatcher, {@link EventDispatcherImpl}, uses an {@link EventRunner} to deliver events. + * You specify the event runner behavior you want by calling {@link TransferStrategyBuilder#withEventRunner(EventRunner)}. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withEventDispatcher(final EventDispatcher eventDispatcher) { + this.eventDispatcher = eventDispatcher; + return this; + } + + /** + * {@link Ds3ClientHelpers.Job} uses the event dispatcher configured into this builder to emit events + * during a transfer. + */ + public EventDispatcher eventDispatcher() { + return eventDispatcher; + } + + /** + * When not directly specifying a {@link TransferRetryDecorator}, this property is used to make a transfer + * retry decorator instance. Doing so uses the behavior this SDK used prior to introducing transfer behavior + * configuration: a value less than 0 causes a failed transfer to be retried indefinitely; a value greater than 0 + * causes a failed transfer to be retried no more than the value specified before giving up. If you do not + * specify a value, this builder will use the same value as used in this SDK prior to introducing transfer behavior. + * @param numRetries + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withNumTransferRetries(final int numRetries) { + this.numTransferRetries = numRetries; + return this; + } + + /** + * The number of threads used to transfer blobs. Random access behavior uses the number of threads you + * specify in this property. Streamed behavior uses a single thread when transferring blobs. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withNumConcurrentTransferThreads(final int numConcurrentTransferThreads) { + this.numConcurrentTransferThreads = numConcurrentTransferThreads; + return this; + } + + /** + * When not directly using {@link TransferStrategyBuilder#withChunkAttemptRetryBehavior(ChunkAttemptRetryBehavior)}, + * this property is used to make a {@link ChunkAttemptRetryBehavior} instance. Doing so uses the behavior this SDK used prior to introducing transfer behavior + * configuration: a value less than 0 causes a failed chunk operation to be retried indefinitely; a value greater than 0 + * causes a failed chunk operation to be retried no more than the value specified before giving up. If you do not + * specify a value, this builder will use the same value as used in this SDK prior to introducing transfer behavior. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withNumChunkAttemptRetries(final int numChunkAttemptRetries) { + this.numChunkAttemptRetries = numChunkAttemptRetries; + return this; + } + + /** + * When not directly using {@link TransferStrategyBuilder#withChunkAttemptRetryDelayBehavior(ChunkAttemptRetryDelayBehavior)}, + * this property is used to create a {@link ChunkAttemptRetryDelayBehavior} instance. Doing so uses the behavior this SDK used prior to introducing transfer behavior + * configuration: a value less than 0 causes a failed chunk operation to use the retry delay value a Black Pearl returns + * as part of the chunk operation failure response; a value greater than 0 + * causes a failed chunk operation to the value specified in the {@code retryDelayInSeconds} parameter. If you do not + * specify a value, this builder will use the same value as used in this SDK prior to introducing transfer behavior. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withChunkRetryDelayInSeconds(final int retryDelayInSeconds) { + this.chunkRetryDelayInSeconds = retryDelayInSeconds; + return this; + } + + /** + * The {@link Ds3Client} instance representing the Black Pearl you want to work with. The primary way to + * get a client instance is to call {@link Ds3ClientBuilder#fromEnv()}. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withDs3Client(final Ds3Client ds3Client) { + this.ds3Client = ds3Client; + return this; + } + + /** + * The {@link MasterObjectList} returned primarily retrieved from a call to {@link Ds3Client#putBulkJobSpectraS3(PutBulkJobSpectraS3Request)} + * or {@link Ds3Client#getBulkJobSpectraS3(GetBulkJobSpectraS3Request)}. The master object list is used in + * creating the ability to work with a blob strategy. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withMasterObjectList(final MasterObjectList masterObjectList) { + this.masterObjectList = masterObjectList; + jobId = masterObjectList.getJobId().toString(); + bucketName = masterObjectList.getBucketName(); + return this; + } + + public MasterObjectList masterObjectList() { + return masterObjectList; + } + + /** + * When getting portions of objects from a Black Pearl, object ranges are the mechanism used to tell the Blavk Pearl + * which portions of obejcts to get. + * @param rangesForBlobs When retrieving whole objects, you supply an empty range, e.g.: + * 
+     *                       {@code
+     *                       final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
+     *                           .withDs3Client(client)
+     *                           .withMasterObjectList(masterObjectList)
+     *                           .withChannelBuilder(new FileObjectGetter(tempDirectory))
+     *                           .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(),
+     *                               PartialObjectHelpers.getPartialObjectsRanges(objects)));
+     *                       }
+     *
+ * + * When retrieving partial objects, you can build a list by doing something like: + * 
+     *                       {@code
+     *                       final List filesToGet = new ArrayList<>();
+     *                       final int offsetIntoFirstRange = 10;
+     *                       filesToGet.add(new PartialDs3Object(FILE_NAME, Range.byLength(200000, 100000)));
+     *                       filesToGet.add(new PartialDs3Object(FILE_NAME, Range.byLength(100000, 100000)));
+     *                       filesToGet.add(new PartialDs3Object(FILE_NAME, Range.byLength(offsetIntoFirstRange, 100000)));
+     *
+     *                       final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
+     *                           .withDs3Client(client)
+     *                           .withMasterObjectList(masterObjectList)
+     *                           .withChannelBuilder(new FileObjectGetter(tempDirectory))
+     *                           .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(),
+     *                               PartialObjectHelpers.getPartialObjectsRanges(filesToGet)));
+     *                       }
+     *
+ * + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withRangesForBlobs(final ImmutableMap> rangesForBlobs) { + this.rangesForBlobs = rangesForBlobs; + return this; + } + + /** + * Supply an instance of {@link MetadataAccess} to record metadata as part pf a put transfer. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withMetadataAccess(final MetadataAccess metadataAccess) { + this.metadataAccess = metadataAccess; + return this; + } + + /** + * Supply an instance of {@link ChunkAttemptRetryBehavior} when you need to have specialized control over retrying + * a failed chunk operation. The 2 primary implementations for this are {@link MaxChunkAttemptsRetryBehavior} + * and {@link ContinueForeverChunkAttemptsRetryBehavior}. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withChunkAttemptRetryBehavior(final ChunkAttemptRetryBehavior chunkAttemptRetryBehavior) { + this.chunkAttemptRetryBehavior = chunkAttemptRetryBehavior; + return this; + } + + /** + * Supply an instance of {@link ChunkAttemptRetryDelayBehavior} when you need to have specialized control over the length + * of time to delay between retrying a filed chunk operation. The 2 primary implementations are {@link ClientDefinedChunkAttemptRetryDelayBehavior}, + * which will always use the delay interval a client specifies, and {@link BlackPearlChunkAttemptRetryDelayBehavior}, + * which will use the retry delay a Black Pearl returns in its payload when a chunk operation cannot be completed. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withChunkAttemptRetryDelayBehavior(final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior) { + this.chunkAttemptRetryDelayBehavior = chunkAttemptRetryDelayBehavior; + return this; + } + + /** + * Specify an {@link EventRunner} to use when dispatching events. The primary reason for specifying this is to determine + * which threading model is used to deliver events. If it is important that events are delivered on the same thread, + * meaning that the deleivery order is deterministic, use {@link SameThreadEventRunner}, which is the event runner instance this SDK uses. + * You may specify the {@link com.spectralogic.ds3client.helpers.events.ConcurrentEventRunner} if you want events + * delivered on separate threads. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder withEventRunner(final EventRunner eventRunner) { + this.eventRunner = eventRunner; + return this; + } + + /** + * Setting this property causes this SDK to read and write channels from beginning to end sequentially and causes + * blob retrieval to be in order. Transfers happen on a single-threaded executor. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder usingStreamedTransferBehavior() { + transferBehaviorType = TransferBehaviorType.StreamingTransferBehavior; + return this; + } + + /** + * Setting this property causes this SDK to read and write channels in no particular order and + * specifies no order in retrieving blobs. Transfers happen in a multi-threaded executor. + * @return The instance of this builder, with the intent that you can string together the behaviors you wish. + */ + public TransferStrategyBuilder usingRandomAccessTransferBehavior() { + transferBehaviorType = TransferBehaviorType.RandomAccessTransferBehavior; + return this; + } + + /** + * Get an instance of {@link TransferStrategy} configured with this builders current settings. Once you + * have an instance, you initiate a put by calling {@link TransferStrategy#transfer()}. + * @return Once you have an instance, you initiate a get by calling {@link TransferStrategy#transfer()}. + */ + public TransferStrategy makePutTransferStrategy() { + failureActivity = FailureEvent.FailureActivity.PuttingObject; + + switch (transferBehaviorType) { + case StreamingTransferBehavior: + return makeStreamingPutTransferStrategy(); + + case RandomAccessTransferBehavior: + return makeRandomAccessPutTransferStrategy(); + + case OriginalSdkTransferBehavior: + default: + return makeOriginalSdkSemanticsPutTransferStrategy(); + } + } + + private TransferStrategy makeStreamingPutTransferStrategy() { + maybeMakeStreamedPutChannelStrategy(); + getOrMakeTransferRetryDecorator(); + + return makeTransferStrategy( + new BlobStrategyMaker() { + @Override + public BlobStrategy makeBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher) + { + return new PutSequentialBlobStrategy(ds3Client, + masterObjectList, + eventDispatcher, + getOrMakeChunkAttemptRetryBehavior(), + getOrMakeChunkAllocationRetryDelayBehavior() + ); + } + }, + new TransferMethodMaker() { + @Override + public TransferMethod makeTransferMethod() { + return makePutTransferMethod(); + } + }); + } + + private void maybeMakeStreamedPutChannelStrategy() { + if (channelStrategy == null) { + Preconditions.checkNotNull(channelBuilder, "channelBuilder my not be null"); + + channelStrategy = new SequentialChannelStrategy(new SequentialFileReaderChannelStrategy(channelBuilder), + channelBuilder, new NullChannelPreparable()); + } + } + + private TransferRetryDecorator getOrMakeTransferRetryDecorator() { + if (transferRetryDecorator != null) { + return transferRetryDecorator; + } + + if (numTransferRetries > 0) { + transferRetryDecorator = new MaxNumObjectTransferAttemptsDecorator(numTransferRetries); + } else { + transferRetryDecorator = new ContinueForeverTransferRetryDecorator(); + } + + return transferRetryDecorator; + } + + private TransferStrategy makeTransferStrategy(final BlobStrategyMaker blobStrategyMaker, + final TransferMethodMaker transferMethodMaker) + { + Preconditions.checkNotNull(ds3Client, "ds3Client may not be null."); + Preconditions.checkNotNull(eventDispatcher, "eventDispatcher may not be null."); + Preconditions.checkNotNull(masterObjectList, "masterObjectList may not be null."); + Preconditions.checkState( ! Guard.isStringNullOrEmpty(jobId), "jobId may not be null or an emptystring"); + Preconditions.checkNotNull(blobStrategyMaker, "blobStrategyMaker may not be null."); + Preconditions.checkNotNull(transferMethodMaker, "transferMethodMaker may not be null."); + Preconditions.checkNotNull(channelStrategy, "channelStrategy may not be null"); + Preconditions.checkNotNull(transferRetryDecorator, "transferRetryDecorator may not be null"); + + Guard.throwOnNullOrEmptyString(bucketName, "bucketName may not be null or empty."); + + Guard.throwOnNullOrEmptyString(jobId, "jobId may not be null or empty."); + + maybeMakeBlobStrategy(blobStrategyMaker); + + eventDispatcher.attachBlobTransferredEventObserver(new BlobTransferredEventObserver(new UpdateStrategy() { + @Override + public void update(final BulkObject eventData) { + jobPartTracker.completePart(eventData.getName(), new ObjectPart(eventData.getOffset(), eventData.getLength())); + } + })); + + return makeTransferStrategy(transferMethodMaker.makeTransferMethod()); + } + + private void maybeMakeBlobStrategy(final BlobStrategyMaker blobStrategyMaker) { + if (blobStrategy == null) { + blobStrategy = blobStrategyMaker.makeBlobStrategy(ds3Client, masterObjectList, eventDispatcher); + } + } + + private TransferStrategy makeTransferStrategy(final TransferMethod transferMethod) { + switch (transferBehaviorType) { + case StreamingTransferBehavior: + return new SingleThreadedTransferStrategy(blobStrategy, + jobState, + eventDispatcher, + masterObjectList, + failureActivity) + .withTransferMethod(transferMethod); + + case RandomAccessTransferBehavior: + return new MultiThreadedTransferStrategy(blobStrategy, + jobState, + numConcurrentTransferThreads, + eventDispatcher, + masterObjectList, + failureActivity) + .withTransferMethod(transferMethod); + + case OriginalSdkTransferBehavior: + default: + return makeOriginalSdkSemanticsTransferStrategy(transferMethod); + } + } + + private ChunkAttemptRetryBehavior getOrMakeChunkAttemptRetryBehavior() { + if (chunkAttemptRetryBehavior != null) { + return chunkAttemptRetryBehavior; + } + + if (numChunkAttemptRetries > 0) { + chunkAttemptRetryBehavior = new MaxChunkAttemptsRetryBehavior(numChunkAttemptRetries); + } else { + chunkAttemptRetryBehavior = new ContinueForeverChunkAttemptsRetryBehavior(); + } + + return chunkAttemptRetryBehavior; + } + + private ChunkAttemptRetryDelayBehavior getOrMakeChunkAllocationRetryDelayBehavior() { + Preconditions.checkNotNull(eventDispatcher, "eventDispatcher may not be null."); + + if (chunkAttemptRetryDelayBehavior != null) { + return chunkAttemptRetryDelayBehavior; + } + + if (chunkRetryDelayInSeconds > 0) { + chunkAttemptRetryDelayBehavior = new ClientDefinedChunkAttemptRetryDelayBehavior(chunkRetryDelayInSeconds, eventDispatcher); + } else { + chunkAttemptRetryDelayBehavior = new BlackPearlChunkAttemptRetryDelayBehavior(eventDispatcher); + } + + return chunkAttemptRetryDelayBehavior; + } + + private TransferStrategy makeOriginalSdkSemanticsPutTransferStrategy() { + maybeMakeRandomAccessPutChannelStrategy(); + getOrMakeTransferRetryDecorator(); + + return makeTransferStrategy( + new BlobStrategyMaker() { + @Override + public BlobStrategy makeBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher) + { + return new PutSequentialBlobStrategy(ds3Client, + masterObjectList, + eventDispatcher, + getOrMakeChunkAttemptRetryBehavior(), + getOrMakeChunkAllocationRetryDelayBehavior() + ); + } + }, + new TransferMethodMaker() { + @Override + public TransferMethod makeTransferMethod() { + return makePutTransferMethod(); + } + }); + } + + private void maybeMakeRandomAccessPutChannelStrategy() { + if (channelStrategy == null) { + Preconditions.checkNotNull(channelBuilder, "channelBuilder my not be null"); + channelStrategy = new RandomAccessChannelStrategy(channelBuilder, rangesForBlobs, new NullChannelPreparable()); + } + } + + private TransferMethod makePutTransferMethod() { + getOrMakeJobStateForPutJob(); + + if (checksumType != ChecksumType.Type.NONE) { + maybeAddChecksumFunction(); + } + + final TransferMethod transferMethod = new PutJobTransferMethod(channelStrategy, + bucketName, jobId, eventDispatcher, checksumFunction, checksumType, metadataAccess); + + if (transferRetryDecorator != null) { + return transferRetryDecorator.wrap(transferMethod); + } + + return transferMethod; + } + + private JobState getOrMakeJobStateForPutJob() { + if (jobState != null) { + return jobState; + } + + Preconditions.checkNotNull(masterObjectList, "masterObjectList may not be null."); + Preconditions.checkNotNull(eventDispatcher, "eventDispatcher may not be null."); + Preconditions.checkNotNull(eventRunner, "eventRunner may not be null."); + + List chunks = masterObjectList.getObjects(); + + if (chunks == null) { + chunks = new ArrayList<>(); + } + + final List chunksNotYetCompleted = StrategyUtils.filterChunks(chunks); + + getOrMakeJobPartTrackerForPutJob(chunksNotYetCompleted); + + jobState = new JobState(chunksNotYetCompleted, jobPartTracker); + + return jobState; + } + + private void maybeAddChecksumFunction() { + if (checksumFunction == null) { + makeDefaultChecksumFunction(); + } + } + + private TransferStrategy makeOriginalSdkSemanticsTransferStrategy(final TransferMethod transferMethod) { + if (numConcurrentTransferThreads > 1) { + return new MultiThreadedTransferStrategy(blobStrategy, + jobState, + numConcurrentTransferThreads, + eventDispatcher, + masterObjectList, + failureActivity) + .withTransferMethod(transferMethod); + } else { + return new SingleThreadedTransferStrategy(blobStrategy, + jobState, + eventDispatcher, + masterObjectList, + failureActivity) + .withTransferMethod(transferMethod); + } + } + + private JobPartTracker getOrMakeJobPartTrackerForPutJob(final List chunksNotYetCompleted) { + if (jobPartTracker != null) { + return jobPartTracker; + } + + final JobPartTracker result = JobPartTrackerFactory.buildPartTracker(Iterables.concat(getBlobs(chunksNotYetCompleted)), eventRunner); + + result.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + eventDispatcher.emitObjectCompletedEvent(name); + } + }); + + jobPartTracker = result; + + return jobPartTracker; + } + + private ImmutableList getBlobs(final List chunks) { + final ImmutableList.Builder builder = ImmutableList.builder(); + for (final Objects objects : chunks) { + builder.addAll(objects.getObjects()); + } + return builder.build(); + } + + private void makeDefaultChecksumFunction() { + final ChecksumFunction newChecksumFunction = new ChecksumFunction() { + @Override + public String compute(final BulkObject obj, final ByteChannel channel) { + String checksum = null; + + try + { + final InputStream dataStream = new SeekableByteChannelInputStream(channelStrategy.acquireChannelForBlob(obj)); + + dataStream.mark(Integer.MAX_VALUE); + + final Hasher hasher = ChecksumUtils.getHasher(checksumType); + + checksum = ChecksumUtils.hashInputStream(hasher, dataStream); + + LOG.info("Computed checksum for blob: {}", checksum); + + dataStream.reset(); + } catch (final IOException e) { + eventDispatcher.emitFailureEvent(FailureEvent.builder() + .withObjectNamed(obj.getName()) + .withCausalException(e) + .usingSystemWithEndpoint(ds3Client.getConnectionDetails().getEndpoint()) + .doingWhat(FailureEvent.FailureActivity.ComputingChecksum) + .build()); + LOG.error("Error computing checksum.", e); + } + + return checksum; + } + }; + + checksumFunction = newChecksumFunction; + } + + private TransferStrategy makeRandomAccessPutTransferStrategy() { + maybeMakeRandomAccessPutChannelStrategy(); + getOrMakeTransferRetryDecorator(); + + return makeTransferStrategy( + new BlobStrategyMaker() { + @Override + public BlobStrategy makeBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher) + { + return new PutSequentialBlobStrategy(ds3Client, + masterObjectList, + eventDispatcher, + getOrMakeChunkAttemptRetryBehavior(), + getOrMakeChunkAllocationRetryDelayBehavior() + ); + } + }, + new TransferMethodMaker() { + @Override + public TransferMethod makeTransferMethod() { + return makePutTransferMethod(); + } + }); + } + + /** + * Get an instance of {@link TransferStrategy} configured with this builders current settings. + * @return Once you have an instance, you initiate a get by calling {@link TransferStrategy#transfer()}. + */ + public TransferStrategy makeGetTransferStrategy() { + failureActivity = FailureEvent.FailureActivity.GettingObject; + + switch (transferBehaviorType) { + case StreamingTransferBehavior: + return makeStreamingGetTransferStrategy(); + + case RandomAccessTransferBehavior: + case OriginalSdkTransferBehavior: + default: + return makeOriginalSdkSemanticsGetTransferStrategy(); + } + } + + private TransferStrategy makeStreamingGetTransferStrategy() { + maybeMakeSequentialGetChannelStrategy(); + getOrMakeTransferRetryDecorator(); + + return makeTransferStrategy( + new BlobStrategyMaker() { + @Override + public BlobStrategy makeBlobStrategy(final Ds3Client client, final MasterObjectList masterObjectList, final EventDispatcher eventDispatcher) { + return new GetSequentialBlobStrategy(ds3Client, + masterObjectList, + eventDispatcher, + getOrMakeChunkAttemptRetryBehavior(), + getOrMakeChunkAllocationRetryDelayBehavior()); + } + }, + new TransferMethodMaker() { + @Override + public TransferMethod makeTransferMethod() { + return makeGetTransferMethod(); + } + }); + } + + private void maybeMakeSequentialGetChannelStrategy() { + if (channelStrategy == null) { + Preconditions.checkNotNull(channelBuilder, "channelBuilder my not be null"); + + channelStrategy = new SequentialChannelStrategy(new SequentialFileWriterChannelStrategy(channelBuilder), + channelBuilder, new TruncatingChannelPreparable()); + } + } + + private TransferMethod makeGetTransferMethod() { + getOrMakeJobStateForGetJob(); + + final TransferMethod transferMethod = new GetJobNetworkFailureRetryDecorator(channelStrategy, + bucketName, jobId, eventDispatcher, rangesForBlobs); + + if (transferRetryDecorator != null) { + return transferRetryDecorator.wrap(transferMethod); + } + + return transferMethod; + } + + private JobState getOrMakeJobStateForGetJob() { + if (jobState != null) { + return jobState; + } + + Preconditions.checkNotNull(masterObjectList, "masterObjectList may not be null."); + Preconditions.checkNotNull(eventDispatcher, "eventDispatcher may not be null."); + Preconditions.checkNotNull(eventRunner, "eventRunner may not be null."); + + final List chunks = masterObjectList.getObjects(); + + getOrMakeJobPartTrackerForGetJob(chunks); + + jobState = new JobState(chunks, jobPartTracker); + + return jobState; + } + + private JobPartTracker getOrMakeJobPartTrackerForGetJob(final List chunks) { + if (jobPartTracker != null) { + return jobPartTracker; + } + + final JobPartTracker result = JobPartTrackerFactory.buildPartTracker(getBlobs(chunks), eventRunner); + + result.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + eventDispatcher.emitObjectCompletedEvent(name); + } + }); + + jobPartTracker = result; + + return jobPartTracker; + } + + private TransferStrategy makeOriginalSdkSemanticsGetTransferStrategy() { + maybeMakeRandomAccessGetChannelStrategy(); + getOrMakeTransferRetryDecorator(); + + return makeTransferStrategy( + new BlobStrategyMaker() { + @Override + public BlobStrategy makeBlobStrategy(final Ds3Client client, + final MasterObjectList masterObjectList, + final EventDispatcher eventDispatcher) { + return new GetSequentialBlobStrategy(ds3Client, + masterObjectList, + eventDispatcher, + getOrMakeChunkAttemptRetryBehavior(), + getOrMakeChunkAllocationRetryDelayBehavior()); + } + }, + new TransferMethodMaker() { + @Override + public TransferMethod makeTransferMethod() { + return makeGetTransferMethod(); + } + }); + } + + private void maybeMakeRandomAccessGetChannelStrategy() { + if (channelStrategy == null) { + Preconditions.checkNotNull(channelBuilder, "channelBuilder my not be null"); + + channelStrategy = new RandomAccessChannelStrategy(channelBuilder, rangesForBlobs, new TruncatingChannelPreparable()); + } + } + + private enum TransferBehaviorType { + OriginalSdkTransferBehavior, + StreamingTransferBehavior, + RandomAccessTransferBehavior + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/UpdateStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/UpdateStrategy.java new file mode 100644 index 000000000..7828109be --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/UpdateStrategy.java @@ -0,0 +1,23 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +/** + * This is a marker interface intended to allow for building different ways of emitting events. + * @param The context supplied to an observer registered to receive events. + */ +public interface UpdateStrategy extends Observer { +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/WaitingForChunksObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/WaitingForChunksObserver.java new file mode 100644 index 000000000..951bcd224 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/WaitingForChunksObserver.java @@ -0,0 +1,41 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.helpers.WaitingForChunksListener; + +/** + * This class emits an event when chunks we are creating or retrieving from a Black Pearl + * are not yet available. The event reports the number of seconds we will wait before + * trying again. + */ +public class WaitingForChunksObserver extends AbstractObserver { + public WaitingForChunksObserver(final WaitingForChunksListener waitingForChunksListener) { + super(new UpdateStrategy() { + @Override + public void update(final Integer eventData) { + waitingForChunksListener.waiting(eventData); + } + }); + + Preconditions.checkNotNull(waitingForChunksListener, "waitingForChunksListener may not be null."); + } + + public WaitingForChunksObserver(final UpdateStrategy updateStrategy) { + super(updateStrategy); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/hashing/ChecksumUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/hashing/ChecksumUtils.java new file mode 100644 index 000000000..198f0db38 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/hashing/ChecksumUtils.java @@ -0,0 +1,61 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils.hashing; + +import com.spectralogic.ds3client.models.ChecksumType; + +import java.io.IOException; +import java.io.InputStream; + +public final class ChecksumUtils { + private static final int READ_BUFFER_SIZE = 10 * 1024 * 1024; + + private ChecksumUtils() {} + + public static Hasher getHasher(final ChecksumType.Type checksumType) { + switch (checksumType) { + case MD5: + return new MD5Hasher(); + case SHA_256: + return new SHA256Hasher(); + case SHA_512: + return new SHA512Hasher(); + case CRC_32: + return new CRC32Hasher(); + case CRC_32C: + return new CRC32CHasher(); + default: + throw new RuntimeException("Unknown checksum type " + checksumType.toString()); + } + } + + public static String hashInputStream(final Hasher digest, final InputStream stream) throws IOException { + final byte[] buffer = new byte[READ_BUFFER_SIZE]; + int bytesRead; + + while (true) { + bytesRead = stream.read(buffer); + + if (bytesRead < 0) { + break; + } + + digest.update(buffer, 0, bytesRead); + } + + return digest.digest(); + } +} diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/AutoCloseableCache_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/AutoCloseableCache_Test.java deleted file mode 100644 index a228f3673..000000000 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/AutoCloseableCache_Test.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * ****************************************************************************** - * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.helpers; - -import com.spectralogic.ds3client.helpers.AutoCloseableCache.ValueBuilder; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.sameInstance; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; - -public class AutoCloseableCache_Test { - @Test - public void returnsSameInstanceWhenCalledTwice() throws Exception { - final AutoCloseableCache closeableCache = - new AutoCloseableCache<>(new CloseableImplValueBuilder()); - final CloseableImpl item = closeableCache.get("foo"); - assertThat(item.getKey(), is("foo")); - assertThat(closeableCache.get("foo"), is(sameInstance(item))); - closeableCache.close(); - } - - @Test - public void cacheDisposesAllItemsExactlyOnce() throws Exception { - final AutoCloseableCache closeableCache = - new AutoCloseableCache<>(new CloseableImplValueBuilder()); - - final CloseableImpl item1 = closeableCache.get("foo"); - final CloseableImpl item2 = closeableCache.get("bar"); - - assertThat(item1.getCloseCallCount(), is(0)); - assertThat(item2.getCloseCallCount(), is(0)); - - closeableCache.close(); - - assertThat(item1.getCloseCallCount(), is(1)); - assertThat(item2.getCloseCallCount(), is(1)); - - closeableCache.close(); - - assertThat(item1.getCloseCallCount(), is(1)); - assertThat(item2.getCloseCallCount(), is(1)); - } - - @Test - public void cacheCanCloseItemsEarly() throws Exception { - final AutoCloseableCache closeableCache = - new AutoCloseableCache<>(new CloseableImplValueBuilder()); - - final CloseableImpl item1 = closeableCache.get("foo"); - final CloseableImpl item2 = closeableCache.get("bar"); - - assertThat(item1.getCloseCallCount(), is(0)); - assertThat(item2.getCloseCallCount(), is(0)); - - closeableCache.close("foo"); - - assertThat(item1.getCloseCallCount(), is(1)); - assertThat(item2.getCloseCallCount(), is(0)); - - try { - closeableCache.get("foo"); - fail(); - } catch (final IllegalStateException e) { - } - - closeableCache.close(); - - assertThat(item1.getCloseCallCount(), is(1)); - assertThat(item2.getCloseCallCount(), is(1)); - } - - @Test - public void cacheCanCloseBeforeGet() throws Exception { - final AutoCloseableCache closeableCache = - new AutoCloseableCache<>(new CloseableImplValueBuilder()); - - closeableCache.close("foo"); - - try { - closeableCache.get("foo"); - fail(); - } catch (final IllegalStateException e) { - } - } - - @Test - public void cacheCallsFailWhenDisposed() throws Exception { - final AutoCloseableCache closeableCache = - new AutoCloseableCache<>(new CloseableImplValueBuilder()); - closeableCache.close(); - - try { - closeableCache.get("foo"); - fail(); - } catch (final IllegalStateException e) { - } - try { - closeableCache.close("foo"); - fail(); - } catch (final IllegalStateException e) { - } - } - - private final class CloseableImplValueBuilder implements ValueBuilder { - @Override - public CloseableImpl get(final String key) { - return new CloseableImpl(key); - } - } - - private static final class CloseableImpl implements AutoCloseable { - private final String key; - private int closeCallCount = 0; - - public CloseableImpl(final String key) { - this.key = key; - } - - public String getKey() { - return this.key; - } - - public int getCloseCallCount() { - return this.closeCallCount; - } - - @Override - public void close() throws Exception { - this.closeCallCount++; - } - } -} diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RangeHelperImpl_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelperImpl_Test.java similarity index 99% rename from ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RangeHelperImpl_Test.java rename to ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelperImpl_Test.java index 7cb229450..9b67890db 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RangeHelperImpl_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelperImpl_Test.java @@ -13,7 +13,7 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.helpers; +package com.spectralogic.ds3client.helpers.strategy.transferstrategy; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; From eeff4fe0efdcfe23675ff059f3a1e736a9f9c327 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Wed, 5 Apr 2017 10:29:31 -0600 Subject: [PATCH 002/229] Merging transfer strategy stuff with main. --- .../ds3client/integration/PutJobManagement_Test.java | 2 -- .../channelbuilders/WriteOnlySeekableByteChannel.java | 2 +- .../GetJobNetworkFailureRetryDecorator.java | 10 +++++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index 10a77cb29..c9b653fbc 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -1819,7 +1819,6 @@ public void onFailure(final FailureEvent failureEvent) { } } - /* @Test public void testPutting15000Files() throws IOException, URISyntaxException { final String tempPathPrefix = null; @@ -1880,5 +1879,4 @@ public void testPutting15000Files() throws IOException, URISyntaxException { deleteAllContents(client, BUCKET_NAME); } } - */ } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/channelbuilders/WriteOnlySeekableByteChannel.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/channelbuilders/WriteOnlySeekableByteChannel.java index 3aaeceda0..73b12303c 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/channelbuilders/WriteOnlySeekableByteChannel.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/channelbuilders/WriteOnlySeekableByteChannel.java @@ -68,7 +68,7 @@ public long size() throws IOException { @Override public SeekableByteChannel truncate(final long size) throws IOException { - throw new NotImplementedException(); + return this; } @Override diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java index 487d680b7..d651fc8bb 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java @@ -26,7 +26,7 @@ public class GetJobNetworkFailureRetryDecorator implements TransferMethod { private final EventDispatcher eventDispatcher; private final ImmutableMap> rangesForBlobs; - private final Map transferStateMap = new HashMap<>(); + private final Map transferMethodMap = new HashMap<>(); public GetJobNetworkFailureRetryDecorator(final ChannelStrategy channelStrategy, final String bucketName, @@ -52,7 +52,7 @@ public void transferJobPart(final JobPart jobPart) throws IOException { } private synchronized TransferMethod getOrMakeTransferMethod(final JobPart jobPart) { - TransferState transferState = transferStateMap.get(jobPart); + TransferState transferState = transferMethodMap.get(jobPart); if (transferState != null) { return transferState.getTransferMethod(); @@ -60,7 +60,7 @@ private synchronized TransferMethod getOrMakeTransferMethod(final JobPart jobPar transferState = makeInitialTransferState(); - transferStateMap.put(jobPart, transferState); + transferMethodMap.put(jobPart, transferState); return transferState.getTransferMethod(); } @@ -79,7 +79,7 @@ private TransferState makeInitialTransferState() { } private synchronized void makeTransferMethodForPartialRetry(final JobPart jobPart, final long numBytesTransferred) { - final TransferState existingTransferState = transferStateMap.get(jobPart); + final TransferState existingTransferState = transferMethodMap.get(jobPart); ImmutableCollection ranges = initializeRanges(jobPart.getBulkObject(), existingTransferState); final Long numBytesToTransfer = initializeNumBytesToTransfer(existingTransferState, ranges); @@ -98,7 +98,7 @@ private synchronized void makeTransferMethodForPartialRetry(final JobPart jobPar destinationChannelOffset) ); - transferStateMap.put(jobPart, newTransferState); + transferMethodMap.put(jobPart, newTransferState); } private ImmutableCollection initializeRanges(final BulkObject blob, final TransferState existingTransferState) { From 37fb188bda77d76d511e360f0364d0c0f0e7066b Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Wed, 5 Apr 2017 10:43:10 -0600 Subject: [PATCH 003/229] Merging transfer strategy stuff with main. --- .../blobstrategy/AbstractBlobStrategy.java | 2 +- .../BlackPearlChunkAttemptRetryDelayBehavior.java | 15 +++++++++++++++ .../strategy/blobstrategy/BlobStrategy.java | 2 +- .../strategy/blobstrategy/BlobStrategyMaker.java | 2 +- .../blobstrategy/ChunkAttemptRetryBehavior.java | 2 +- .../ChunkAttemptRetryDelayBehavior.java | 2 +- ...ientDefinedChunkAttemptRetryDelayBehavior.java | 15 +++++++++++++++ ...ContinueForeverChunkAttemptsRetryBehavior.java | 2 +- .../MaxChunkAttemptsRetryBehavior.java | 15 +++++++++++++++ .../channelstrategy/ChannelPreparable.java | 2 +- .../strategy/channelstrategy/ChannelStrategy.java | 2 +- .../channelstrategy/NullChannelPreparable.java | 2 +- .../RandomAccessChannelStrategy.java | 2 +- .../SeekableByteChannelDecorator.java | 2 +- .../SequentialChannelStrategy.java | 2 +- .../SequentialFileReaderChannelStrategy.java | 2 +- .../SequentialFileWriterChannelStrategy.java | 2 +- .../TruncatingChannelPreparable.java | 2 +- .../transferstrategy/AbstractObserver.java | 2 +- .../AbstractTransferStrategy.java | 2 +- .../BlobTransferredEventObserver.java | 2 +- .../strategy/transferstrategy/ChecksumEvent.java | 2 +- .../transferstrategy/ChecksumObserver.java | 2 +- .../ContinueForeverTransferRetryDecorator.java | 15 +++++++++++++++ .../transferstrategy/DataTransferredObserver.java | 3 +-- .../transferstrategy/EventDispatcher.java | 2 +- .../transferstrategy/EventDispatcherImpl.java | 2 +- .../transferstrategy/FailureEventObserver.java | 2 +- .../GetJobNetworkFailureRetryDecorator.java | 15 +++++++++++++++ .../GetJobPartialBlobTransferMethod.java | 15 +++++++++++++++ .../transferstrategy/GetJobTransferMethod.java | 2 +- .../MaxNumObjectTransferAttemptsDecorator.java | 2 +- .../MetaDataReceivedObserver.java | 2 +- .../MultiThreadedTransferStrategy.java | 2 +- .../transferstrategy/ObjectCompletedObserver.java | 2 +- .../strategy/transferstrategy/Observer.java | 2 +- .../transferstrategy/PutJobTransferMethod.java | 3 +-- .../strategy/transferstrategy/RangeHelper.java | 15 +++++++++++++++ .../SingleThreadedTransferStrategy.java | 2 +- .../strategy/transferstrategy/TransferMethod.java | 2 +- .../transferstrategy/TransferMethodMaker.java | 2 +- .../transferstrategy/TransferRetryDecorator.java | 2 +- .../transferstrategy/TransferStrategy.java | 2 +- .../transferstrategy/TransferStrategyBuilder.java | 2 +- .../strategy/transferstrategy/UpdateStrategy.java | 2 +- .../WaitingForChunksObserver.java | 2 +- 46 files changed, 144 insertions(+), 41 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/AbstractBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/AbstractBlobStrategy.java index 450ba31dd..e9d9d4a3f 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/AbstractBlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/AbstractBlobStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlackPearlChunkAttemptRetryDelayBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlackPearlChunkAttemptRetryDelayBehavior.java index bd4c3aa8c..46ba540b8 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlackPearlChunkAttemptRetryDelayBehavior.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlackPearlChunkAttemptRetryDelayBehavior.java @@ -1,3 +1,18 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + package com.spectralogic.ds3client.helpers.strategy.blobstrategy; import com.google.common.base.Preconditions; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategy.java index ec1ad0848..6b94c20e9 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategyMaker.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategyMaker.java index 2dfde31e5..40d254142 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategyMaker.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/BlobStrategyMaker.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryBehavior.java index b2be8250f..dc379524b 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryBehavior.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryBehavior.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryDelayBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryDelayBehavior.java index c70a886cc..9ab58ce01 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryDelayBehavior.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ChunkAttemptRetryDelayBehavior.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ClientDefinedChunkAttemptRetryDelayBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ClientDefinedChunkAttemptRetryDelayBehavior.java index 81d031d88..167e60bfd 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ClientDefinedChunkAttemptRetryDelayBehavior.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ClientDefinedChunkAttemptRetryDelayBehavior.java @@ -1,3 +1,18 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + package com.spectralogic.ds3client.helpers.strategy.blobstrategy; import com.google.common.base.Preconditions; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ContinueForeverChunkAttemptsRetryBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ContinueForeverChunkAttemptsRetryBehavior.java index b917b683e..6a32bc084 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ContinueForeverChunkAttemptsRetryBehavior.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/ContinueForeverChunkAttemptsRetryBehavior.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/MaxChunkAttemptsRetryBehavior.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/MaxChunkAttemptsRetryBehavior.java index c9d832597..4fbc70507 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/MaxChunkAttemptsRetryBehavior.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/MaxChunkAttemptsRetryBehavior.java @@ -1,3 +1,18 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + package com.spectralogic.ds3client.helpers.strategy.blobstrategy; import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelPreparable.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelPreparable.java index aaec19ebb..4f61319d2 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelPreparable.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelPreparable.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelStrategy.java index 965d455d9..0cae311e1 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/ChannelStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/NullChannelPreparable.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/NullChannelPreparable.java index a3ac53908..f152ce3a5 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/NullChannelPreparable.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/NullChannelPreparable.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/RandomAccessChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/RandomAccessChannelStrategy.java index 87c4da13d..11a837db8 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/RandomAccessChannelStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/RandomAccessChannelStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java index 1ed73bd5a..a6feecf4b 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java index 06e441c49..23a672967 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileReaderChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileReaderChannelStrategy.java index e10a07d86..fe12359f2 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileReaderChannelStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileReaderChannelStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileWriterChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileWriterChannelStrategy.java index 2122aa1f8..7126e82d6 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileWriterChannelStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialFileWriterChannelStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/TruncatingChannelPreparable.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/TruncatingChannelPreparable.java index f589505f0..932fe7990 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/TruncatingChannelPreparable.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/TruncatingChannelPreparable.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractObserver.java index 8c03f5a74..69fac1560 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractObserver.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java index a8648d340..4ed231da5 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/BlobTransferredEventObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/BlobTransferredEventObserver.java index 15e15fe28..fc0c38655 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/BlobTransferredEventObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/BlobTransferredEventObserver.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumEvent.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumEvent.java index 3b1aeadf7..6011b0815 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumEvent.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumEvent.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java index 73f3bdf72..15fe85cf3 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ContinueForeverTransferRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ContinueForeverTransferRetryDecorator.java index 64e200200..c2a4954ed 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ContinueForeverTransferRetryDecorator.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ContinueForeverTransferRetryDecorator.java @@ -1,3 +1,18 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + package com.spectralogic.ds3client.helpers.strategy.transferstrategy; import com.google.common.base.Preconditions; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java index a4b0c6976..d263d6c94 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * @@ -17,7 +17,6 @@ import com.google.common.base.Preconditions; import com.spectralogic.ds3client.helpers.DataTransferredListener; -import org.slf4j.Logger; public class DataTransferredObserver extends AbstractObserver { private DataTransferredListener dataTransferredListener; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java index ee26f2abc..0810ac7bb 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java index 905066ae0..946648f78 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java index 0a5c6aed2..31886c892 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java index d651fc8bb..9197ed700 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java @@ -1,3 +1,18 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + package com.spectralogic.ds3client.helpers.strategy.transferstrategy; import com.google.common.collect.ImmutableCollection; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java index 262ff499d..abe6e5c8d 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java @@ -1,3 +1,18 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + package com.spectralogic.ds3client.helpers.strategy.transferstrategy; import com.google.common.base.Preconditions; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java index 1e4bdb9d0..caac43a3c 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MaxNumObjectTransferAttemptsDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MaxNumObjectTransferAttemptsDecorator.java index 5fe828d77..436e42f91 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MaxNumObjectTransferAttemptsDecorator.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MaxNumObjectTransferAttemptsDecorator.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MetaDataReceivedObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MetaDataReceivedObserver.java index 60d1229f5..80b7af56d 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MetaDataReceivedObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MetaDataReceivedObserver.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java index 49eb033e3..bc79af4e9 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java index 61dee7966..be5e03145 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/Observer.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/Observer.java index 2fc916811..8d777a7dc 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/Observer.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/Observer.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java index a9bbe0c92..753f5ee9a 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * @@ -18,7 +18,6 @@ import com.google.common.collect.ImmutableMap; import com.spectralogic.ds3client.commands.PutObjectRequest; import com.spectralogic.ds3client.helpers.ChecksumFunction; -import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; import com.spectralogic.ds3client.helpers.JobPart; import com.spectralogic.ds3client.helpers.MetadataAccess; import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java index 17f008171..90414d5c4 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java @@ -1,3 +1,18 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + package com.spectralogic.ds3client.helpers.strategy.transferstrategy; import com.google.common.base.Preconditions; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java index a37debb2f..f64afb5c5 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethod.java index 603d9d441..8be16f802 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethod.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethod.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethodMaker.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethodMaker.java index b7a394969..f0231aeea 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethodMaker.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferMethodMaker.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferRetryDecorator.java index 22ec734e7..b2853d747 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferRetryDecorator.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferRetryDecorator.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java index fd4088ae1..f67ccf349 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java index 51e437385..064149cd7 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/UpdateStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/UpdateStrategy.java index 7828109be..f5271276a 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/UpdateStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/UpdateStrategy.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/WaitingForChunksObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/WaitingForChunksObserver.java index 951bcd224..6fecc1db7 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/WaitingForChunksObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/WaitingForChunksObserver.java @@ -1,6 +1,6 @@ /* * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"). You may not use * this file except in compliance with the License. A copy of the License is located at * From e913634b36cd31fe2848fb6dd90d4f1f3357b945 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Wed, 5 Apr 2017 11:17:32 -0600 Subject: [PATCH 004/229] Addressing codacity comments --- .../ds3client/integration/GetJobManagement_Test.java | 7 ++----- .../ds3client/integration/PutJobManagement_Test.java | 4 ++-- .../test/helpers/Ds3ClientShimWithFailedGetObject.java | 1 - .../ds3client/helpers/events/FailureEvent.java | 2 -- .../strategy/blobstrategy/PutSequentialBlobStrategy.java | 2 +- .../channelstrategy/SeekableByteChannelDecorator.java | 2 +- .../strategy/transferstrategy/ChecksumObserver.java | 3 --- .../strategy/transferstrategy/DataTransferredObserver.java | 4 ---- .../strategy/transferstrategy/FailureEventObserver.java | 4 ---- .../strategy/transferstrategy/ObjectCompletedObserver.java | 4 ---- .../helpers/strategy/transferstrategy/RangeHelper.java | 4 ++-- .../strategy/transferstrategy/TransferStrategyBuilder.java | 3 +-- 12 files changed, 9 insertions(+), 31 deletions(-) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index 1b75e99fb..d5f7ad85c 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -881,9 +881,6 @@ public void onFailure(final FailureEvent failureEvent) { } }); - final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim - .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); - readJob.transfer(new FileObjectGetter(tempDirectory)); final File originalFile = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME).toFile(); @@ -1174,8 +1171,8 @@ private interface ChannelMonitorable { } private class UserSuppliedPutChannelStrategy implements ChannelStrategy { - final ChannelMonitorable channelMonitorable; - final ChannelStrategy wrappedPutStrategy; + private final ChannelMonitorable channelMonitorable; + private final ChannelStrategy wrappedPutStrategy; private UserSuppliedPutChannelStrategy(final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder, final ChannelMonitorable channelMonitorable) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index c9b653fbc..dc45817bb 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -1492,8 +1492,8 @@ private interface ChannelMonitorable { } private class UserSuppliedPutChannelStrategy implements ChannelStrategy { - final ChannelMonitorable channelMonitorable; - final ChannelStrategy wrappedPutStrategy; + private final ChannelMonitorable channelMonitorable; + private final ChannelStrategy wrappedPutStrategy; private UserSuppliedPutChannelStrategy(final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder, diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShimWithFailedGetObject.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShimWithFailedGetObject.java index f2c9d3a61..54ac8f549 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShimWithFailedGetObject.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/Ds3ClientShimWithFailedGetObject.java @@ -20,7 +20,6 @@ import com.spectralogic.ds3client.commands.GetObjectRequest; import com.spectralogic.ds3client.commands.GetObjectResponse; import com.spectralogic.ds3client.models.JobNode; -import com.spectralogic.ds3client.networking.ConnectionDetails; import java.io.IOException; import java.lang.reflect.InvocationTargetException; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java index 31d385062..b1145a5e6 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java @@ -80,8 +80,6 @@ public static class Builder { private String usingSystemWithEndpoint; private Throwable causalException; - public Builder() { } - public Builder doingWhat(final FailureActivity what) { this.doingWhat = what; return this; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java index 35312e02d..c55982b22 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java @@ -120,6 +120,6 @@ private Objects tryAllocateChunk(final Objects filtered) throws IOException { @Override public void blobCompleted(final BulkObject bulkObject) { - + // Intentionally not implemented } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java index a6feecf4b..2acdc4dab 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java @@ -34,7 +34,7 @@ class SeekableByteChannelDecorator implements SeekableByteChannel { this.seekableByteChannel = seekableByteChannel; } - SeekableByteChannel wrappedSeekableByteChannel() { + protected SeekableByteChannel wrappedSeekableByteChannel() { return seekableByteChannel; } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java index 15fe85cf3..68310bcd8 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ChecksumObserver.java @@ -23,7 +23,6 @@ * transferred to a Black Pearl. */ public class ChecksumObserver extends AbstractObserver { - private ChecksumListener checksumListener; /** * @param checksumListener An instance of {@link ChecksumListener} wrapped in an event updater. @@ -37,8 +36,6 @@ public void update(final ChecksumEvent eventData) { }); Preconditions.checkNotNull(checksumListener, "checksumListener may not be null."); - - this.checksumListener = checksumListener; } /** diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java index d263d6c94..d25191cac 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/DataTransferredObserver.java @@ -19,8 +19,6 @@ import com.spectralogic.ds3client.helpers.DataTransferredListener; public class DataTransferredObserver extends AbstractObserver { - private DataTransferredListener dataTransferredListener; - public DataTransferredObserver(final DataTransferredListener dataTransferredListener) { super(new UpdateStrategy() { @Override @@ -30,8 +28,6 @@ public void update(final Long eventData) { }); Preconditions.checkNotNull(dataTransferredListener, "dataTransferredListener may not be null."); - - this.dataTransferredListener = dataTransferredListener; } public DataTransferredObserver(final UpdateStrategy updateStrategy) { diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java index 31886c892..b273b27bf 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/FailureEventObserver.java @@ -24,8 +24,6 @@ * during a blob transfer. */ public class FailureEventObserver extends AbstractObserver { - private FailureEventListener failureEventListener; - /** * @param failureEventListener An instance of {@link FailureEventListener} wrapped in an event updater. */ @@ -38,8 +36,6 @@ public void update(final FailureEvent eventData) { }); Preconditions.checkNotNull(failureEventListener, "failureEventListener may not be null."); - - this.failureEventListener = failureEventListener; } /** diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java index be5e03145..5524194d7 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/ObjectCompletedObserver.java @@ -22,8 +22,6 @@ * An {@link Observer} that enables sending event when a DS3 object transfer completes. */ public class ObjectCompletedObserver extends AbstractObserver { - private ObjectCompletedListener objectCompletedListener; - /** * @param objectCompletedListener An {@link ObjectCompletedListener} wrapped in an event * updater. @@ -37,8 +35,6 @@ public void update(final String eventData) { }); Preconditions.checkNotNull(objectCompletedListener, "objectCompletedListener may not be null"); - - this.objectCompletedListener = objectCompletedListener; } /** diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java index 90414d5c4..c06264ea7 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/RangeHelper.java @@ -29,7 +29,7 @@ final class RangeHelper { private RangeHelper() {} - static ImmutableCollection replaceRange(final ImmutableCollection existingRanges, + protected static ImmutableCollection replaceRange(final ImmutableCollection existingRanges, final long numBytesTransferred, final long intendedNumBytesToTransfer) { @@ -72,7 +72,7 @@ private static void addRemainingRanges(final UnmodifiableIterator existin } } - static long transferSizeForRanges(final ImmutableCollection existingRanges) { + protected static long transferSizeForRanges(final ImmutableCollection existingRanges) { long result = 0; if (Guard.isNullOrEmpty(existingRanges)) { diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java index 064149cd7..866b24177 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java @@ -54,7 +54,6 @@ import com.spectralogic.ds3client.helpers.strategy.channelstrategy.TruncatingChannelPreparable; import com.spectralogic.ds3client.models.BulkObject; import com.spectralogic.ds3client.models.ChecksumType; -import com.spectralogic.ds3client.models.JobChunkClientProcessingOrderGuarantee; import com.spectralogic.ds3client.models.MasterObjectList; import com.spectralogic.ds3client.models.Objects; import com.spectralogic.ds3client.models.common.Range; @@ -145,7 +144,7 @@ public final class TransferStrategyBuilder { * Use an instance of {@link BlobStrategy} you wish to create or retrieve blobs from a Black Pearl. There are * 2 primary implementations: {@link PutSequentialBlobStrategy}; and {@link GetSequentialBlobStrategy}. Blob * retrieval order is specified in the - * {@link com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request#withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee)} + * {@link com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request#withChunkClientProcessingOrderGuarantee(com.spectralogic.ds3client.models.JobChunkClientProcessingOrderGuarantee)} * HTTP request. * @return The instance of this builder, with the intent that you can string together the behaviors you wish, * for example: From 470294d39b7448f34bb49f2347c87c60b1c97f2c Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Thu, 20 Apr 2017 12:51:51 -0600 Subject: [PATCH 005/229] Adding checks to not process named pipes, sockets or device files. --- .../integration/GetJobManagement_Test.java | 29 +++++++++++++++ .../integration/PutJobManagement_Test.java | 36 +++++++++++++++++++ .../ds3client/helpers/FileObjectGetter.java | 6 ++++ .../ds3client/helpers/FileObjectPutter.java | 4 +++ .../ds3client/utils/FileUtils.java | 14 ++++++++ 5 files changed, 89 insertions(+) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index d5f7ad85c..3540888b6 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -75,6 +75,7 @@ import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -1365,4 +1366,32 @@ private interface ChunkAttemptRetryBehaviorMonitorable { void invoke(); void reset(); } + + @Test + public void testThatFifoIsNotProcessed() throws IOException, InterruptedException { + Assume.assumeFalse(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String BEOWULF_FILE_NAME = "beowulf.txt"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), BEOWULF_FILE_NAME)).waitFor(); + + final List ds3Objects = Arrays.asList(new Ds3Object(BEOWULF_FILE_NAME)); + + final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME, ds3Objects); + readJob.transfer(new FileObjectPutter(tempDirectory)); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(BEOWULF_FILE_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + + assertTrue(caughtException.get()); + } } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index dc45817bb..f0803a2db 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -83,6 +83,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.UUID; @@ -1879,4 +1880,39 @@ public void testPutting15000Files() throws IOException, URISyntaxException { deleteAllContents(client, BUCKET_NAME); } } + + @Test + public void testThatFifoIsNotProcessed() throws IOException, InterruptedException { + Assume.assumeFalse(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String A_FILE_NAME = "aFile.txt"; + final String C_FILE_NAME = "cFile.txt"; + final String FIFO_NAME = "bFifo"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Files.createFile(Paths.get(tempDirectory.toString(), A_FILE_NAME)); + Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); + Files.createFile(Paths.get(tempDirectory.toString(), C_FILE_NAME)); + + final List ds3Objects = Arrays.asList(new Ds3Object(A_FILE_NAME), + new Ds3Object(FIFO_NAME), + new Ds3Object(C_FILE_NAME)); + + final Ds3ClientHelpers.Job writeJob = HELPERS.startWriteJob(BUCKET_NAME, ds3Objects); + writeJob.transfer(new FileObjectPutter(tempDirectory)); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(FIFO_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + deleteAllContents(client, BUCKET_NAME); + } + + assertTrue(caughtException.get()); + } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java index 24e5c5ca3..041edffce 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java @@ -16,6 +16,7 @@ package com.spectralogic.ds3client.helpers; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; +import com.spectralogic.ds3client.utils.FileUtils; import java.io.IOException; import java.nio.channels.FileChannel; @@ -45,6 +46,11 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { if (parentPath != null) { Files.createDirectories(parentPath); } + + if ( ! FileUtils.isRegularFile(parentPath)) { + throw new UnrecoverableIOException(objectPath + " is not a regular file."); + } + return FileChannel.open( objectPath, StandardOpenOption.WRITE, diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java index ed5256308..cdb46a925 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java @@ -43,6 +43,10 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { final Path path = this.root.resolve(key); + if ( ! FileUtils.isRegularFile(path)) { + throw new UnrecoverableIOException(path + " is not a regular file."); + } + return FileChannel.open(FileUtils.resolveForSymbolic(path), StandardOpenOption.READ); } } diff --git a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java index dfade3fd3..055073d63 100644 --- a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java +++ b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; public final class FileUtils { private FileUtils() {} @@ -39,4 +40,17 @@ public static Path resolveForSymbolic(final Path path) throws IOException { } return path; } + + public static boolean isRegularFile(final Path path) { + if ( ! Platform.isWindows()) { + try { + final BasicFileAttributes fileAttributes = Files.readAttributes(path, BasicFileAttributes.class); + return !fileAttributes.isOther(); + } catch (final IOException e) { + return false; + } + } + + return true; + } } \ No newline at end of file From 82a68ba243d84737316aebd175bf7fbfd90af2b6 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Thu, 20 Apr 2017 13:15:56 -0600 Subject: [PATCH 006/229] Adding checks to not process named pipes, sockets or device files. --- .../ds3client/helpers/FileObjectGetter.java | 2 +- .../helpers/FileObjectGetter_Test.java | 80 +++++++++++++++++++ .../helpers/FileObjectPutter_Test.java | 53 ++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java index 041edffce..c4d55fcd4 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java @@ -47,7 +47,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { Files.createDirectories(parentPath); } - if ( ! FileUtils.isRegularFile(parentPath)) { + if ( ! FileUtils.isRegularFile(objectPath)) { throw new UnrecoverableIOException(objectPath + " is not a regular file."); } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java new file mode 100644 index 000000000..82c7ac73b --- /dev/null +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java @@ -0,0 +1,80 @@ +/* + * ****************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.spectralogic.ds3client.utils.Platform; +import org.apache.commons.io.FileUtils; +import org.junit.Assume; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FileObjectGetter_Test { + @Test + public void testThatNamedPipeThrows() throws IOException, InterruptedException { + Assume.assumeFalse(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String FIFO_NAME = "bFifo"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); + new FileObjectGetter(tempDirectory).buildChannel(FIFO_NAME); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(FIFO_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + + assertTrue(caughtException.get()); + } + + @Test + public void testThatFileReportsAsRegularOnWindows() throws IOException, InterruptedException { + Assume.assumeTrue(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String A_FILE_NAME = "aFile.txt"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Files.createFile(Paths.get(tempDirectory.toString(), A_FILE_NAME)); + new FileObjectGetter(tempDirectory).buildChannel(A_FILE_NAME); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(A_FILE_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + + assertFalse(caughtException.get()); + } +} diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java index 12c2507e8..3be147717 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java @@ -16,6 +16,8 @@ package com.spectralogic.ds3client.helpers; import com.spectralogic.ds3client.utils.Platform; +import org.apache.commons.io.FileUtils; +import org.junit.Assume; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,9 +29,12 @@ import java.nio.channels.SeekableByteChannel; import java.nio.charset.Charset; import java.nio.file.*; +import java.util.concurrent.atomic.AtomicBoolean; import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; public class FileObjectPutter_Test { @@ -124,4 +129,52 @@ private static String getParentDir(final Path path) { final int lastIndex = parentPath.lastIndexOf(File.separator); return parentPath.substring(lastIndex + 1); } + + @Test + public void testThatNamedPipeThrows() throws IOException, InterruptedException { + Assume.assumeFalse(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String FIFO_NAME = "bFifo"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); + new FileObjectPutter(tempDirectory).buildChannel(FIFO_NAME); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(FIFO_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + + assertTrue(caughtException.get()); + } + + @Test + public void testThatFileReportsAsRegularOnWindows() throws IOException, InterruptedException { + Assume.assumeTrue(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String A_FILE_NAME = "aFile.txt"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Files.createFile(Paths.get(tempDirectory.toString(), A_FILE_NAME)); + new FileObjectPutter(tempDirectory).buildChannel(A_FILE_NAME); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(A_FILE_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + + assertFalse(caughtException.get()); + } } From 8c409d66aa730d3c4c13317f2d6fd3c0ec875333 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Thu, 20 Apr 2017 16:52:59 -0600 Subject: [PATCH 007/229] Backing out prior commits that added checks to not process named pipes, sockets or device files. The java library function that checks file types does not work reliably on linux. --- .../integration/GetJobManagement_Test.java | 28 --------------- .../integration/PutJobManagement_Test.java | 35 ------------------- .../ds3client/helpers/FileObjectGetter.java | 4 --- .../ds3client/helpers/FileObjectPutter.java | 4 --- .../AbstractTransferStrategy.java | 2 +- .../helpers/FileObjectGetter_Test.java | 24 ------------- .../helpers/FileObjectPutter_Test.java | 24 ------------- .../ds3client/utils/FileUtils.java | 13 ------- 8 files changed, 1 insertion(+), 133 deletions(-) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index 3540888b6..871e59461 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -1366,32 +1366,4 @@ private interface ChunkAttemptRetryBehaviorMonitorable { void invoke(); void reset(); } - - @Test - public void testThatFifoIsNotProcessed() throws IOException, InterruptedException { - Assume.assumeFalse(Platform.isWindows()); - - final String tempPathPrefix = null; - final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); - - final String BEOWULF_FILE_NAME = "beowulf.txt"; - - final AtomicBoolean caughtException = new AtomicBoolean(false); - - try { - Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), BEOWULF_FILE_NAME)).waitFor(); - - final List ds3Objects = Arrays.asList(new Ds3Object(BEOWULF_FILE_NAME)); - - final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME, ds3Objects); - readJob.transfer(new FileObjectPutter(tempDirectory)); - } catch (final UnrecoverableIOException e) { - assertTrue(e.getMessage().contains(BEOWULF_FILE_NAME)); - caughtException.set(true); - } finally { - FileUtils.deleteDirectory(tempDirectory.toFile()); - } - - assertTrue(caughtException.get()); - } } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index f0803a2db..50d8d3d59 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -1880,39 +1880,4 @@ public void testPutting15000Files() throws IOException, URISyntaxException { deleteAllContents(client, BUCKET_NAME); } } - - @Test - public void testThatFifoIsNotProcessed() throws IOException, InterruptedException { - Assume.assumeFalse(Platform.isWindows()); - - final String tempPathPrefix = null; - final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); - - final String A_FILE_NAME = "aFile.txt"; - final String C_FILE_NAME = "cFile.txt"; - final String FIFO_NAME = "bFifo"; - - final AtomicBoolean caughtException = new AtomicBoolean(false); - - try { - Files.createFile(Paths.get(tempDirectory.toString(), A_FILE_NAME)); - Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); - Files.createFile(Paths.get(tempDirectory.toString(), C_FILE_NAME)); - - final List ds3Objects = Arrays.asList(new Ds3Object(A_FILE_NAME), - new Ds3Object(FIFO_NAME), - new Ds3Object(C_FILE_NAME)); - - final Ds3ClientHelpers.Job writeJob = HELPERS.startWriteJob(BUCKET_NAME, ds3Objects); - writeJob.transfer(new FileObjectPutter(tempDirectory)); - } catch (final UnrecoverableIOException e) { - assertTrue(e.getMessage().contains(FIFO_NAME)); - caughtException.set(true); - } finally { - FileUtils.deleteDirectory(tempDirectory.toFile()); - deleteAllContents(client, BUCKET_NAME); - } - - assertTrue(caughtException.get()); - } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java index c4d55fcd4..3ee5f8a55 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java @@ -47,10 +47,6 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { Files.createDirectories(parentPath); } - if ( ! FileUtils.isRegularFile(objectPath)) { - throw new UnrecoverableIOException(objectPath + " is not a regular file."); - } - return FileChannel.open( objectPath, StandardOpenOption.WRITE, diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java index cdb46a925..ed5256308 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java @@ -43,10 +43,6 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { final Path path = this.root.resolve(key); - if ( ! FileUtils.isRegularFile(path)) { - throw new UnrecoverableIOException(path + " is not a regular file."); - } - return FileChannel.open(FileUtils.resolveForSymbolic(path), StandardOpenOption.READ); } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java index 4ed231da5..73df3ef8d 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java @@ -120,7 +120,7 @@ public Void call() throws Exception { })); } - runTransferTasks(ImmutableList.copyOf(transferTasksListBuilder.build())); + runTransferTasks(transferTasksListBuilder.build()); } catch (final IOException | RuntimeException e) { throw e; } catch (final Exception e) { diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java index 82c7ac73b..f378637da 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java @@ -30,30 +30,6 @@ import static org.junit.Assert.assertTrue; public class FileObjectGetter_Test { - @Test - public void testThatNamedPipeThrows() throws IOException, InterruptedException { - Assume.assumeFalse(Platform.isWindows()); - - final String tempPathPrefix = null; - final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); - - final String FIFO_NAME = "bFifo"; - - final AtomicBoolean caughtException = new AtomicBoolean(false); - - try { - Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); - new FileObjectGetter(tempDirectory).buildChannel(FIFO_NAME); - } catch (final UnrecoverableIOException e) { - assertTrue(e.getMessage().contains(FIFO_NAME)); - caughtException.set(true); - } finally { - FileUtils.deleteDirectory(tempDirectory.toFile()); - } - - assertTrue(caughtException.get()); - } - @Test public void testThatFileReportsAsRegularOnWindows() throws IOException, InterruptedException { Assume.assumeTrue(Platform.isWindows()); diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java index 3be147717..68813747a 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java @@ -130,30 +130,6 @@ private static String getParentDir(final Path path) { return parentPath.substring(lastIndex + 1); } - @Test - public void testThatNamedPipeThrows() throws IOException, InterruptedException { - Assume.assumeFalse(Platform.isWindows()); - - final String tempPathPrefix = null; - final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); - - final String FIFO_NAME = "bFifo"; - - final AtomicBoolean caughtException = new AtomicBoolean(false); - - try { - Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); - new FileObjectPutter(tempDirectory).buildChannel(FIFO_NAME); - } catch (final UnrecoverableIOException e) { - assertTrue(e.getMessage().contains(FIFO_NAME)); - caughtException.set(true); - } finally { - FileUtils.deleteDirectory(tempDirectory.toFile()); - } - - assertTrue(caughtException.get()); - } - @Test public void testThatFileReportsAsRegularOnWindows() throws IOException, InterruptedException { Assume.assumeTrue(Platform.isWindows()); diff --git a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java index 055073d63..491a611b2 100644 --- a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java +++ b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java @@ -40,17 +40,4 @@ public static Path resolveForSymbolic(final Path path) throws IOException { } return path; } - - public static boolean isRegularFile(final Path path) { - if ( ! Platform.isWindows()) { - try { - final BasicFileAttributes fileAttributes = Files.readAttributes(path, BasicFileAttributes.class); - return !fileAttributes.isOther(); - } catch (final IOException e) { - return false; - } - } - - return true; - } } \ No newline at end of file From 56ae37a69f51c73eb644f3ecd99a687fb5619c1a Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Mon, 24 Apr 2017 13:34:10 -0600 Subject: [PATCH 008/229] Addign checks to not process files that are not "regular files" or directories. Doing so because trying to transfer fifo's or device special files is bad. --- .../integration/GetJobManagement_Test.java | 28 +++++++++++++++ .../integration/PutJobManagement_Test.java | 35 +++++++++++++++++++ .../ds3client/helpers/FileObjectGetter.java | 10 ++++-- .../ds3client/helpers/FileObjectPutter.java | 4 +++ .../helpers/FileObjectGetter_Test.java | 24 +++++++++++++ .../helpers/FileObjectPutter_Test.java | 24 +++++++++++++ .../ds3client/utils/FileUtils.java | 16 +++++++++ 7 files changed, 138 insertions(+), 3 deletions(-) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index 871e59461..3540888b6 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -1366,4 +1366,32 @@ private interface ChunkAttemptRetryBehaviorMonitorable { void invoke(); void reset(); } + + @Test + public void testThatFifoIsNotProcessed() throws IOException, InterruptedException { + Assume.assumeFalse(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String BEOWULF_FILE_NAME = "beowulf.txt"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), BEOWULF_FILE_NAME)).waitFor(); + + final List ds3Objects = Arrays.asList(new Ds3Object(BEOWULF_FILE_NAME)); + + final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME, ds3Objects); + readJob.transfer(new FileObjectPutter(tempDirectory)); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(BEOWULF_FILE_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + + assertTrue(caughtException.get()); + } } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index 50d8d3d59..f0803a2db 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -1880,4 +1880,39 @@ public void testPutting15000Files() throws IOException, URISyntaxException { deleteAllContents(client, BUCKET_NAME); } } + + @Test + public void testThatFifoIsNotProcessed() throws IOException, InterruptedException { + Assume.assumeFalse(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String A_FILE_NAME = "aFile.txt"; + final String C_FILE_NAME = "cFile.txt"; + final String FIFO_NAME = "bFifo"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Files.createFile(Paths.get(tempDirectory.toString(), A_FILE_NAME)); + Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); + Files.createFile(Paths.get(tempDirectory.toString(), C_FILE_NAME)); + + final List ds3Objects = Arrays.asList(new Ds3Object(A_FILE_NAME), + new Ds3Object(FIFO_NAME), + new Ds3Object(C_FILE_NAME)); + + final Ds3ClientHelpers.Job writeJob = HELPERS.startWriteJob(BUCKET_NAME, ds3Objects); + writeJob.transfer(new FileObjectPutter(tempDirectory)); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(FIFO_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + deleteAllContents(client, BUCKET_NAME); + } + + assertTrue(caughtException.get()); + } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java index 3ee5f8a55..4015f7664 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java @@ -47,10 +47,14 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { Files.createDirectories(parentPath); } + if ( ! FileUtils.isTransferablePath(objectPath)) { + throw new UnrecoverableIOException(objectPath + " is not a regular file."); + } + return FileChannel.open( - objectPath, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE + objectPath, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE ); } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java index ed5256308..f9f568e3b 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectPutter.java @@ -43,6 +43,10 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { final Path path = this.root.resolve(key); + if ( ! FileUtils.isTransferablePath(path)) { + throw new UnrecoverableIOException(path + " is not a regular file."); + } + return FileChannel.open(FileUtils.resolveForSymbolic(path), StandardOpenOption.READ); } } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java index f378637da..82c7ac73b 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java @@ -30,6 +30,30 @@ import static org.junit.Assert.assertTrue; public class FileObjectGetter_Test { + @Test + public void testThatNamedPipeThrows() throws IOException, InterruptedException { + Assume.assumeFalse(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String FIFO_NAME = "bFifo"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); + new FileObjectGetter(tempDirectory).buildChannel(FIFO_NAME); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(FIFO_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + + assertTrue(caughtException.get()); + } + @Test public void testThatFileReportsAsRegularOnWindows() throws IOException, InterruptedException { Assume.assumeTrue(Platform.isWindows()); diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java index 68813747a..bc457f138 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectPutter_Test.java @@ -153,4 +153,28 @@ public void testThatFileReportsAsRegularOnWindows() throws IOException, Interrup assertFalse(caughtException.get()); } + + @Test + public void testThatNamedPipeThrows() throws IOException, InterruptedException { + Assume.assumeFalse(Platform.isWindows()); + + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String FIFO_NAME = "bFifo"; + + final AtomicBoolean caughtException = new AtomicBoolean(false); + + try { + Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), FIFO_NAME)).waitFor(); + new FileObjectPutter(tempDirectory).buildChannel(FIFO_NAME); + } catch (final UnrecoverableIOException e) { + assertTrue(e.getMessage().contains(FIFO_NAME)); + caughtException.set(true); + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + + assertTrue(caughtException.get()); + } } diff --git a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java index 491a611b2..00e9f88bb 100644 --- a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java +++ b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/FileUtils.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; @@ -40,4 +41,19 @@ public static Path resolveForSymbolic(final Path path) throws IOException { } return path; } + + public static boolean isTransferablePath(final Path path) { + if ( ! Platform.isWindows()) { + try { + final BasicFileAttributes fileAttributes = Files.readAttributes(path, BasicFileAttributes.class); + return fileAttributes.isRegularFile() || fileAttributes.isDirectory(); + } catch (final NoSuchFileException e) { + return true; + } catch (final IOException e) { + return false; + } + } + + return true; + } } \ No newline at end of file From 1c051c638d593543d4f10db722341781e14ddb28 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Mon, 24 Apr 2017 14:38:32 -0600 Subject: [PATCH 009/229] Fixing a bug in an integration test where we didn't properly detect whether we are running as root or not. --- .../ds3client/integration/GetJobManagement_Test.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index 3540888b6..02df3663e 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -403,10 +403,16 @@ private boolean iAmRoot() { whoamiProcess.waitFor(); try (final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(whoamiProcess.getInputStream()))) { - final String userId = bufferedReader.readLine(); - return userId.equals("0"); + final String[] uidFields = userId.split("uid="); + if (uidFields.length > 1) { + final String[] uidStrings = uidFields[1].split("\\("); + if (uidStrings.length > 1) { + final int uid = Integer.parseInt(uidStrings[0]); + return uid == 0; + } + } } } catch (final IOException | InterruptedException e) { LOG.error("Error getting user id.", e); From ea964de6553ead4ca557e0cce121ce8a9e10ce4a Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Thu, 27 Apr 2017 16:54:32 -0600 Subject: [PATCH 010/229] Adding an explicit test to make sure that failure in a put job does not cause the put job to stop. Could have sworn we had one of these, but I guess not. --- .../integration/PutJobManagement_Test.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index f0803a2db..90e4ee11c 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -80,6 +80,7 @@ import java.nio.channels.Channels; import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -1915,4 +1916,57 @@ public void testThatFifoIsNotProcessed() throws IOException, InterruptedExceptio assertTrue(caughtException.get()); } + + @Test + public void testThatNonExistentFileDoesNotStopPutJob() throws IOException { + final String tempPathPrefix = null; + final Path tempDirectoryForFilesToPut = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final Path tempDirectoryForFilesToGet = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + final String A_FILE_NAME = "aFile.txt"; + final String B_FILE_NAME = "bFile.txt"; + final String C_FILE_NAME = "cFile.txt"; + + final Path aFilePutPath = Files.createFile(Paths.get(tempDirectoryForFilesToPut.toString(), A_FILE_NAME)); + final Path cFilePutPath = Files.createFile(Paths.get(tempDirectoryForFilesToPut.toString(), C_FILE_NAME)); + + final AtomicBoolean caughtNoSuchFileException = new AtomicBoolean(false); + final AtomicBoolean getJobRan = new AtomicBoolean(false); + + try { + FileUtils.writeStringToFile(aFilePutPath.toFile(), "aFile"); + FileUtils.writeStringToFile(cFilePutPath.toFile(), "cFile"); + + final int sizeOfNonExistentFile = 5; + + final List ds3Objects = Arrays.asList(new Ds3Object(A_FILE_NAME, Files.size(aFilePutPath)), + new Ds3Object(B_FILE_NAME, sizeOfNonExistentFile), + new Ds3Object(C_FILE_NAME, Files.size(cFilePutPath))); + + final Ds3ClientHelpers.Job writeJob = HELPERS.startWriteJob(BUCKET_NAME, ds3Objects); + writeJob.transfer(new FileObjectPutter(tempDirectoryForFilesToPut)); + } catch (final NoSuchFileException e) { + caughtNoSuchFileException.set(true); + + final Path aFileGetPath = Files.createFile(Paths.get(tempDirectoryForFilesToGet.toString(), A_FILE_NAME)); + final Path cFileGetPath = Files.createFile(Paths.get(tempDirectoryForFilesToGet.toString(), C_FILE_NAME)); + + final List ds3Objects = Arrays.asList(new Ds3Object(A_FILE_NAME), new Ds3Object(C_FILE_NAME)); + + final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME, ds3Objects); + readJob.transfer(new FileObjectGetter(tempDirectoryForFilesToGet)); + + getJobRan.set(true); + + assertTrue(FileUtils.contentEquals(aFileGetPath.toFile(), aFilePutPath.toFile())); + assertTrue(FileUtils.contentEquals(cFileGetPath.toFile(), cFilePutPath.toFile())); + } finally { + FileUtils.deleteDirectory(tempDirectoryForFilesToGet.toFile()); + FileUtils.deleteDirectory(tempDirectoryForFilesToPut.toFile()); + deleteAllContents(client, BUCKET_NAME); + } + + assertTrue(caughtNoSuchFileException.get()); + assertTrue(getJobRan.get()); + } } From 7367ebcfe2c19c788d56656a2f383ba485add180 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 16 May 2017 15:25:20 -0600 Subject: [PATCH 011/229] Removing duplicated FailureEvent class --- .../helpers/events/FailureEvent.java | 112 ------------------ .../helpers/events/FailureEvent.java | 9 +- 2 files changed, 5 insertions(+), 116 deletions(-) delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java deleted file mode 100644 index b1145a5e6..000000000 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * **************************************************************************** - * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.helpers.events; - -import com.google.common.base.Preconditions; -import com.spectralogic.ds3client.utils.Guard; - -public class FailureEvent { - private final FailureActivity doingWhat; - private final String objectName; - private final String endpoint; - private final Throwable causalException; - - private FailureEvent(final FailureActivity what, final String objectName, final String endpoint, final Throwable causalException) { - this.doingWhat = what; - this.objectName = objectName; - this.endpoint = endpoint; - this.causalException = causalException; - } - - public FailureActivity doingWhat() { - return doingWhat; - } - - public String withObjectNamed() { - return objectName; - } - - public String usingSystemWithEndpoint() { - return endpoint; - } - - public Throwable getCausalException() { - return causalException; - } - - @Override - public String toString() { - return "Failure " + doingWhat().getActivityText() + " with object named \"" + withObjectNamed() + "\" using system with endpoint " + usingSystemWithEndpoint(); - } - - public static FailureEvent.Builder builder() { - return new FailureEvent.Builder(); - } - - public enum FailureActivity { - PuttingObject("putting object"), - GettingObject("getting object"), - ComputingChecksum("computing checksum"), - RecordingMetadata("recording metadata"), - RestoringMetadata("restoring metadata"); - - private final String activityText; - - FailureActivity(final String activityText) { - this.activityText = activityText; - } - - public String getActivityText() { - return activityText; - } - } - - public static class Builder { - private FailureActivity doingWhat; - private String withObjectNamed; - private String usingSystemWithEndpoint; - private Throwable causalException; - - public Builder doingWhat(final FailureActivity what) { - this.doingWhat = what; - return this; - } - - public Builder withObjectNamed(final String objectName) { - this.withObjectNamed = objectName; - return this; - } - - public Builder usingSystemWithEndpoint(final String endpoint) { - this.usingSystemWithEndpoint = endpoint; - return this; - } - - public Builder withCausalException(final Throwable causalException) { - this.causalException = causalException; - return this; - } - - public FailureEvent build() { - Preconditions.checkNotNull(doingWhat, "The failed activity may not be null or empty."); - Guard.throwOnNullOrEmptyString(withObjectNamed, "The name of the object involved in the activity may not be null or empty."); - Guard.throwOnNullOrEmptyString(usingSystemWithEndpoint, "The endpoint referenced in the activity may not be null or empty."); - Preconditions.checkNotNull(causalException, "The exception causing a failure may not be null."); - - return new FailureEvent(doingWhat, withObjectNamed, usingSystemWithEndpoint, causalException); - } - } -} diff --git a/ds3-utils/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java index 36268ecd5..15cf96890 100644 --- a/ds3-utils/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java +++ b/ds3-utils/src/main/java/com/spectralogic/ds3client/helpers/events/FailureEvent.java @@ -47,18 +47,19 @@ public Throwable getCausalException() { return causalException; } - public static FailureEvent.Builder builder() { - return new FailureEvent.Builder(); - } - @Override public String toString() { return "Failure " + doingWhat().getActivityText() + " with object named \"" + withObjectNamed() + "\" using system with endpoint " + usingSystemWithEndpoint(); } + public static FailureEvent.Builder builder() { + return new FailureEvent.Builder(); + } + public enum FailureActivity { PuttingObject("putting object"), GettingObject("getting object"), + ComputingChecksum("computing checksum"), RecordingMetadata("recording metadata"), RestoringMetadata("restoring metadata"); From 8a4fa15e327e84384938e5f7e860dccbe819a37d Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 23 May 2017 10:57:16 -0600 Subject: [PATCH 012/229] Adding the ability for a user to supply a transfer strategy of their own in read and write jobs. --- .../integration/GetJobManagement_Test.java | 36 +++++++++++++++ .../integration/PutJobManagement_Test.java | 45 +++++++++++++++++++ .../ds3client/helpers/Ds3ClientHelpers.java | 26 +++++++++++ .../helpers/Ds3ClientHelpersImpl.java | 13 ++++++ .../ds3client/helpers/JobImpl.java | 4 ++ .../ds3client/helpers/ReadJobImpl.java | 20 ++++++++- .../ds3client/helpers/WriteJobImpl.java | 19 +++++++- .../helpers/util/PartialObjectHelpers.java | 2 +- .../spectralogic/ds3client/utils/Guard.java | 16 +++++-- 9 files changed, 172 insertions(+), 9 deletions(-) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index 02df3663e..75b59e6d5 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -1021,6 +1021,42 @@ public void testGetJobUsingTransferStrategy() throws IOException, InterruptedExc } } + @Test + public void testReadJobUsingTransferStrategy() throws IOException, InterruptedException { + final String tempPathPrefix = null; + final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final String fileName = "beowulf.txt"; + + try { + final List objects = Lists.newArrayList(new Ds3Object(fileName)); + + final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(BUCKET_NAME, objects); + + final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client.getBulkJobSpectraS3(getBulkJobSpectraS3Request); + + final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList(); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(masterObjectList) + .withChannelBuilder(new FileObjectGetter(tempDirectory)) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), + PartialObjectHelpers.getPartialObjectsRanges(objects))); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy(); + + Ds3ClientHelpers.wrap(client).startReadJob(transferStrategy).transfer(); + + final Collection filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false); + + for (final File file : filesInTempDirectory) { + assertEquals(fileName, file.getName()); + } + } finally { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } + @Test public void testGetJobWithUserSuppliedBlobStrategy() throws IOException, InterruptedException { final String tempPathPrefix = null; diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index 90e4ee11c..ec279b579 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -1320,6 +1320,51 @@ public void testPutJobUsingTransferStrategy() throws IOException, InterruptedExc } } + @Test + public void testWriteJobUsingTransferStrategy() throws IOException, InterruptedException, URISyntaxException { + final String DIR_NAME = "books/"; + final String[] FILE_NAMES = new String[]{"beowulf.txt"}; + + try { + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final List bookTitles = new ArrayList<>(); + final List objectsToWrite = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objectsToWrite.add(obj); + } + + final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(objectsToWrite)); + final PutBulkJobSpectraS3Response putBulkJobSpectraS3Response = client.putBulkJobSpectraS3(request); + + final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder() + .withDs3Client(client) + .withMasterObjectList(putBulkJobSpectraS3Response.getMasterObjectList()) + .withChannelBuilder(new FileObjectPutter(dirPath)); + + final TransferStrategy transferStrategy = transferStrategyBuilder.makePutTransferStrategy(); + + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client); + + final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(transferStrategy); + + writeJob.transfer(); + + final Iterable bucketContentsIterable = ds3ClientHelpers.listObjects(BUCKET_NAME); + + for (final Contents bucketContents : bucketContentsIterable) { + assertEquals(FILE_NAMES[0], bucketContents.getKey()); + } + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + @Test public void testPutJobWithUserSuppliedBlobStrategy() throws IOException, InterruptedException, URISyntaxException { final String DIR_NAME = "books/"; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java index 825bd64c2..f64caee00 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java @@ -20,6 +20,7 @@ import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.helpers.options.ReadJobOptions; import com.spectralogic.ds3client.helpers.options.WriteJobOptions; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy; import com.spectralogic.ds3client.models.Contents; import com.spectralogic.ds3client.models.bulk.Ds3Object; import com.spectralogic.ds3client.utils.Predicate; @@ -118,6 +119,8 @@ public interface Job { */ void transfer(final ObjectChannelBuilder channelBuilder) throws IOException; + + void transfer() throws IOException; } /** @@ -171,6 +174,17 @@ public abstract Ds3ClientHelpers.Job startWriteJob(final String bucket, final It public abstract Ds3ClientHelpers.Job startWriteJob(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) throws IOException; + /** + * Performs a bulk put job creation request and returns a {@link WriteJobImpl}. The reason for using this + * method call is to gain fine-grained control over put job behavior using a transfer strategy. The most + * convenient way to create a transfer strategy is by using the {@link com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder} + * @param transferStrategy A strategy that provides fine-grained control over transfer behavior. + * @return An instance of {@link WriteJobImpl} + * @throws IOException + */ + public abstract Ds3ClientHelpers.Job startWriteJob(final TransferStrategy transferStrategy) + throws IOException; + /** * Performs a bulk put job creation request and returns an {@link WriteJobImpl} configured to perform a "streamed" * transfer. "Streamed" means that channels and blobs are read or written sequentially. @@ -219,6 +233,18 @@ public abstract Ds3ClientHelpers.Job startReadJob(final String bucket, final Ite public abstract Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException; + /** + * Performs a bulk get job creation request and returns an {@link ReadJobImpl}. + * The reason for using this + * method call is to gain fine-grained control over get job behavior using a transfer strategy. The most + * convenient way to create a transfer strategy is by using the {@link com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder} + * @param transferStrategy A strategy that provides fine-grained control over transfer behavior. + * @return An instance of {@link ReadJobImpl}. + * @throws IOException + */ + public abstract Ds3ClientHelpers.Job startReadJob(final TransferStrategy transferStrategy) + throws IOException; + /** * Performs a bulk get job creation request and returns an {@link ReadJobImpl} configured to perform a "streamed" * transfer. "Streamed" means that channels and blobs are read or written sequentially. diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java index e3270ef1e..8e91be789 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java @@ -31,6 +31,7 @@ import com.spectralogic.ds3client.helpers.options.WriteJobOptions; import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl; +import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy; import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers; import com.spectralogic.ds3client.models.*; @@ -161,6 +162,13 @@ private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket, return new WriteJobImpl(transferStrategyBuilder); } + @Override + public Job startWriteJob(final TransferStrategy transferStrategy) + throws IOException + { + return new WriteJobImpl(transferStrategy); + } + @Override public Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite) throws IOException { return startWriteJobUsingStreamedBehavior(bucket, objectsToWrite, WriteJobOptions.create()); @@ -242,6 +250,11 @@ private Ds3ClientHelpers.Job innerStartReadJob(final List objects, return new ReadJobImpl(transferStrategyBuilder); } + @Override + public Job startReadJob(final TransferStrategy transferStrategy) throws IOException { + return new ReadJobImpl(transferStrategy); + } + @Override public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead) throws IOException { return startReadJobUsingStreamedBehavior(bucket, objectsToRead, ReadJobOptions.create()); diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java index 9ba6c57aa..eb94742db 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java @@ -32,6 +32,10 @@ abstract class JobImpl implements Job { private final TransferStrategyBuilder transferStrategyBuilder; + public JobImpl() { + this(null); + } + public JobImpl(final TransferStrategyBuilder transferStrategyBuilder) { this.transferStrategyBuilder = transferStrategyBuilder; } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java index adb915cc4..394e43369 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java @@ -15,6 +15,7 @@ package com.spectralogic.ds3client.helpers; +import com.google.common.base.Preconditions; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; import com.spectralogic.ds3client.helpers.strategy.transferstrategy.MetaDataReceivedObserver; import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy; @@ -23,6 +24,11 @@ import java.io.IOException; class ReadJobImpl extends JobImpl { + private TransferStrategy transferStrategy; + + public ReadJobImpl(final TransferStrategy transferStrategy) { + this.transferStrategy = transferStrategy; + } public ReadJobImpl(final TransferStrategyBuilder transferStrategyBuilder) { super(transferStrategyBuilder); @@ -52,12 +58,22 @@ public Ds3ClientHelpers.Job withChecksum(final ChecksumFunction checksumFunction @Override public void transfer(final ObjectChannelBuilder channelBuilder) throws IOException { + if (transferStrategy != null) { + transfer(); + return; + } + super.transfer(channelBuilder); - transfer(transferStrategyBuilder().makeGetTransferStrategy()); + transferStrategy = transferStrategyBuilder().makeGetTransferStrategy(); + + transfer(); } - public void transfer(final TransferStrategy transferStrategy) throws IOException { + @Override + public void transfer() throws IOException { + Preconditions.checkNotNull(transferStrategy, "transferStrategy may not be null."); + running = true; transferStrategy.transfer(); } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java index 897d60d37..e1b394162 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java @@ -15,6 +15,7 @@ package com.spectralogic.ds3client.helpers; +import com.google.common.base.Preconditions; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy; import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder; @@ -24,6 +25,11 @@ public class WriteJobImpl extends JobImpl { private MetadataAccess metadataAccess = null; private ChecksumFunction checksumFunction = null; + private TransferStrategy transferStrategy; + + public WriteJobImpl(final TransferStrategy transferStrategy) { + this.transferStrategy = transferStrategy; + } public WriteJobImpl(final TransferStrategyBuilder transferStrategyBuilder) { super(transferStrategyBuilder); @@ -54,15 +60,24 @@ public Ds3ClientHelpers.Job withChecksum(final ChecksumFunction checksumFunction @Override public void transfer(final ObjectChannelBuilder channelBuilder) throws IOException { + if (transferStrategy != null) { + transfer(); + return; + } + super.transfer(channelBuilder); transferStrategyBuilder().withChecksumFunction(checksumFunction); transferStrategyBuilder().withMetadataAccess(metadataAccess); - transfer(transferStrategyBuilder().makePutTransferStrategy()); + transferStrategy = transferStrategyBuilder().makePutTransferStrategy(); + transfer(); } - public void transfer(final TransferStrategy transferStrategy) throws IOException { + @Override + public void transfer() throws IOException { + Preconditions.checkNotNull(transferStrategy, "transferStrategy may not be null."); + running = true; transferStrategy.transfer(); } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java index d81883b4c..89f6776ff 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java @@ -32,7 +32,7 @@ private PartialObjectHelpers() { //pass } - public static ImmutableMultimap getPartialObjectsRanges(final List objects) { + public static ImmutableMultimap getPartialObjectsRanges(final Iterable objects) { final ImmutableMultimap.Builder builder = ImmutableMultimap.builder(); for (final Ds3Object obj : objects) { diff --git a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/Guard.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/Guard.java index 811f0be6e..81d60e669 100644 --- a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/Guard.java +++ b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/Guard.java @@ -26,10 +26,22 @@ private Guard() { //pass } + public static boolean isNotNullAndNotEmpty(final Collection collection) { + return !isNullOrEmpty(collection); + } + public static boolean isNullOrEmpty(final Collection collection) { return collection == null || collection.isEmpty(); } + public static boolean isNotNullAndNotEmpty(final Iterable iterable) { + return !isNullOrEmpty(iterable); + } + + public static boolean isNullOrEmpty(final Iterable iterable) { + return iterable == null || ! iterable.iterator().hasNext(); + } + public static boolean isStringNullOrEmpty(final String str) { return str == null || str.isEmpty(); } @@ -40,10 +52,6 @@ public static void throwOnNullOrEmptyString(final String str, final String messa } } - public static boolean isNotNullAndNotEmpty(final Collection collection) { - return !isNullOrEmpty(collection); - } - public static boolean isMultiMapNullOrEmpty(final Multimap multimap) { return multimap == null || multimap.isEmpty(); } From 5fedb9dbc70599bbb6f7bb98f6de1443f68e52cf Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 23 May 2017 11:01:13 -0600 Subject: [PATCH 013/229] Fixing typos and removing unused variables. --- .../strategy/blobstrategy/PutSequentialBlobStrategy.java | 2 -- .../strategy/transferstrategy/TransferStrategyBuilder.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java index c55982b22..9828152b7 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java @@ -45,7 +45,6 @@ public class PutSequentialBlobStrategy extends AbstractBlobStrategy { private final static Logger LOG = LoggerFactory.getLogger(PutSequentialBlobStrategy.class); - private final ImmutableMap uuidJobNodeImmutableMap; private final Iterator filteredChunkIterator; public PutSequentialBlobStrategy(final Ds3Client client, @@ -56,7 +55,6 @@ public PutSequentialBlobStrategy(final Ds3Client client, { super(client, masterObjectList, eventDispatcher, retryBehavior, chunkAttemptRetryDelayBehavior); this.filteredChunkIterator = filterChunks(masterObjectList.getObjects()).iterator(); - this.uuidJobNodeImmutableMap = buildNodeMap(masterObjectList.getNodes()); } @Override diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java index 866b24177..8cbd643b9 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java @@ -94,7 +94,7 @@ * interface instance, calling {@link TransferStrategy#transfer()} on the TransferStrategy interface initiates * data movement. * - * If using a {@link TransferStrategy} directly, as oppsed to going through a {@link Ds3ClientHelpers.Job}, the minimum + * If using a {@link TransferStrategy} directly, as opposed to going through a {@link Ds3ClientHelpers.Job}, the minimum * needed as configuration items in a put transfer strategy builder are: *
    *
  • {@link TransferStrategyBuilder#withDs3Client(Ds3Client)}
    • From f9d048c496a0161f78f5d2a6ca729c9e4b536ac0 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 23 May 2017 11:15:22 -0600 Subject: [PATCH 014/229] Removing unused imports --- .../strategy/blobstrategy/PutSequentialBlobStrategy.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java index 9828152b7..ce2779eae 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java @@ -18,14 +18,12 @@ import com.google.common.base.Function; import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableMap; import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.commands.spectrads3.AllocateJobChunkSpectraS3Request; import com.spectralogic.ds3client.commands.spectrads3.AllocateJobChunkSpectraS3Response; import com.spectralogic.ds3client.helpers.JobPart; import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher; import com.spectralogic.ds3client.models.BulkObject; -import com.spectralogic.ds3client.models.JobNode; import com.spectralogic.ds3client.models.MasterObjectList; import com.spectralogic.ds3client.models.Objects; import org.slf4j.Logger; @@ -34,9 +32,6 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.Iterator; -import java.util.UUID; - -import static com.spectralogic.ds3client.helpers.strategy.StrategyUtils.buildNodeMap; import static com.spectralogic.ds3client.helpers.strategy.StrategyUtils.filterChunks; /** From 017df13a7a26ec46fdd4629dc6fd4e9c363aeb95 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 23 May 2017 13:00:35 -0600 Subject: [PATCH 015/229] Removing redundant precondition check --- .../strategy/transferstrategy/TransferStrategyBuilder.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java index 8cbd643b9..77b8a34bf 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java @@ -526,14 +526,11 @@ private TransferStrategy makeTransferStrategy(final BlobStrategyMaker blobStrate Preconditions.checkNotNull(ds3Client, "ds3Client may not be null."); Preconditions.checkNotNull(eventDispatcher, "eventDispatcher may not be null."); Preconditions.checkNotNull(masterObjectList, "masterObjectList may not be null."); - Preconditions.checkState( ! Guard.isStringNullOrEmpty(jobId), "jobId may not be null or an emptystring"); Preconditions.checkNotNull(blobStrategyMaker, "blobStrategyMaker may not be null."); Preconditions.checkNotNull(transferMethodMaker, "transferMethodMaker may not be null."); Preconditions.checkNotNull(channelStrategy, "channelStrategy may not be null"); Preconditions.checkNotNull(transferRetryDecorator, "transferRetryDecorator may not be null"); - Guard.throwOnNullOrEmptyString(bucketName, "bucketName may not be null or empty."); - Guard.throwOnNullOrEmptyString(jobId, "jobId may not be null or empty."); maybeMakeBlobStrategy(blobStrategyMaker); From f8eddc9ba042ae4fd740c12df19f2b9ff58396ab Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 23 May 2017 13:09:20 -0600 Subject: [PATCH 016/229] Javadoc for transfer method intended for use when users supply their own transfer strategy. --- .../com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java index f64caee00..00087d2a0 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java @@ -120,6 +120,11 @@ public interface Job { void transfer(final ObjectChannelBuilder channelBuilder) throws IOException; + /** + * Transfers the files in this job intended for use when you supply your own {@link TransferStrategy}. + * See {@link Ds3ClientHelpers#startReadJob(TransferStrategy)} and {@link Ds3ClientHelpers#startWriteJob(TransferStrategy)}. + * @throws IOException + */ void transfer() throws IOException; } From d947f5bed9dbcfbfe27c2d5ca47584e2d5184cc6 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 23 May 2017 14:41:13 -0600 Subject: [PATCH 017/229] Beginning to use a somewhat pipelined way to transfer stuff to and from a black pearl. --- ds3-sdk/build.gradle | 1 - .../ds3client/helpers/JobPart.java | 7 ++ .../ds3client/helpers/JobState.java | 39 ++------ .../GetSequentialBlobStrategy.java | 94 +++++-------------- .../PutSequentialBlobStrategy.java | 47 +++++----- .../AbstractTransferStrategy.java | 72 ++++++-------- .../TransferStrategyBuilder.java | 4 +- 7 files changed, 92 insertions(+), 172 deletions(-) diff --git a/ds3-sdk/build.gradle b/ds3-sdk/build.gradle index 6688a07f5..87f7c6925 100644 --- a/ds3-sdk/build.gradle +++ b/ds3-sdk/build.gradle @@ -84,5 +84,4 @@ dependencies { compile 'com.google.guava:guava:20.0' compile 'com.google.code.findbugs:annotations:3.0.1' testCompile 'org.hamcrest:hamcrest-library:1.3' - testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0' } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java index 12ad17d0f..3055e9f21 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java @@ -38,4 +38,11 @@ public BulkObject getBulkObject() { return bulkObject; } + @Override + public String toString() { + return "JobPart{" + + "client=" + client + + ", bulkObject=" + bulkObject + + '}'; + } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java index 3a802d954..99d641ec4 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java @@ -19,44 +19,23 @@ import com.spectralogic.ds3client.models.Objects; import java.util.Collection; -import java.util.HashSet; -import java.util.concurrent.atomic.AtomicInteger; public class JobState { - private final AtomicInteger objectsRemaining; - private final JobPartTracker partTracker; + final int numBlobsToTransfer; - public JobState(final Collection filteredChunks,final JobPartTracker partTracker) { - this.objectsRemaining = new AtomicInteger(getObjectCount(filteredChunks)); - this.partTracker = partTracker.attachObjectCompletedListener(new ObjectCompletedListenerImpl()); - } - - public boolean hasObjects() { - return this.objectsRemaining.get() > 0; - } + public JobState(final Collection filteredChunks) { + int numBlobsInChunks = 0; - private static int getObjectCount(final Collection chunks) { - final HashSet result = new HashSet<>(); - for (final Objects chunk : chunks) { - for (final BulkObject bulkObject : chunk.getObjects()) { - result.add(bulkObject.getName()); + for (final Objects chunk : filteredChunks) { + for (final BulkObject ignored : chunk.getObjects()) { + ++numBlobsInChunks; } } - return result.size(); - } - private final class ObjectCompletedListenerImpl implements ObjectCompletedListener { - @Override - public void objectCompleted(final String name) { - JobState.this.objectsRemaining.decrementAndGet(); - } + numBlobsToTransfer = numBlobsInChunks; } - @Override - public String toString() { - return "JobState{" + - "objectsRemaining=" + objectsRemaining + - ", partTracker=" + partTracker + - '}'; + public int numBlobsToTransfer() { + return numBlobsToTransfer; } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java index 63742c4f9..3e6759412 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java @@ -16,10 +16,7 @@ package com.spectralogic.ds3client.helpers.strategy.blobstrategy; import com.google.common.base.Function; -import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.commands.spectrads3.GetJobChunksReadyForClientProcessingSpectraS3Request; import com.spectralogic.ds3client.commands.spectrads3.GetJobChunksReadyForClientProcessingSpectraS3Response; @@ -28,22 +25,17 @@ import com.spectralogic.ds3client.models.BulkObject; import com.spectralogic.ds3client.models.MasterObjectList; import com.spectralogic.ds3client.models.Objects; +import org.slf4j.LoggerFactory; +import org.slf4j.Logger; import javax.annotation.Nullable; import java.io.IOException; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; /** * A subclass of {@link BlobStrategy} used in get transfers. */ public class GetSequentialBlobStrategy extends AbstractBlobStrategy { - private final Set processedChunks; - - private final Object lock = new Object(); - private final Set activeBlobs = new HashSet<>(); - private ImmutableList outstandingJobParts; + private static final Logger LOG = LoggerFactory.getLogger(GetSequentialBlobStrategy.class); public GetSequentialBlobStrategy(final Ds3Client client, final MasterObjectList masterObjectList, @@ -52,69 +44,36 @@ public GetSequentialBlobStrategy(final Ds3Client client, final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior) { super(client, masterObjectList, eventDispatcher, retryBehavior, chunkAttemptRetryDelayBehavior); - this.processedChunks = new HashSet<>(); } @Override - public Iterable getWork() throws IOException, InterruptedException { - - final MasterObjectList available = getAvailable(); + public synchronized Iterable getWork() throws IOException, InterruptedException { + LOG.info("---> Getting available blobs."); - // filter any chunks that have been processed - final FluentIterable chunks = FluentIterable.from(available.getObjects()); + // get chunks that have blobs ready for transfer from black pearl + final MasterObjectList masterObjectListWithAvailableChunks = masterObjectListWithAvailableChunks(); - final ImmutableList.Builder filteredPartsBuilder = ImmutableList.builder(); + final FluentIterable chunks = FluentIterable.from(masterObjectListWithAvailableChunks.getObjects()); - final FluentIterable jobParts = chunks.filter(new Predicate() { - @Override - public boolean apply(@Nullable final Objects input) { - return !processedChunks.contains(input.getChunkId()); - } - }).transformAndConcat(new Function>() { + return chunks.transformAndConcat(new Function>() { @Nullable @Override - public Iterable apply(@Nullable final Objects objects) { - return FluentIterable.from(objects.getObjects()).transform(new Function() { - @Nullable - @Override - public JobPart apply(@Nullable final BulkObject blob) { - return new JobPart(client(), blob); - - // TODO: When we get to the point where BP enables clustering, we'll want to be able to get the - // client connection info correct for the server on which a chunk resides. StrategyUtils.getClient - // appears to work to support the clustering scenario, but we don't need it right now, and holding - // connection info in a collection potentially exposes lifetime management issues that we haven't - // fully explored. - // return new JobPart(StrategyUtils.getClient(jobNodes, objects.getNodeId(), getClient()), blob); - } - }); + public Iterable apply(@Nullable final Objects chunk) { + return FluentIterable.from(chunk.getObjects()) + .transform(new Function() { + @Nullable + @Override + public JobPart apply(@Nullable final BulkObject blob) { + return new JobPart(client(), blob); + } + }); } }); - - final FluentIterable nextWorkParts = FluentIterable.from( - getNextIterable(jobParts) - ).filter(new Predicate() { - @Override - public boolean apply(@Nullable final JobPart input) { - final String blobName = input.getBulkObject().getName(); - synchronized (lock) { - if (activeBlobs.contains(blobName)) { - filteredPartsBuilder.add(input); - return false; - } else { - activeBlobs.add(blobName); - } - } - return true; - } - }); - - outstandingJobParts = filteredPartsBuilder.build(); - - return nextWorkParts; } - private MasterObjectList getAvailable() throws IOException, InterruptedException { + private MasterObjectList masterObjectListWithAvailableChunks() throws IOException, InterruptedException { + LOG.info("---> Getting blobs from black pearl."); + do { final GetJobChunksReadyForClientProcessingSpectraS3Response availableJobChunks = client().getJobChunksReadyForClientProcessingSpectraS3(new GetJobChunksReadyForClientProcessingSpectraS3Request(masterObjectList().getJobId().toString())); @@ -137,21 +96,12 @@ private MasterObjectList getAvailable() throws IOException, InterruptedException } while(true); } - private Iterable getNextIterable(final FluentIterable jobParts) { - if (outstandingJobParts == null) { - return jobParts; - } - return Iterables.concat(outstandingJobParts, jobParts); - } - /** * Emit an event when a blob is transferred. * @param bulkObject The transferred blob. */ @Override public void blobCompleted(final BulkObject bulkObject) { - synchronized (lock) { - activeBlobs.remove(bulkObject.getName()); - } + // Intentionally not implemented } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java index ce2779eae..35478a302 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java @@ -32,6 +32,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.Iterator; + import static com.spectralogic.ds3client.helpers.strategy.StrategyUtils.filterChunks; /** @@ -40,7 +41,7 @@ public class PutSequentialBlobStrategy extends AbstractBlobStrategy { private final static Logger LOG = LoggerFactory.getLogger(PutSequentialBlobStrategy.class); - private final Iterator filteredChunkIterator; + private final Iterator chunksThatContainBlobs; public PutSequentialBlobStrategy(final Ds3Client client, final MasterObjectList masterObjectList, @@ -49,33 +50,35 @@ public PutSequentialBlobStrategy(final Ds3Client client, final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior) { super(client, masterObjectList, eventDispatcher, retryBehavior, chunkAttemptRetryDelayBehavior); - this.filteredChunkIterator = filterChunks(masterObjectList.getObjects()).iterator(); + this.chunksThatContainBlobs = filterChunks(masterObjectList.getObjects()).iterator(); } @Override public Iterable getWork() throws IOException, InterruptedException { - final Objects nextChunk = allocateChunk(filteredChunkIterator.next()); + final Objects nextChunk = allocateChunk(chunksThatContainBlobs.next()); LOG.debug("Allocating chunk: {}", nextChunk.getChunkId().toString()); - return FluentIterable.from(nextChunk.getObjects()).filter(new Predicate() { - @Override - public boolean apply(@Nullable final BulkObject input) { - return !input.getInCache(); - } - }).transform(new Function() { - @Nullable - @Override - public JobPart apply(@Nullable final BulkObject blob) { - return new JobPart(client(), blob); - - // TODO: When we get to the point where BP enables clustering, we'll want to be able to get the - // client connection info correct for the server on which a chunk resides. StrategyUtils.getClient - // appears to work to support the clustering scenario, but we don't need it right now, and holding - // connection info in a collection potentially exposes lifetime management issues that we haven't - // fully explored. - // return new JobPart(StrategyUtils.getClient(uuidJobNodeImmutableMap,nextChunk.getNodeId(), getClient()), input); - } - }); + return FluentIterable.from(nextChunk.getObjects()) + .filter(new Predicate() { + @Override + public boolean apply(@Nullable final BulkObject input) { + return !input.getInCache(); + } + }) + .transform(new Function() { + @Nullable + @Override + public JobPart apply(@Nullable final BulkObject blob) { + return new JobPart(client(), blob); + + // TODO: When we get to the point where BP enables clustering, we'll want to be able to get the + // client connection info correct for the server on which a chunk resides. StrategyUtils.getClient + // appears to work to support the clustering scenario, but we don't need it right now, and holding + // connection info in a collection potentially exposes lifetime management issues that we haven't + // fully explored. + // return new JobPart(StrategyUtils.getClient(uuidJobNodeImmutableMap,nextChunk.getNodeId(), getClient()), input); + } + }); } private Objects allocateChunk(final Objects filtered) throws IOException { diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java index 73df3ef8d..013f6ecdd 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java @@ -15,9 +15,7 @@ package com.spectralogic.ds3client.helpers.strategy.transferstrategy; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListeningExecutorService; import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request; @@ -31,7 +29,8 @@ import java.io.IOException; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,11 +43,11 @@ abstract class AbstractTransferStrategy implements TransferStrategy { private static final Logger LOG = LoggerFactory.getLogger(AbstractTransferStrategy.class); private final BlobStrategy blobStrategy; - private final JobState jobState; private final ListeningExecutorService executorService; private final EventDispatcher eventDispatcher; private final MasterObjectList masterObjectList; private final FailureEvent.FailureActivity failureActivity; + private final AtomicInteger totalNumBlobsToTransfer; private TransferMethod transferMethod; @@ -74,7 +73,7 @@ public AbstractTransferStrategy(final BlobStrategy blobStrategy, final FailureEvent.FailureActivity failureActivity) { this.blobStrategy = blobStrategy; - this.jobState = jobState; + totalNumBlobsToTransfer = new AtomicInteger(jobState.numBlobsToTransfer()); this.executorService = executorService; this.eventDispatcher = eventDispatcher; this.masterObjectList = masterObjectList; @@ -98,30 +97,36 @@ public AbstractTransferStrategy withTransferMethod(final TransferMethod transfer */ @Override public void transfer() throws IOException { - while (jobState.hasObjects()) { - transferJobParts(); - } + do { + try { + final Iterable jobParts = blobStrategy.getWork(); + final CountDownLatch countDownLatch = new CountDownLatch(Iterables.size(jobParts)); + transferJobParts(jobParts, countDownLatch); + countDownLatch.await(); + } catch (final InterruptedException e) { + LOG.info("Error getting entries from work queue.", e); + } + } while (totalNumBlobsToTransfer.get() > 0); } - private void transferJobParts() throws IOException { + private void transferJobParts(final Iterable jobParts, final CountDownLatch countDownLatch) throws IOException { try { try { - final ImmutableList.Builder> transferTasksListBuilder = ImmutableList.builder(); - - final Iterable workQueue = blobStrategy.getWork(); - - for (final JobPart jobPart : workQueue) { - transferTasksListBuilder.add(executorService.submit(new Callable() { + for (final JobPart jobPart : jobParts) { + executorService.submit(new Callable() { @Override public Void call() throws Exception { - transferMethod.transferJobPart(jobPart); - return null; + try { + transferMethod.transferJobPart(jobPart); + } finally { + totalNumBlobsToTransfer.decrementAndGet(); + countDownLatch.countDown(); + return null; + } } - })); + }); } - - runTransferTasks(transferTasksListBuilder.build()); - } catch (final IOException | RuntimeException e) { + } catch (final RuntimeException e) { throw e; } catch (final Exception e) { throw new RuntimeException(e); @@ -132,29 +137,6 @@ public Void call() throws Exception { } } - private void runTransferTasks(final Iterable> transferTasks) throws IOException { - try { - Futures.allAsList(transferTasks).get(); - } catch (final InterruptedException e) { - throw new RuntimeException(e); - } catch (final ExecutionException e) { - // The future throws a wrapper exception, but we want don't want to expose that this was implemented with futures. - final Throwable cause = e.getCause(); - - // Throw each of the advertised thrown exceptions. - if (cause instanceof IOException) { - throw (IOException)cause; - } - - // The rest we don't know about, so we'll just forward them. - if (cause instanceof RuntimeException) { - throw (RuntimeException)cause; - } else { - throw new RuntimeException(cause); - } - } - } - private void emitFailureEvent(final FailureEvent failureEvent) { eventDispatcher.emitFailureEvent(failureEvent); } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java index 77b8a34bf..3d871bfcb 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java @@ -676,7 +676,7 @@ private JobState getOrMakeJobStateForPutJob() { getOrMakeJobPartTrackerForPutJob(chunksNotYetCompleted); - jobState = new JobState(chunksNotYetCompleted, jobPartTracker); + jobState = new JobState(chunksNotYetCompleted); return jobState; } @@ -872,7 +872,7 @@ private JobState getOrMakeJobStateForGetJob() { getOrMakeJobPartTrackerForGetJob(chunks); - jobState = new JobState(chunks, jobPartTracker); + jobState = new JobState(chunks); return jobState; } From 87f98d7c3957e8fce087f9aa82c4abe7444afcc5 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Thu, 25 May 2017 15:16:52 -0600 Subject: [PATCH 018/229] Beginning to use a somewhat pipelined way to transfer stuff to and from a black pearl. --- .../ds3client/helpers/Ds3ClientHelpers_Test.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java index 762188ec3..3be6d3758 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java @@ -208,8 +208,15 @@ public void testWriteObjectsWithFailedPut() throws IOException, ParseException { final PutBulkJobSpectraS3Response buildBulkPutResponse = buildBulkPutResponse(); Mockito.when(ds3Client.putBulkJobSpectraS3(Mockito.any(PutBulkJobSpectraS3Request.class))).thenReturn(buildBulkPutResponse); - final AllocateJobChunkSpectraS3Response allocateResponse = buildAllocateResponse2(); - Mockito.when(ds3Client.allocateJobChunkSpectraS3(hasChunkId(CHUNK_ID_1))).thenReturn(allocateResponse); + final AllocateJobChunkSpectraS3Response allocateResponse2 = buildAllocateResponse2(); + final AllocateJobChunkSpectraS3Response allocateResponse3 = buildAllocateResponse3(); + + Mockito.when(ds3Client.allocateJobChunkSpectraS3(hasChunkId(CHUNK_ID_1))).thenReturn(allocateResponse2); + + Mockito.when(ds3Client.allocateJobChunkSpectraS3(hasChunkId(CHUNK_ID_1))) + .thenReturn(allocateResponse2); + Mockito.when(ds3Client.allocateJobChunkSpectraS3(hasChunkId(CHUNK_ID_2))) + .thenReturn(allocateResponse3); final PutObjectResponse putResponse = mock(PutObjectResponse.class); Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "foo", jobId, 0, "foo co"))).thenThrow(new StubException()); From 031524be43f0a3b8599b34ea1239f551ec78ad19 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Thu, 25 May 2017 15:24:13 -0600 Subject: [PATCH 019/229] Beginning to use a somewhat pipelined way to transfer stuff to and from a black pearl. --- .../strategy/blobstrategy/GetSequentialBlobStrategy.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java index 3e6759412..e8c329b0a 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java @@ -48,8 +48,6 @@ public GetSequentialBlobStrategy(final Ds3Client client, @Override public synchronized Iterable getWork() throws IOException, InterruptedException { - LOG.info("---> Getting available blobs."); - // get chunks that have blobs ready for transfer from black pearl final MasterObjectList masterObjectListWithAvailableChunks = masterObjectListWithAvailableChunks(); @@ -72,8 +70,6 @@ public JobPart apply(@Nullable final BulkObject blob) { } private MasterObjectList masterObjectListWithAvailableChunks() throws IOException, InterruptedException { - LOG.info("---> Getting blobs from black pearl."); - do { final GetJobChunksReadyForClientProcessingSpectraS3Response availableJobChunks = client().getJobChunksReadyForClientProcessingSpectraS3(new GetJobChunksReadyForClientProcessingSpectraS3Request(masterObjectList().getJobId().toString())); From a431ab1860d76f8068ab839e1d81696e779115c3 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Fri, 26 May 2017 14:50:20 -0600 Subject: [PATCH 020/229] Beginning to use a somewhat pipelined way to transfer stuff to and from a black pearl. --- .../AbstractTransferStrategy.java | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java index 013f6ecdd..b553a9807 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java @@ -47,7 +47,7 @@ abstract class AbstractTransferStrategy implements TransferStrategy { private final EventDispatcher eventDispatcher; private final MasterObjectList masterObjectList; private final FailureEvent.FailureActivity failureActivity; - private final AtomicInteger totalNumBlobsToTransfer; + private AtomicInteger totalNumBlobsToTransfer; private TransferMethod transferMethod; @@ -105,35 +105,37 @@ public void transfer() throws IOException { countDownLatch.await(); } catch (final InterruptedException e) { LOG.info("Error getting entries from work queue.", e); + Thread.currentThread().interrupt(); + } catch (final Throwable t) { + emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk())); + totalNumBlobsToTransfer.decrementAndGet(); } } while (totalNumBlobsToTransfer.get() > 0); } private void transferJobParts(final Iterable jobParts, final CountDownLatch countDownLatch) throws IOException { - try { - try { - for (final JobPart jobPart : jobParts) { - executorService.submit(new Callable() { - @Override - public Void call() throws Exception { - try { - transferMethod.transferJobPart(jobPart); - } finally { - totalNumBlobsToTransfer.decrementAndGet(); - countDownLatch.countDown(); - return null; - } + for (final JobPart jobPart : jobParts) { + executorService.submit(new Callable() { + @Override + public Void call() throws Exception { + try { + try { + transferMethod.transferJobPart(jobPart); + } catch (final RuntimeException e) { + throw e; + } catch (final Exception e) { + throw new RuntimeException(e); + } finally { + totalNumBlobsToTransfer.decrementAndGet(); + countDownLatch.countDown(); + return null; } - }); + } catch (final Throwable t) { + emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk())); + throw t; + } } - } catch (final RuntimeException e) { - throw e; - } catch (final Exception e) { - throw new RuntimeException(e); - } - } catch (final Throwable t) { - emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk())); - throw t; + }); } } From d5df671d49ae440290e4d1e07274999c89c505b5 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Fri, 26 May 2017 16:18:57 -0600 Subject: [PATCH 021/229] Paying attention to thread interruption state. --- .../strategy/transferstrategy/AbstractTransferStrategy.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java index b553a9807..f44aa05ee 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java @@ -97,7 +97,7 @@ public AbstractTransferStrategy withTransferMethod(final TransferMethod transfer */ @Override public void transfer() throws IOException { - do { + while (totalNumBlobsToTransfer.get() > 0 && ! Thread.currentThread().isInterrupted()) { try { final Iterable jobParts = blobStrategy.getWork(); final CountDownLatch countDownLatch = new CountDownLatch(Iterables.size(jobParts)); @@ -110,7 +110,7 @@ public void transfer() throws IOException { emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk())); totalNumBlobsToTransfer.decrementAndGet(); } - } while (totalNumBlobsToTransfer.get() > 0); + } } private void transferJobParts(final Iterable jobParts, final CountDownLatch countDownLatch) throws IOException { From 97d8a0c5d1b5c59c6f44424cf298fea381edff8f Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Fri, 26 May 2017 16:40:59 -0600 Subject: [PATCH 022/229] Missing final --- .../strategy/transferstrategy/AbstractTransferStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java index f44aa05ee..250bb5968 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java @@ -47,7 +47,7 @@ abstract class AbstractTransferStrategy implements TransferStrategy { private final EventDispatcher eventDispatcher; private final MasterObjectList masterObjectList; private final FailureEvent.FailureActivity failureActivity; - private AtomicInteger totalNumBlobsToTransfer; + private final AtomicInteger totalNumBlobsToTransfer; private TransferMethod transferMethod; From 90cc0c4a9ca7458d5ce1768a1d670017e3e1f6ed Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 30 May 2017 11:37:11 -0600 Subject: [PATCH 023/229] Propagating exception caught during transfer --- .../integration/GetJobManagement_Test.java | 11 +++- .../AbstractTransferStrategy.java | 51 +++++++++++++------ .../helpers/Ds3ClientHelpers_Test.java | 2 +- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index 75b59e6d5..e7a0f6da7 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -431,11 +431,13 @@ private void enableWritePermissionForRoot(final String fileOrDirName) { } } - @Test(expected = AccessControlException.class) + @Test public void testReadRetrybugWhenChannelThrowsAccessException() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { final String tempPathPrefix = null; final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final AtomicBoolean caughtException = new AtomicBoolean(false); + try { final String DIR_NAME = "largeFiles/"; final String FILE_NAME = "lesmis-copies.txt"; @@ -444,7 +446,7 @@ public void testReadRetrybugWhenChannelThrowsAccessException() throws IOExceptio final long bookSize = Files.size(objPath); final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize); - final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl)client); + final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client); final int maxNumBlockAllocationRetries = 1; final int maxNumObjectTransferAttempts = 3; @@ -465,9 +467,14 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { throw new AccessControlException(key); } }); + } catch (final IOException e) { + caughtException.set(true); + assertTrue(e.getCause() instanceof AccessControlException); } finally { FileUtils.deleteDirectory(tempDirectoryPath.toFile()); } + + assertTrue(caughtException.get()); } @Test diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java index 250bb5968..217b34acd 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java @@ -31,6 +31,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +50,8 @@ abstract class AbstractTransferStrategy implements TransferStrategy { private final FailureEvent.FailureActivity failureActivity; private final AtomicInteger totalNumBlobsToTransfer; + private final AtomicReference cachedException = new AtomicReference<>(); + private TransferMethod transferMethod; /** @@ -97,6 +100,8 @@ public AbstractTransferStrategy withTransferMethod(final TransferMethod transfer */ @Override public void transfer() throws IOException { + cachedException.set(null); + while (totalNumBlobsToTransfer.get() > 0 && ! Thread.currentThread().isInterrupted()) { try { final Iterable jobParts = blobStrategy.getWork(); @@ -107,8 +112,27 @@ public void transfer() throws IOException { LOG.info("Error getting entries from work queue.", e); Thread.currentThread().interrupt(); } catch (final Throwable t) { - emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk())); totalNumBlobsToTransfer.decrementAndGet(); + emitFailureAndSetCachedException(t); + } + } + + if (cachedException.get() != null) { + throw cachedException.get(); + } + } + + private void emitFailureAndSetCachedException(final Throwable t) { + emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk())); + maybeSetCachedException(t); + } + + private synchronized void maybeSetCachedException(final Throwable t) { + if (cachedException.get() == null) { + if (t instanceof IOException) { + cachedException.set((IOException)t); + } else { + cachedException.set(new IOException(t)); } } } @@ -119,20 +143,17 @@ private void transferJobParts(final Iterable jobParts, final CountDownL @Override public Void call() throws Exception { try { - try { - transferMethod.transferJobPart(jobPart); - } catch (final RuntimeException e) { - throw e; - } catch (final Exception e) { - throw new RuntimeException(e); - } finally { - totalNumBlobsToTransfer.decrementAndGet(); - countDownLatch.countDown(); - return null; - } - } catch (final Throwable t) { - emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk())); - throw t; + transferMethod.transferJobPart(jobPart); + } catch (final RuntimeException e) { + emitFailureAndSetCachedException(e); + throw e; + } catch (final Exception e) { + emitFailureAndSetCachedException(e); + throw new RuntimeException(e); + } finally { + totalNumBlobsToTransfer.decrementAndGet(); + countDownLatch.countDown(); + return null; } } }); diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java index 3be6d3758..d5627a1f6 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java @@ -245,7 +245,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { } } - private static final class StubException extends RuntimeException { + private static final class StubException extends IOException { private static final long serialVersionUID = 5121719894916333278L; } From 6d4ee76c8c5cf897db79977d81cf4ed937c1b0bc Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 30 May 2017 12:17:47 -0600 Subject: [PATCH 024/229] Fixing typos and renaming variables to better describe what they represent. --- .../java/com/spectralogic/ds3client/helpers/JobState.java | 4 ++-- .../strategy/transferstrategy/TransferStrategyBuilder.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java index 99d641ec4..47479f4e2 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java @@ -23,10 +23,10 @@ public class JobState { final int numBlobsToTransfer; - public JobState(final Collection filteredChunks) { + public JobState(final Collection chunksThatContainBlobs) { int numBlobsInChunks = 0; - for (final Objects chunk : filteredChunks) { + for (final Objects chunk : chunksThatContainBlobs) { for (final BulkObject ignored : chunk.getObjects()) { ++numBlobsInChunks; } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java index 3d871bfcb..a6c2678dc 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java @@ -345,7 +345,7 @@ public MasterObjectList masterObjectList() { } /** - * When getting portions of objects from a Black Pearl, object ranges are the mechanism used to tell the Blavk Pearl + * When getting portions of objects from a Black Pearl, object ranges are the mechanism used to tell the Black Pearl * which portions of obejcts to get. * @param rangesForBlobs When retrieving whole objects, you supply an empty range, e.g.: * 
    
From 4399ca6a76bfc8b56ab9e97b19fefd56fd1a6f95 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Fri, 2 Jun 2017 11:26:23 -0600
Subject: [PATCH 025/229] Added integration test #1

---
 .../ds3client/integration/Iterators_Test.java |  9 ++---
 .../ds3client/helpers/ContentPrefix.java      | 25 +++++++++++++
 .../ds3client/helpers/Ds3ClientHelpers.java   | 25 +++++++++++++
 .../helpers/Ds3ClientHelpersImpl.java         | 35 +++++++++++++++++--
 .../helpers/pagination/GetBucketLoader.java   |  8 ++++-
 .../pagination/GetBucketLoaderFactory.java    |  6 ++--
 .../helpers/Ds3ClientHelpers_Test.java        | 14 ++++++++
 7 files changed, 113 insertions(+), 9 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ContentPrefix.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index b00eef72d..d3c4bdd56 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -47,6 +47,7 @@ public class Iterators_Test {
     private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(CLIENT);
     private static final String TEST_ENV_NAME = "lazy_iterator_test";
     private static final int RETRIES = 5;
+    private static final String TEST_DELIMITER = null;
 
     private static TempStorageIds envStorageIds;
     private static UUID envDataPolicyId;
@@ -73,7 +74,7 @@ public void emptyGetBucket() throws IOException {
         final String nextMarker = null;
         final int maxKeys = 100;
 
-        emptyTest(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, RETRIES));
+        emptyTest(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, TEST_DELIMITER, nextMarker, maxKeys, RETRIES));
     }
 
     @Test
@@ -81,7 +82,7 @@ public void singlePageGetBucket() throws IOException, URISyntaxException {
         final String prefix = "";
         final String nextMarker = null;
         final int maxKeys = 100;
-        paginate(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, RETRIES));
+        paginate(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, TEST_DELIMITER, nextMarker, maxKeys, RETRIES));
     }
 
 
@@ -90,13 +91,13 @@ public void multiPageGetBucket() throws IOException, URISyntaxException {
         final String prefix = "";
         final String nextMarker = null;
         final int maxKeys = 2;
-        paginate(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, RETRIES));
+        paginate(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, TEST_DELIMITER, maxKeys, RETRIES));
 
     }
 
     @Test
     public void failedRequestGetBucket() {
-        testFailedRequest(new GetBucketLoaderFactory(CLIENT, "Unknown_Bucket",null, null, 1000, 5));
+        testFailedRequest(new GetBucketLoaderFactory(CLIENT, "Unknown_Bucket",null, TEST_DELIMITER,null, 1000, 5));
     }
 
    @Test
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ContentPrefix.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ContentPrefix.java
new file mode 100644
index 000000000..6885a8723
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ContentPrefix.java
@@ -0,0 +1,25 @@
+package com.spectralogic.ds3client.helpers;
+
+import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.common.CommonPrefixes;
+
+import java.util.List;
+
+/**
+ * Because remoteListDirectory returns both a Contents and a CommonPrefixes
+ * We need an object that can shuttle them around.
+ */
+public class ContentPrefix {
+    private final Iterable contents;
+    private final List commonPrefixes;
+
+    ContentPrefix(final Iterable contents, final List commonPrefixes) {
+        this.commonPrefixes = commonPrefixes;
+        this.contents = contents;
+    }
+
+    public Iterable contents() { return contents; }
+    public List commonPrefixes() { return commonPrefixes; }
+
+
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index c029e1ce1..cf8dd56a0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -17,6 +17,7 @@
 
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
@@ -321,6 +322,30 @@ public abstract Iterable listObjects(final String bucket, final String
      */
     public abstract Iterable listObjectsForDirectory(final Path directory) throws IOException;
 
+    /**
+     * Defaults to the path delimiter of '/'
+     * Returns an Iterable of {@link Ds3Object} and a list of {@link com.spectralogic.ds3client.models.common.CommonPrefixes}
+     * @param bucket The bucket
+     * @param keyPrefix Limits the response to keys that begin with the specified prefix
+     * @param nextMarker Specifies the key to start with when listing objects
+     * @param maxKeys Sets the maximum number of keys returned in the response body.
+     * @return {@link ContentPrefix}
+     * @throws IOException
+     */
+    public abstract ContentPrefix remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException;
+
+    /**
+     * Returns an Iterable of {@link Ds3Object} and a list of {@link com.spectralogic.ds3client.models.common.CommonPrefixes}
+     * @param bucket The bucket
+     * @param keyPrefix Limits the response to keys that begin with the specified prefix
+     * @param delimiter Specifies a path delimiter for the S3 query
+     * @param nextMarker Specifies the key to start with when listing objects
+     * @param maxKeys Sets the maximum number of keys returned in the response body.
+     * @return {@link ContentPrefix}
+     * @throws IOException
+     */
+    public abstract ContentPrefix remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException;
+
     /**
      * Returns an Iterable of {@link Ds3Object} that have a prefix added.
      */
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 060b7fc2f..e4068b28d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -31,6 +31,7 @@
 import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers;
 import com.spectralogic.ds3client.models.*;
 import com.spectralogic.ds3client.models.bulk.RequestType;
+import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.models.common.Range;
 import com.spectralogic.ds3client.models.bulk.*;
 import com.spectralogic.ds3client.networking.FailedRequestException;
@@ -55,6 +56,7 @@ class Ds3ClientHelpersImpl extends Ds3ClientHelpers {
     public final static int DEFAULT_RETRY_DELAY = -1;
     public final static int DEFAULT_RETRY_AFTER = -1;
     public final static int DEFAULT_OBJECT_TRANSFER_ATTEMPTS = 5;
+    public final static String DEFAULT_DELIMITER = null;
 
     private final static Logger LOG = LoggerFactory.getLogger(Ds3ClientHelpersImpl.class);
     private final static int DEFAULT_LIST_OBJECTS_RETRIES = 5;
@@ -319,13 +321,13 @@ public Iterable listObjects(final String bucket, final String keyPrefi
     @Override
     public Iterable listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) {
 
-        return new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
+        return new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
     }
 
     @Override
     public Iterable listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys, final int retries) {
 
-        return new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, nextMarker, maxKeys, retries));
+        return new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries));
     }
 
     @Override
@@ -341,6 +343,35 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
         return objects.build();
     }
 
+    @Override
+    public ContentPrefix remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException {
+        return remoteListDirectory(bucket,keyPrefix,"/", nextMarker, maxKeys);
+    }
+
+    public ContentPrefix remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException {
+        final Iterable contents = new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
+        final GetBucketRequest request = new GetBucketRequest(bucket);
+        if (keyPrefix != null) {
+            request.withPrefix(keyPrefix);
+        }
+        if (delimiter != null) {
+            request.withDelimiter(delimiter);
+        }
+        request.withMaxKeys(1);
+        final GetBucketResponse response;
+        final List commonPrefixes;
+        try {
+            response = this.client.getBucket(request);
+            final ListBucketResult result = response.getListBucketResult();
+
+            commonPrefixes = result.getCommonPrefixes();
+
+        } catch (final FailedRequestException e) {
+            throw new RuntimeException("Failed to get the list of objects due to a failed request", e);
+        }
+       return new ContentPrefix(contents, commonPrefixes);
+    }
+
     public Iterable addPrefixToDs3ObjectsList(final Iterable objectsList, final String prefix) {
         final FluentIterable objectIterable = FluentIterable.from(objectsList);
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java
index 43b8cdf0d..6943d7c58 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java
@@ -35,6 +35,7 @@ public class GetBucketLoader implements LazyIterable.LazyLoader {
     private final Ds3Client client;
     private final String bucket;
     private final String prefix;
+    private final String delimiter;
     private final int maxKeys;
     private final int retryCount;
 
@@ -42,10 +43,11 @@ public class GetBucketLoader implements LazyIterable.LazyLoader {
     private boolean truncated;
     private boolean endOfInput = false;
 
-    public GetBucketLoader(final Ds3Client client, final String bucket, final String prefix, final String nextMarker, final int maxKeys, final int retryCount) {
+    public GetBucketLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount) {
         this.client = client;
         this.bucket = bucket;
         this.prefix = prefix;
+        this.delimiter = delimiter;
         this.maxKeys = maxKeys;
         this.retryCount = retryCount;
 
@@ -69,6 +71,10 @@ public List getNextValues() {
                 request.withMarker(nextMarker);
             }
 
+            if (delimiter != null) {
+                request.withDelimiter(delimiter);
+            }
+
             final GetBucketResponse response;
             try {
                 response = this.client.getBucket(request);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
index b7bf51cc5..7ac05747a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
@@ -25,14 +25,16 @@ public class GetBucketLoaderFactory implements LazyIterable.LazyLoaderFactory create() {
-        return new GetBucketLoader(client, bucket, keyPrefix, nextMarker, maxKeys, defaultListObjectsRetries);
+        return new GetBucketLoader(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, defaultListObjectsRetries);
     }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index 762188ec3..33c47e0dc 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -28,6 +28,7 @@
 import com.spectralogic.ds3client.models.Error;
 import com.spectralogic.ds3client.models.Objects;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.networking.ConnectionDetails;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
@@ -257,6 +258,19 @@ public void testListObjects() throws IOException, ParseException {
         checkContents(contentList.get(1), "bar", "f3f98ff00be128139332bcf4b772be43", "2009-10-14T17:50:31.000Z", 12);
         checkContents(contentList.get(2), "baz", "802d45fcb9a3f7d00f1481362edc0ec9", "2009-10-18T17:50:35.000Z", 12);
     }
+
+    @Test
+    public void testRemoteListDirectory() throws IOException {
+        final Ds3Client ds3Client = mock(Ds3Client.class);
+        Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, null))).thenReturn(new StubGetBucketResponse(0));
+        Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, "baz"))).thenReturn(new StubGetBucketResponse(1));
+        Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, "/foo/baz"))).thenReturn(new StubGetBucketResponse(1));
+        final ContentPrefix contentPrefix = Ds3ClientHelpers.wrap(ds3Client).remoteListDirectory(MYBUCKET, null, "/", null, 0);
+        final List contentsList = Lists.newArrayList(contentPrefix.contents());
+        final List commonPrefixes = contentPrefix.commonPrefixes();
+        assertThat(commonPrefixes.size(), is(0));
+        assertThat(contentsList.size(), is(3));
+    }
     
     private static void checkContents(
             final Contents contents,

From 3fb333e5520810181b897d5bcdb74f768e527e67 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Fri, 2 Jun 2017 14:16:32 -0600
Subject: [PATCH 026/229] Bailing on transfer if chunk allocation or creation
 fails

---
 .../strategy/transferstrategy/AbstractTransferStrategy.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index 217b34acd..d16130c00 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -20,6 +20,7 @@
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
 import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
+import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException;
 import com.spectralogic.ds3client.helpers.JobPart;
 import com.spectralogic.ds3client.helpers.JobState;
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
@@ -108,6 +109,9 @@ public void transfer() throws IOException {
                 final CountDownLatch countDownLatch = new CountDownLatch(Iterables.size(jobParts));
                 transferJobParts(jobParts, countDownLatch);
                 countDownLatch.await();
+            } catch (final Ds3NoMoreRetriesException e) {
+                emitFailureEvent(makeFailureEvent(failureActivity, e, firstChunk()));
+                throw e;
             } catch (final InterruptedException e) {
                 LOG.info("Error getting entries from work queue.", e);
                 Thread.currentThread().interrupt();

From 11044b3ba99f90606aee1ceb8cec0ce14b39fa48 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Fri, 2 Jun 2017 16:32:40 -0600
Subject: [PATCH 027/229] Got integration testing working

---
 .../ds3client/integration/Iterators_Test.java | 35 +++++++++++++++++++
 .../helpers/Ds3ClientHelpersImpl.java         |  6 ++--
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index d3c4bdd56..2965b5caf 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -15,10 +15,14 @@
 
 package com.spectralogic.ds3client.integration;
 
+import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.helpers.ContentPrefix;
 import com.spectralogic.ds3client.helpers.pagination.GetBucketLoaderFactory;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.pagination.GetObjectsFullDetailsLoaderFactory;
+import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
@@ -32,11 +36,14 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.UUID;
 
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
 import static com.spectralogic.ds3client.integration.Util.loadBookTestData;
+import static com.spectralogic.ds3client.integration.Util.loadBookTestDataWithPrefix;
 import static org.hamcrest.CoreMatchers.*;
 import static org.junit.Assert.*;
 
@@ -68,6 +75,34 @@ public static void teardown() throws IOException {
         LOG.info("Finished test teardown...");
     }
 
+    @Test
+    public void empty_dericty_test() throws IOException, URISyntaxException {
+        try {
+            HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
+            loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/");
+            final ContentPrefix contentPrefix = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 1000);
+            final List commonPrefixesList = contentPrefix.commonPrefixes();
+            final LazyIterable lazyIterable = (LazyIterable) contentPrefix.contents();
+            final ArrayList contents = Lists.newArrayList(lazyIterable);
+            assertThat(contents.size(), is(4));
+            assertThat(commonPrefixesList.size(), is(0));
+
+            loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/more/");
+            final ContentPrefix anotherContentPrefix = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 1000);
+            final List anotherCommonPrefixesList = anotherContentPrefix.commonPrefixes();
+            final LazyIterable anotherLazyIterable = (LazyIterable) anotherContentPrefix.contents();
+            final ArrayList anotherContents = Lists.newArrayList(anotherLazyIterable);
+            assertThat(anotherContents.size(), is(4));
+            assertThat(anotherCommonPrefixesList.size(), is(1));
+            assertThat(anotherCommonPrefixesList.get(0).getPrefix(), is("books/more/"));
+
+
+        }
+        finally {
+            deleteAllContents(CLIENT, TEST_ENV_NAME);
+        }
+    }
+
     @Test
     public void emptyGetBucket() throws IOException {
         final String prefix = "";
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index e4068b28d..816abfa5b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -357,13 +357,15 @@ public ContentPrefix remoteListDirectory(final String bucket, final String keyPr
         if (delimiter != null) {
             request.withDelimiter(delimiter);
         }
-        request.withMaxKeys(1);
+        if (nextMarker != null) {
+            request.withMarker(nextMarker);
+        }
+        //request.withMaxKeys(1);
         final GetBucketResponse response;
         final List commonPrefixes;
         try {
             response = this.client.getBucket(request);
             final ListBucketResult result = response.getListBucketResult();
-
             commonPrefixes = result.getCommonPrefixes();
 
         } catch (final FailedRequestException e) {

From 64cad9be66a25e95d4a567af74860040f6eaca7d Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Mon, 5 Jun 2017 12:35:34 -0600
Subject: [PATCH 028/229] Fixing integration test that got left out of a merge.

---
 .../integration/GetJobManagement_Test.java    | 37 +++++++++++--------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
index e7a0f6da7..3b659018d 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
@@ -327,7 +327,7 @@ private static void putBigFiles() throws IOException, URISyntaxException {
     }
 
     @Test(expected = AccessDeniedException.class)
-    public void testReadRetrybugFixWithUnwritableDirectory() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InterruptedException {
+    public void testReadRetryBugFixWithUnwritableDirectory() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InterruptedException {
         final String tempPathPrefix = null;
         final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
 
@@ -398,24 +398,29 @@ private boolean iAmRoot() {
             return false;
         }
 
+        BufferedReader idProcessStdOut = null;
+
         try {
-            final Process whoamiProcess = Runtime.getRuntime().exec("id | cut -d \"(\" -f 1 | cut -d \"=\" -f 2");
-            whoamiProcess.waitFor();
-
-            try (final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(whoamiProcess.getInputStream()))) {
-                final String userId = bufferedReader.readLine();
-
-                final String[] uidFields = userId.split("uid=");
-                if (uidFields.length > 1) {
-                    final String[] uidStrings = uidFields[1].split("\\(");
-                    if (uidStrings.length > 1) {
-                        final int uid = Integer.parseInt(uidStrings[0]);
-                        return uid == 0;
-                    }
-                }
-            }
+            final Process idProcess = Runtime.getRuntime().exec("id");
+            idProcessStdOut = new BufferedReader(new InputStreamReader(idProcess.getInputStream()));
+            final String idProcessResult = idProcessStdOut.readLine();
+            idProcess.waitFor();
+
+            final String[] idFields = idProcessResult.split("=");
+
+            final String[] uidString = idFields[1].split("\\(");
+
+            return uidString[0].equals("0");
         } catch (final IOException | InterruptedException e) {
             LOG.error("Error getting user id.", e);
+        } finally {
+            try {
+                if (idProcessStdOut != null) {
+                    idProcessStdOut.close();
+                }
+            } catch (final IOException e) {
+                LOG.error("Failure closing stdout for process that gets user id.");
+            }
         }
 
         return false;

From 18f9a62439995d5b7d33594cf9f20336c1692a8f Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Mon, 5 Jun 2017 12:54:59 -0600
Subject: [PATCH 029/229] Rev version to 3.4.1-SNAPSHOT and updating gradle
 version

---
 .gitignore                               |   1 +
 build.gradle                             |   4 ++--
 gradle/wrapper/gradle-wrapper.jar        | Bin 54224 -> 54208 bytes
 gradle/wrapper/gradle-wrapper.properties |   4 ++--
 gradlew                                  |  11 ++++++-----
 5 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index 204cfe08d..1440e929b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 .gradle/
+.gradletasknamecache
 .idea/
 out/
 coverage-report/
diff --git a/build.gradle b/build.gradle
index 36ddc0040..01e123e56 100644
--- a/build.gradle
+++ b/build.gradle
@@ -15,7 +15,7 @@
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '3.4.0'
+    version = '3.4.1-SNAPSHOT'
 }
 
 subprojects {
@@ -41,7 +41,7 @@ subprojects {
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '3.3'
+    gradleVersion = '3.5'
 }
 
 project(':ds3-sdk') {
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index d6e2637affb74a80bfbe87bd2da57e81b2f3c661..7b61d71aa610f3b12c0d96a3d392c2c47f5a6a78 100644
GIT binary patch
delta 4749
zcmZ`*2{@G78~>PW6S5m)ELkdKiOQBFDUmixL}SYsTrsXvVWQGSh)-FfP0>yEHHO^m
z`<5tKyGf+Tzng^q=bO;?aqoGaGtc+Vd4IomIp;m+ZB50Mr{Z`In&MUoK@cY=g!hv*
ziQ<*w{UyDa4g3FhtYz~wx;9{;g%&1`6dq1LU^k74Rg~T
zzPQ2Q6KajEVvpo}gFV?aY$_&J7k%K-Z1gnnEwg<&sqpc+#(c!5!L$8ezx8$Y&ws7a
zXVD5JzxO3Q;Pn#_eXf>1NDtE)^4(u?HA2qBb&6qK+8QC+IV)tcRWZEM~cQLVoI%b%U*Ly`AD!3butEASw}P9
zRdys51sm~@+a`|ikOfDSZ#CS0qIF`!Erynly&q#DaR+%a$ET`Z=LTIjHVN<7lO)bR
zQh(kMLK{n(_b^ibJcOaHW^aKJx*
z=jU`h?6Lg2XuLL?X7yr>E_lPOY47bF@KIlBwHpybS*4Go#Y0Yqm!Hl#*`loJM=mzl
zdP+OuvsSc~@Ruhq8*)e+96wt({Ua(iMBPbad=4uAXx#ds_cP;dar>WFcnMWOCNVbQ
z5g}G#JB^$>sm}d-2KZep=VfGjZp5~9Z;v{QKb(2$iEt*dN-AXBX3~cq&@0hIRnAk&
z(A;$`a%;>u`FM2IE1DIrNBFrs3!?UZ|0Mb7{PJ}a%{A4YtkAh;sxBKl(Zj#krOhCI%nhMbPm|}
zWaGQ>0W%{TR2x=r%L(BRrt7}lGD}Mv_%6A>#r}c29ifYd^voV-sI}4VaYS!<*JNmU
z^4RTxkl=BP`=i=>H4iRx&TkhmePbpT
ztb0D6sQ5E3Lp1Ir**b8u+l~4hD=X)?vvpHuRk~9GUh(W@qS`ZMfgN-$*@^CoqIG;z
znZ#+9fQN%TIQ?glJVKl94z6bp2))hq`;B{j_UB^v2{+vId$-OY=H1CEhU-g~H$0`8
z+vGLQJ$D|sY)4B8F65piopST7DK9C?7)%Ky4_pmC&K?jl_W1d_Dj~wmvGGfh?s}Gn
zp=Nt3Z*pshTz-AC`7wQjXhd(eK61k20A=ggpL^f-r}G^-_vXv)a1k~O3MXU4c%VGx
z0402F)#NDa7_l_+T}6R&7tTX>eCAO^yJ5Lwu1^bf6|EgV{7r5vZufZ|`Kg9+Cp+z8
zWn%p;WwyAqa`MLDai33y>G{MDkD_CS>xjduhtAx8JZW_QfYR?J9hcK{=!KMqk2{sz
z{hMd*rTYn--cBM_OgxZHFSfZzbIq2lRZJVU&PaXO!k&LOOzn@XCc6vdshV%;9)@_T
zleAi`Ew|2$Maw|(fqb53uRb2xhDpvd8ZlJ!{^`GZ_%4=g-{J20mBZz{zQuOpqbhA)
z&)1Q>q#c^PLtMAwqKLODQ}5BGxbBMcXT&5Ex^7ofHW+DU;{`@oE6q*+Xd$`EWMBHb
z@i!Nn0?oFzRBxAiQ(Ld*8Zw%y%qQ0sdkj5HDCsv!O*rr2V8>Oeq1;>co5j7~`*S|Z
ze%+HDkMA_!-e*IRVa_EYbVT=h+PufBN9Sg0g)cNE5s8M@h#%EJ6vwe_lS
z8k>zR?wYas4M;!uvAyfblYQ?>6SGZPi)v4S4|)7w92)#O42-byU>%naKdVqs6ERrJFUFC?fwga
z>XAvg?Uf3%;%u{5#=k$+y7+xVD505)2j1mn^1o#1(}Q{44Z{H!&WQC%{N+>|C0Z9p
z{McS$-a3-MS6};96h{MN(Djm}QPB)30@nOaotoO_W@!CCfx3q?wqbOE7OtF!^q{A*
zk(BB4U*tN&g`2!wZ@IR87o6T%+t2#rTO0MzT*w5k%hQX2+ak=f!aQWX=Zr6@D(x3es{7<1Q#?s?DZ8!S
zo7ggBGnTz?$m%*r;+nyIb5h82hUZB-;#bodtXnkb`H8zwC
zQ&e7|#9UL=5DZ=4CNm!N`WF2|f@t~L;Q>5aK4+VR!LPMzN6YY>!;uB`FW{+$*`4H3
zV(u2Fmy6oSaxQ@~ijlcp8`(sEKGw*(y1Qk831|*hu|d#o_)wA@9Nr+GtL3>L4T7)H
zhR`u@>!pHjBIx1C`L~A&0I_<(Fr=CH;`19du
zAX*wHwJzP5ZqQrq$`;!D>gT_Oer;)nJDLMd7;4$8Z-{h
zH?OmU1%VbnmV(AFGsBYr4GKz77!Ggy_5UN{|3T^6+EDMgrN5joLwmb
z4^*sLjdBV8R6zheEQy8LS4vkBUtC>wwpz(>I*FNw$dAyr@>X)^;#Ya1RcdekQN#Pv50}FWZ-!M0lrhk
zhKVez3W;pBIFof>4HrCGC4u(D>0D!fG|RcbfLdi3Z45#o)f#HK(T!TXGR&rV`0bC
zm#*Z1P3;6%#QNrPD_If%NMK^Kr{)M>s8e)Wa2C0z5Z=XtUV{irBhv9_xh)irqdOH-
zKnQqGoAZZgfKCLZtg!?{KFS>7J8i1}l28OjUbDSD8Nd`cmc#Bk!xN`u;k^_AOi&0&
z=3Je1G_Vs7HW6Dr*Qo$Rg7J5``OsU3fZ+WNNPgW;0S#lgzm07%zsRCG`DA1Y@O1>t
z4T#fALyU&Fp|NS@c|dT9?lvu{pwLoq%L|Ro8$_)D+ct_UZZgw}fJOpPWrgD#ccSgD
zUuDcK26!k)Dwb>`A1_WbBa&aGCnF5vvC2NG8g?jBtXped1Q#kWOjvF@08PGSf%0J$_c
zB^xpNj&NzqW=tw)TaZ*zli`oa8{i$SOZOL(=U^+cXpqJ<64@$^-ZaBDk>OF`lplC$
zwqcAIW-#@~GQ^dZA46qfmmm({SqY$3$AGTD`fWOBe#T=nb)@Jof*`P|?aKp)YZoSp
zPi=^!^@Yq+=UXJQB84NeJfHV~oSOp1O&G(a#lTwcHa#DL5h
zOiT0aOP4P*Uy>b&8$M6LxfFxVJESl^`gb6QAA;jL9Pzqn!Vr1Xj
zPzg~)pHj$|`k!~w=Y946&viXBbFOorbMAAV1R0yUez7fwb@%0+M+;rDmT64qHmj(RecoF5TD(a=y9@tcJTCK
zWoj}-U3XJAkFTb6ny3oNc%ZQk_e#JvKA~(_S;g=0#^>!4Pm7jB^>Axj9r9hiefgx^
zCaLy(z5AqgAGuQUFtokgMzwsdCq~xo@s)k0TpwHrjTcQ965NKz6q8JFY@L``qQPmC}m3i39i5OK9@D_cd*Znb?`V
z%UJaqeFG=is?=J1N1{QGaCIgGnxpSQse`|_BBOEm9i;L1PSK(S7j&J
zowycUo8|1-VtIS(Xkwj?V1>&%j;61ds$|R@g>r62*LtsuLm(V*=Or!-$TvEXc~hpD+rFGnxE1%*D%lly45JlCC?)?6Q}N#mQHw
zslC`WOW2#KeT)5Oh{fV7{nG7rSEWfMH?W9tPscFitLu4Ks-{INq7jI1!
z&dtAylPV4C`H`HnGjYC|s$^x)Q71ZLlW{j{TZxVBiDxhJrQgg&WP4r>?e|(Vn7Ysy
z{3ftx_S+{k9^cxcs^cV-XK;rWxxY*bkN}^#Rk;~vz*ai0*zCv#**JbaZ-Xu56
zjv1!eb4q6;&KiZ?Q=bXU5@vgO{-Jnje1!_lGVdl!m;bSFM&UD;{z~yA_$<98zjGK2
z<;@QM%DvIkdx2qWh^hVn?hfm7qv8*lVYnXdGKinQ%-i&K-Trweu(8)^oy$(4Ib97@UmE=a0+;@L5jGZhp3kwCdl
zYyFY*U@^K;eB=bkz`UbhahccZTN&NRH?
zi}RfP_%^stm?u@y`L>>BhS(!kem<*Ac*a{`C#}}ZPQKI1fAj6GBTq_H7O8KpS@0GS
z?;m54v!-x-G&|;=W>em&V&ydXTzS2$XspUp;}izCgvqV-h&knD*TheVWqQkH+H__3
ze`ch67uatvFncXz?7iA!9{un*^%A_W$K}(yfKi{RN4m|*jfW(nyt9K27U|hal^#%Y
ziaq*v(5S{sKhQ$lc%LG%xO)KKebhf;t{i?Wn_MiEA(HFQo7Q{J@q+)8ek1#>3E9lt
zv;5+VLwE48V!K7dBp(FW_L)b0P)!irz0GL%xhwv(+XWS88@v8)w$2hAcOdTMU)+=L
zT@zyzQNr7D+SvD8%u{{>p(evEd?{B%Y>CH_>+btlMun$v-i1?1!>lH!xqVx?PcmKV{1c>G&2^4kDgjJzYL(Yr}OuAqsQA*xPZ}H7HeYoB;GZ{=%?H8
zgc;~>R34g;G)nnyYM);Lh%*SL#Byxp3uLy3CWl5IS@4j1501AG7X)paJx06|BmjU2he>e3nX3S$bbu(C
z?qT?`C_4G0XI$KJK%}Q&N@61JSNKJ@9YwmbM3Li<99}IBkp(y*=(QjO
ziDNu=gRKw#c4)6>RZ(!J9QN~DT~uZ-(QBkN5Hv-GATf-oj4m)mvSWsonh6_Ft59X;Re
zC<5@pU}7uy_YwGbi|p@eBlixrg`k6&JA;NIOkO$12pj=`TZ2Bhm%kwhIXVk)UCTPO
zXX75XsRV#c12>H7zfk00$hg4beq>mtl^2b79~7eXH&a5vK_!5Ls);f1^oK?Kwc+sge~%MAdR#jaT{jRUBn)!}aw7mV
zgP;u~)=uDz3W3{C3*+#55FB4C2n%UieX$0y_BK?QEocu;BCH=GDd&r=XA?JiKuI
zA4_D=LmZz3j6*?1u|`G3bfcVZF$~+zZiMa3UAmI9hlo)s^dL&(`#?#C{d(4;>6O=&
z`XvK;1%RFohMsCA9nW1!{Da
Date: Mon, 5 Jun 2017 14:17:28 -0600
Subject: [PATCH 030/229] Working on integration tests for edge case.

---
 .../ds3client/integration/Iterators_Test.java | 96 ++++++++++++++++++-
 .../helpers/Ds3ClientHelpersImpl.java         |  2 +-
 2 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 2965b5caf..68b54b299 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -17,17 +17,24 @@
 
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.GetBucketRequest;
+import com.spectralogic.ds3client.commands.GetBucketResponse;
+import com.spectralogic.ds3client.commands.PutObjectRequest;
+import com.spectralogic.ds3client.commands.spectrads3.GetBucketSpectraS3Request;
 import com.spectralogic.ds3client.helpers.ContentPrefix;
 import com.spectralogic.ds3client.helpers.pagination.GetBucketLoaderFactory;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.pagination.GetObjectsFullDetailsLoaderFactory;
 import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.ListBucketResult;
 import com.spectralogic.ds3client.models.common.CommonPrefixes;
+import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.networking.FailedRequestException;
+import com.sun.xml.internal.ws.policy.privateutil.PolicyUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -80,7 +87,7 @@ public void empty_dericty_test() throws IOException, URISyntaxException {
         try {
             HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/");
-            final ContentPrefix contentPrefix = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 1000);
+            final ContentPrefix contentPrefix = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
             final List commonPrefixesList = contentPrefix.commonPrefixes();
             final LazyIterable lazyIterable = (LazyIterable) contentPrefix.contents();
             final ArrayList contents = Lists.newArrayList(lazyIterable);
@@ -96,13 +103,98 @@ public void empty_dericty_test() throws IOException, URISyntaxException {
             assertThat(anotherCommonPrefixesList.size(), is(1));
             assertThat(anotherCommonPrefixesList.get(0).getPrefix(), is("books/more/"));
 
-
+            for (int i = 0; i < 100; i++) {
+                loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/" + i + "/");
+            }
+            for (int i = 0; i < 20; i++) {
+                loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/" + i);
+            }
+            final ContentPrefix bulkContentPrefix = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 84);
+            final List bulkCommonPrefixesList = bulkContentPrefix.commonPrefixes();
+            final LazyIterable bulkLazyIterable = (LazyIterable) bulkContentPrefix.contents();
+            final ArrayList bulkContents = Lists.newArrayList(bulkLazyIterable);
+            for(CommonPrefixes c : bulkCommonPrefixesList) {
+                LOG.info(c.getPrefix());
+            }
+            //assertThat(bulkCommonPrefixesList.size(), is(101));
+            assertThat(bulkContents.size(), is(84));
         }
         finally {
             deleteAllContents(CLIENT, TEST_ENV_NAME);
         }
     }
 
+    public void putString(String objectName, String objectContent) throws IOException {
+        final byte[] content = objectContent.getBytes();
+        final PutObjectRequest putObjectRequest = new PutObjectRequest(TEST_ENV_NAME, objectName, new ByteArraySeekableByteChannel(content), content.length);
+        CLIENT.putObject(putObjectRequest);
+    }
+
+    public void loopWithNextMarker(final String bucket, final String prefix, final String delimiter) throws IOException {
+        GetBucketRequest getBucketRequest = new GetBucketRequest(bucket).withPrefix(prefix).withDelimiter(delimiter).withMaxKeys(3);
+        GetBucketResponse clientBucket = CLIENT.getBucket(getBucketRequest);
+        ListBucketResult listBucketResult = clientBucket.getListBucketResult();
+        logPrefix(listBucketResult);
+        logObjects(listBucketResult);
+        final String nextMarker = listBucketResult.getNextMarker();
+        System.out.println("next marker " + nextMarker);
+        loopWithNextMarker(bucket,prefix,delimiter,nextMarker);
+    }
+
+    public void loopWithNextMarker(final String bucket, final String prefix, final String delimiter, final String marker) throws IOException {
+        if (marker == null) {
+            return;
+        }
+        GetBucketRequest getBucketRequest = new GetBucketRequest(bucket).withPrefix(prefix).withDelimiter(delimiter).withMaxKeys(3).withMarker(marker);
+        GetBucketResponse clientBucket = CLIENT.getBucket(getBucketRequest);
+        ListBucketResult listBucketResult = clientBucket.getListBucketResult();
+        logPrefix(listBucketResult);
+        logObjects(listBucketResult);
+        final String nextMarker = listBucketResult.getNextMarker();
+        System.out.println("next marker " + nextMarker);
+        loopWithNextMarker(bucket,prefix,delimiter,nextMarker);
+    }
+
+    public void logPrefix(ListBucketResult listBucketResult) {
+        for (CommonPrefixes commonPrefixes : listBucketResult.getCommonPrefixes()) {
+            System.out.println("Prefix: " + commonPrefixes.getPrefix());
+        }
+    }
+
+    public void logObjects(ListBucketResult listBucketResult) {
+        for (Contents contents : listBucketResult.getObjects()) {
+            System.out.println("Object: " + contents.getKey());
+        }
+    }
+
+    @Test
+    public void matchS3() throws IOException {
+        final String[] names = {
+                "dir/a.txt",
+                "dir/foo.txt",
+                "dir/foo1.txt",
+                "dir/foo2.txt",
+                "dir/baz/foo.txt",
+                "dir/tar/foo.txt",
+                "dir/1/foo.txt",
+                "dir/2/foo.txt",
+                "dir/3/foo.txt",
+                "dir/4/foo.txt"
+        };
+        final String content = "content";
+        try {
+            HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
+            for(final String s : names) {
+                putString(s, content);
+            }
+            loopWithNextMarker(TEST_ENV_NAME,"dir/", "/");
+            assertThat(true, is(true));
+
+        } finally {
+            deleteAllContents(CLIENT, TEST_ENV_NAME);
+        }
+    }
+
     @Test
     public void emptyGetBucket() throws IOException {
         final String prefix = "";
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 816abfa5b..d0b2c22b1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -360,7 +360,7 @@ public ContentPrefix remoteListDirectory(final String bucket, final String keyPr
         if (nextMarker != null) {
             request.withMarker(nextMarker);
         }
-        //request.withMaxKeys(1);
+        request.withMaxKeys(10);
         final GetBucketResponse response;
         final List commonPrefixes;
         try {

From 89a03647da497ae50d1bc0ac0266b80058b11553 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Tue, 6 Jun 2017 08:57:36 -0600
Subject: [PATCH 031/229] Working on a larger re-write

---
 .../spectralogic/ds3client/integration/Iterators_Test.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 68b54b299..26b40dc89 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -164,6 +164,7 @@ public void logPrefix(ListBucketResult listBucketResult) {
     public void logObjects(ListBucketResult listBucketResult) {
         for (Contents contents : listBucketResult.getObjects()) {
             System.out.println("Object: " + contents.getKey());
+            contents.
         }
     }
 
@@ -179,7 +180,8 @@ public void matchS3() throws IOException {
                 "dir/1/foo.txt",
                 "dir/2/foo.txt",
                 "dir/3/foo.txt",
-                "dir/4/foo.txt"
+                "dir/4/foo.txt",
+
         };
         final String content = "content";
         try {
@@ -187,6 +189,7 @@ public void matchS3() throws IOException {
             for(final String s : names) {
                 putString(s, content);
             }
+            putString("dir/4/", "");
             loopWithNextMarker(TEST_ENV_NAME,"dir/", "/");
             assertThat(true, is(true));
 

From 1b4af4450090282de7a74ce3f30aeeb5f16a8f9a Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 6 Jun 2017 11:25:21 -0600
Subject: [PATCH 032/229] Fixing a test that really does not make sense to run
 as root.  Since there are other tests that exercise the ability to detect an
 error, not running this particular one as root is not a concern.

---
 .../ds3client/integration/GetJobManagement_Test.java            | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
index 3b659018d..1754d7b32 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
@@ -328,6 +328,8 @@ private static void putBigFiles() throws IOException, URISyntaxException {
 
     @Test(expected = AccessDeniedException.class)
     public void testReadRetryBugFixWithUnwritableDirectory() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InterruptedException {
+        Assume.assumeFalse(iAmRoot());
+
         final String tempPathPrefix = null;
         final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
 

From 00717e0de0bbf83252daf729ac2bf822c0c4acb0 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Tue, 6 Jun 2017 11:56:42 -0600
Subject: [PATCH 033/229] Getting my GetBucketKeyLoaderFactory ready.

---
 .../ds3client/integration/Iterators_Test.java |  1 -
 .../pagination/GetBucketKeyLoader.java        | 98 +++++++++++++++++++
 .../pagination/GetBucketKeyLoaderFactory.java | 31 ++++++
 .../ds3client/models/FileSystemKey.java       | 78 +++++++++++++++
 4 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 26b40dc89..6e6e5a71c 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -164,7 +164,6 @@ public void logPrefix(ListBucketResult listBucketResult) {
     public void logObjects(ListBucketResult listBucketResult) {
         for (Contents contents : listBucketResult.getObjects()) {
             System.out.println("Object: " + contents.getKey());
-            contents.
         }
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
new file mode 100644
index 000000000..38d93115d
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -0,0 +1,98 @@
+package com.spectralogic.ds3client.helpers.pagination;
+
+import com.google.common.collect.ImmutableList;
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.GetBucketRequest;
+import com.spectralogic.ds3client.commands.GetBucketResponse;
+import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.FileSystemKey;
+import com.spectralogic.ds3client.models.ListBucketResult;
+import com.spectralogic.ds3client.models.common.CommonPrefixes;
+import com.spectralogic.ds3client.networking.FailedRequestException;
+import com.spectralogic.ds3client.networking.TooManyRetriesException;
+import com.spectralogic.ds3client.utils.Guard;
+import com.spectralogic.ds3client.utils.collections.LazyIterable;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+public class GetBucketKeyLoader implements LazyIterable.LazyLoader{
+    private static final int DEFAULT_MAX_KEYS = 1000;
+
+    private final Ds3Client client;
+    private final String bucket;
+    private final String prefix;
+    private final String delimiter;
+    private final int maxKeys;
+    private final int retryCount;
+
+    private String nextMarker;
+    private boolean truncated;
+    private boolean endOfInput = false;
+
+    public GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount) {
+        this.client = client;
+        this.bucket = bucket;
+        this.prefix = prefix;
+        this.delimiter = delimiter;
+        this.nextMarker = nextMarker;
+        this.maxKeys = Math.min(maxKeys,DEFAULT_MAX_KEYS);
+        this.retryCount = retryCount;
+        this.nextMarker = nextMarker;
+        this.truncated  = nextMarker != null;
+    }
+
+    @Override
+    public List getNextValues() {
+        if(endOfInput) {
+            return Collections.emptyList();
+        }
+        int retryAttempt = 0;
+        while(true) {
+            final GetBucketRequest request = new GetBucketRequest(bucket);
+            request.withMaxKeys(maxKeys);
+            if (prefix != null) {
+                request.withPrefix(prefix);
+            }
+            if (truncated) {
+                request.withMarker(nextMarker);
+            }
+            if (delimiter != null) {
+                request.withDelimiter(delimiter);
+            }
+            GetBucketResponse response;
+            try {
+                response = this.client.getBucket(request);
+            } catch (final FailedRequestException e) {
+                throw new RuntimeException("Failed to get the list of objects due to a failed request", e);
+            } catch (final IOException e) {
+               if (retryAttempt >= retryCount) {
+                   throw new TooManyRetriesException("Failed to get the next set of objects from the getBucketKey request after " + retryCount + " retries", e);
+               }
+               response = null;
+            }
+            if (response == null) {
+                retryAttempt++;
+            } else {
+                final ListBucketResult result = response.getListBucketResult();
+                truncated = result.getTruncated();
+                this.nextMarker = result.getNextMarker();
+                if (Guard.isStringNullOrEmpty(nextMarker) && !truncated) {
+                    endOfInput = true;
+                }
+                final List contents = result.getObjects();
+                final List prefixes = result.getCommonPrefixes();
+                final ImmutableList.Builder fileSystemKeys = new ImmutableList.Builder<>();
+                for(CommonPrefixes prefix: prefixes) {
+                    fileSystemKeys.add(new FileSystemKey(prefix));
+                }
+                for(Contents content : contents) {
+                    fileSystemKeys.add(new FileSystemKey(content));
+                }
+                return fileSystemKeys.build();
+            }
+
+        }
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
new file mode 100644
index 000000000..55abd7aae
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -0,0 +1,31 @@
+package com.spectralogic.ds3client.helpers.pagination;
+
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.models.FileSystemKey;
+import com.spectralogic.ds3client.utils.collections.LazyIterable;
+
+public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFactory {
+
+    private final Ds3Client client;
+    private final String bucket;
+    private final String keyPrefix;
+    private final String nextMarker;
+    private final String delimiter;
+    private final int maxKeys;
+    private final int defaultListObjectsRetries;
+
+    public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries) {
+        this.client = client;
+        this.bucket = bucket;
+        this.keyPrefix = keyPrefix;
+        this.delimiter = delimiter;
+        this.nextMarker = nextMarker;
+        this.maxKeys = maxKeys;
+        this.defaultListObjectsRetries = defaultListObjectsRetries;
+    }
+
+    @Override
+    public LazyIterable.LazyLoader create() {
+        return new GetBucketKeyLoader(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, defaultListObjectsRetries);
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
new file mode 100644
index 000000000..3a003b8cc
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
@@ -0,0 +1,78 @@
+package com.spectralogic.ds3client.models;
+
+import com.spectralogic.ds3client.models.common.CommonPrefixes;
+
+import java.util.Date;
+
+public class FileSystemKey {
+    private final String prefix;
+    private final String eTag;
+    private final String key;
+    private final Date lastModified;
+    private final User owner;
+    private final Long size;
+    private final String storageClass;
+
+    private FileSystemKey(final String prefixString) {
+        prefix = prefixString;
+        eTag = null;
+        key = null;
+        lastModified = null;
+        owner = null;
+        size = null;
+        storageClass = null;
+    }
+
+    public FileSystemKey(final CommonPrefixes commonPrefixes) {
+        this(commonPrefixes.getPrefix());
+    }
+
+    private FileSystemKey(final String eTag, final  String key, final  Date lastModified, final User owner, final long size, final String storageClass) {
+        this.prefix = null;
+        this.eTag = eTag;
+        this.key = key;
+        this.lastModified = lastModified;
+        this.owner = owner;
+        this.size = size;
+        this.storageClass = storageClass;
+    }
+
+    public FileSystemKey(final Contents contents) {
+        this(contents.getETag(), contents.getKey(), contents.getLastModified(), contents.getOwner(), contents.getSize(), contents.getStorageClass());
+    }
+
+    public boolean isPrefix() {
+        return prefix != null;
+    }
+
+    public boolean isContents() {
+        return prefix == null;
+    }
+
+    public CommonPrefixes toCommonPrefixes() throws RuntimeException{
+       if(isContents()) {
+           throw new RuntimeException("Could not create CommonPrefixes");
+       }
+       final CommonPrefixes commonPrefixes = new CommonPrefixes();
+       commonPrefixes.setPrefix(prefix);
+       return commonPrefixes;
+    }
+
+    public Contents toContents() throws RuntimeException {
+        if(isPrefix()) {
+            throw new RuntimeException("Could not create Contents");
+        }
+        final Contents contents = new Contents();
+        contents.setETag(eTag);
+        contents.setKey(key);
+        contents.setLastModified(lastModified);
+        contents.setOwner(owner);
+        contents.setSize(size);
+        contents.setStorageClass(storageClass);
+        return contents;
+    }
+
+    public String getName() {
+        return isPrefix() ? prefix : key;
+    }
+}
\ No newline at end of file

From dc1ebb40540bcb73612eac52e0cebfa2c1d6cd8b Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Tue, 6 Jun 2017 13:17:45 -0600
Subject: [PATCH 034/229] Basic tests work and the structure is coming
 together.

---
 .../ds3client/integration/Iterators_Test.java | 38 ++++---------------
 .../ds3client/helpers/Ds3ClientHelpers.java   | 11 +++---
 .../helpers/Ds3ClientHelpersImpl.java         | 33 +++-------------
 .../pagination/GetBucketKeyLoader.java        | 13 ++-----
 .../helpers/Ds3ClientHelpers_Test.java        |  6 +--
 5 files changed, 23 insertions(+), 78 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 6e6e5a71c..91b690a64 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -26,6 +26,7 @@
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.pagination.GetObjectsFullDetailsLoaderFactory;
 import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.FileSystemKey;
 import com.spectralogic.ds3client.models.ListBucketResult;
 import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
@@ -83,41 +84,18 @@ public static void teardown() throws IOException {
     }
 
     @Test
-    public void empty_dericty_test() throws IOException, URISyntaxException {
+    public void IterateWithDelimiter() throws IOException, URISyntaxException {
         try {
             HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/");
-            final ContentPrefix contentPrefix = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
-            final List commonPrefixesList = contentPrefix.commonPrefixes();
-            final LazyIterable lazyIterable = (LazyIterable) contentPrefix.contents();
-            final ArrayList contents = Lists.newArrayList(lazyIterable);
-            assertThat(contents.size(), is(4));
-            assertThat(commonPrefixesList.size(), is(0));
+            Iterable  fileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
+            List fileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
+            assertThat(fileSystemKeys.size(), is(4));
 
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/more/");
-            final ContentPrefix anotherContentPrefix = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 1000);
-            final List anotherCommonPrefixesList = anotherContentPrefix.commonPrefixes();
-            final LazyIterable anotherLazyIterable = (LazyIterable) anotherContentPrefix.contents();
-            final ArrayList anotherContents = Lists.newArrayList(anotherLazyIterable);
-            assertThat(anotherContents.size(), is(4));
-            assertThat(anotherCommonPrefixesList.size(), is(1));
-            assertThat(anotherCommonPrefixesList.get(0).getPrefix(), is("books/more/"));
-
-            for (int i = 0; i < 100; i++) {
-                loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/" + i + "/");
-            }
-            for (int i = 0; i < 20; i++) {
-                loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/" + i);
-            }
-            final ContentPrefix bulkContentPrefix = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 84);
-            final List bulkCommonPrefixesList = bulkContentPrefix.commonPrefixes();
-            final LazyIterable bulkLazyIterable = (LazyIterable) bulkContentPrefix.contents();
-            final ArrayList bulkContents = Lists.newArrayList(bulkLazyIterable);
-            for(CommonPrefixes c : bulkCommonPrefixesList) {
-                LOG.info(c.getPrefix());
-            }
-            //assertThat(bulkCommonPrefixesList.size(), is(101));
-            assertThat(bulkContents.size(), is(84));
+            Iterable  anotherFileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
+            List anotherFileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
+            assertThat(anotherFileSystemKeys.size(), is(5));
         }
         finally {
             deleteAllContents(CLIENT, TEST_ENV_NAME);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index bc619c262..a1c8934d9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -23,6 +23,7 @@
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.FileSystemKey;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.utils.Predicate;
 
@@ -470,27 +471,25 @@ public abstract Iterable listObjects(final String bucket, final String
 
     /**
      * Defaults to the path delimiter of '/'
-     * Returns an Iterable of {@link Ds3Object} and a list of {@link com.spectralogic.ds3client.models.common.CommonPrefixes}
+     * Returns an Iterable of {@link FileSystemKey}
      * @param bucket The bucket
      * @param keyPrefix Limits the response to keys that begin with the specified prefix
      * @param nextMarker Specifies the key to start with when listing objects
      * @param maxKeys Sets the maximum number of keys returned in the response body.
      * @return {@link ContentPrefix}
-     * @throws IOException
      */
-    public abstract ContentPrefix remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException;
+    public abstract Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException;
 
     /**
-     * Returns an Iterable of {@link Ds3Object} and a list of {@link com.spectralogic.ds3client.models.common.CommonPrefixes}
+     * Returns an Iterable of {@link FileSystemKey}
      * @param bucket The bucket
      * @param keyPrefix Limits the response to keys that begin with the specified prefix
      * @param delimiter Specifies a path delimiter for the S3 query
      * @param nextMarker Specifies the key to start with when listing objects
      * @param maxKeys Sets the maximum number of keys returned in the response body.
      * @return {@link ContentPrefix}
-     * @throws IOException
      */
-    public abstract ContentPrefix remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException;
+    public abstract Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException;
 
     /**
      * Returns an Iterable of {@link Ds3Object} that have a prefix added.
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index ea44e848a..8b6eedddb 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -24,6 +24,7 @@
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
+import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
 import com.spectralogic.ds3client.helpers.pagination.GetBucketLoaderFactory;
 import com.spectralogic.ds3client.commands.*;
 import com.spectralogic.ds3client.commands.spectrads3.*;
@@ -35,8 +36,6 @@
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;
 import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers;
 import com.spectralogic.ds3client.models.*;
-import com.spectralogic.ds3client.models.bulk.RequestType;
-import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.models.common.Range;
 import com.spectralogic.ds3client.models.bulk.*;
 import com.spectralogic.ds3client.networking.FailedRequestException;
@@ -530,34 +529,12 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
     }
 
     @Override
-    public ContentPrefix remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException {
-        return remoteListDirectory(bucket,keyPrefix,"/", nextMarker, maxKeys);
+    public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException {
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, "/", nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
     }
 
-    public ContentPrefix remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException {
-        final Iterable contents = new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
-        final GetBucketRequest request = new GetBucketRequest(bucket);
-        if (keyPrefix != null) {
-            request.withPrefix(keyPrefix);
-        }
-        if (delimiter != null) {
-            request.withDelimiter(delimiter);
-        }
-        if (nextMarker != null) {
-            request.withMarker(nextMarker);
-        }
-        request.withMaxKeys(10);
-        final GetBucketResponse response;
-        final List commonPrefixes;
-        try {
-            response = this.client.getBucket(request);
-            final ListBucketResult result = response.getListBucketResult();
-            commonPrefixes = result.getCommonPrefixes();
-
-        } catch (final FailedRequestException e) {
-            throw new RuntimeException("Failed to get the list of objects due to a failed request", e);
-        }
-       return new ContentPrefix(contents, commonPrefixes);
+    public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException {
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
     }
 
     public Iterable addPrefixToDs3ObjectsList(final Iterable objectsList, final String prefix) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index 38d93115d..8032232e5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -52,15 +52,9 @@ public List getNextValues() {
         while(true) {
             final GetBucketRequest request = new GetBucketRequest(bucket);
             request.withMaxKeys(maxKeys);
-            if (prefix != null) {
-                request.withPrefix(prefix);
-            }
-            if (truncated) {
-                request.withMarker(nextMarker);
-            }
-            if (delimiter != null) {
-                request.withDelimiter(delimiter);
-            }
+            if (prefix != null) request.withPrefix(prefix);
+            if (truncated)  request.withMarker(nextMarker);
+            if (delimiter != null) request.withDelimiter(delimiter);
             GetBucketResponse response;
             try {
                 response = this.client.getBucket(request);
@@ -92,7 +86,6 @@ public List getNextValues() {
                 }
                 return fileSystemKeys.build();
             }
-
         }
     }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index bfb83f633..f715ed751 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -272,10 +272,8 @@ public void testRemoteListDirectory() throws IOException {
         Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, null))).thenReturn(new StubGetBucketResponse(0));
         Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, "baz"))).thenReturn(new StubGetBucketResponse(1));
         Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, "/foo/baz"))).thenReturn(new StubGetBucketResponse(1));
-        final ContentPrefix contentPrefix = Ds3ClientHelpers.wrap(ds3Client).remoteListDirectory(MYBUCKET, null, "/", null, 0);
-        final List contentsList = Lists.newArrayList(contentPrefix.contents());
-        final List commonPrefixes = contentPrefix.commonPrefixes();
-        assertThat(commonPrefixes.size(), is(0));
+        Iterable fileSystemKeys = Ds3ClientHelpers.wrap(ds3Client).remoteListDirectory(MYBUCKET, null, "/", null, 0);
+        final List contentsList = Lists.newArrayList(fileSystemKeys);
         assertThat(contentsList.size(), is(3));
     }
     

From cc7faf51aa350f417aa5347606db4e2b49d6a2d7 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Tue, 6 Jun 2017 13:24:44 -0600
Subject: [PATCH 035/229] Removing unused artifacts.

---
 .../ds3client/integration/Iterators_Test.java |  4 ---
 .../ds3client/helpers/ContentPrefix.java      | 25 -------------------
 2 files changed, 29 deletions(-)
 delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ContentPrefix.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 91b690a64..2cb9642a6 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -20,8 +20,6 @@
 import com.spectralogic.ds3client.commands.GetBucketRequest;
 import com.spectralogic.ds3client.commands.GetBucketResponse;
 import com.spectralogic.ds3client.commands.PutObjectRequest;
-import com.spectralogic.ds3client.commands.spectrads3.GetBucketSpectraS3Request;
-import com.spectralogic.ds3client.helpers.ContentPrefix;
 import com.spectralogic.ds3client.helpers.pagination.GetBucketLoaderFactory;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.pagination.GetObjectsFullDetailsLoaderFactory;
@@ -35,7 +33,6 @@
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.networking.FailedRequestException;
-import com.sun.xml.internal.ws.policy.privateutil.PolicyUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -44,7 +41,6 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ContentPrefix.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ContentPrefix.java
deleted file mode 100644
index 6885a8723..000000000
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ContentPrefix.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package com.spectralogic.ds3client.helpers;
-
-import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.models.common.CommonPrefixes;
-
-import java.util.List;
-
-/**
- * Because remoteListDirectory returns both a Contents and a CommonPrefixes
- * We need an object that can shuttle them around.
- */
-public class ContentPrefix {
-    private final Iterable contents;
-    private final List commonPrefixes;
-
-    ContentPrefix(final Iterable contents, final List commonPrefixes) {
-        this.commonPrefixes = commonPrefixes;
-        this.contents = contents;
-    }
-
-    public Iterable contents() { return contents; }
-    public List commonPrefixes() { return commonPrefixes; }
-
-
-}

From 5be6a2267120745512c36eb1fe7316a8da4ca86f Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Tue, 6 Jun 2017 13:29:07 -0600
Subject: [PATCH 036/229] Removing unused artifacts.

---
 .../ds3client/integration/Iterators_Test.java | 73 -------------------
 1 file changed, 73 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 2cb9642a6..3bd7e35dd 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -98,79 +98,6 @@ public void IterateWithDelimiter() throws IOException, URISyntaxException {
         }
     }
 
-    public void putString(String objectName, String objectContent) throws IOException {
-        final byte[] content = objectContent.getBytes();
-        final PutObjectRequest putObjectRequest = new PutObjectRequest(TEST_ENV_NAME, objectName, new ByteArraySeekableByteChannel(content), content.length);
-        CLIENT.putObject(putObjectRequest);
-    }
-
-    public void loopWithNextMarker(final String bucket, final String prefix, final String delimiter) throws IOException {
-        GetBucketRequest getBucketRequest = new GetBucketRequest(bucket).withPrefix(prefix).withDelimiter(delimiter).withMaxKeys(3);
-        GetBucketResponse clientBucket = CLIENT.getBucket(getBucketRequest);
-        ListBucketResult listBucketResult = clientBucket.getListBucketResult();
-        logPrefix(listBucketResult);
-        logObjects(listBucketResult);
-        final String nextMarker = listBucketResult.getNextMarker();
-        System.out.println("next marker " + nextMarker);
-        loopWithNextMarker(bucket,prefix,delimiter,nextMarker);
-    }
-
-    public void loopWithNextMarker(final String bucket, final String prefix, final String delimiter, final String marker) throws IOException {
-        if (marker == null) {
-            return;
-        }
-        GetBucketRequest getBucketRequest = new GetBucketRequest(bucket).withPrefix(prefix).withDelimiter(delimiter).withMaxKeys(3).withMarker(marker);
-        GetBucketResponse clientBucket = CLIENT.getBucket(getBucketRequest);
-        ListBucketResult listBucketResult = clientBucket.getListBucketResult();
-        logPrefix(listBucketResult);
-        logObjects(listBucketResult);
-        final String nextMarker = listBucketResult.getNextMarker();
-        System.out.println("next marker " + nextMarker);
-        loopWithNextMarker(bucket,prefix,delimiter,nextMarker);
-    }
-
-    public void logPrefix(ListBucketResult listBucketResult) {
-        for (CommonPrefixes commonPrefixes : listBucketResult.getCommonPrefixes()) {
-            System.out.println("Prefix: " + commonPrefixes.getPrefix());
-        }
-    }
-
-    public void logObjects(ListBucketResult listBucketResult) {
-        for (Contents contents : listBucketResult.getObjects()) {
-            System.out.println("Object: " + contents.getKey());
-        }
-    }
-
-    @Test
-    public void matchS3() throws IOException {
-        final String[] names = {
-                "dir/a.txt",
-                "dir/foo.txt",
-                "dir/foo1.txt",
-                "dir/foo2.txt",
-                "dir/baz/foo.txt",
-                "dir/tar/foo.txt",
-                "dir/1/foo.txt",
-                "dir/2/foo.txt",
-                "dir/3/foo.txt",
-                "dir/4/foo.txt",
-
-        };
-        final String content = "content";
-        try {
-            HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
-            for(final String s : names) {
-                putString(s, content);
-            }
-            putString("dir/4/", "");
-            loopWithNextMarker(TEST_ENV_NAME,"dir/", "/");
-            assertThat(true, is(true));
-
-        } finally {
-            deleteAllContents(CLIENT, TEST_ENV_NAME);
-        }
-    }
-
     @Test
     public void emptyGetBucket() throws IOException {
         final String prefix = "";

From 1c1ef28d5ab43e9c2e36a1c543e61d506d503275 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 6 Jun 2017 17:21:41 -0600
Subject: [PATCH 037/229] Best effort

---
 .../integration/GetJobManagement_Test.java    |   1 -
 .../helpers/Ds3ClientHelpersImpl.java         |   3 +-
 .../ds3client/helpers/JobPart.java            |  12 +-
 .../ds3client/helpers/JobState.java           | 103 ++++++++++++-
 .../GetSequentialBlobStrategy.java            |   2 -
 .../AbstractTransferStrategy.java             |  59 ++++---
 .../GetJobNetworkFailureRetryDecorator.java   |   2 +-
 .../GetJobPartialBlobTransferMethod.java      |   2 +-
 .../GetJobTransferMethod.java                 |   6 +-
 .../PutJobTransferMethod.java                 |   4 +-
 .../helpers/Ds3ClientHelpers_Test.java        |   7 +-
 .../TransferStrategy_Test.java                | 145 ++++++++++++++++++
 12 files changed, 296 insertions(+), 50 deletions(-)
 create mode 100644 ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
index 3b659018d..17b40324d 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
@@ -728,7 +728,6 @@ public void dataTransferred(final long size) {
         }
     }
 
-
     @Test
     public void testFiringFailureHandlerWhenGettingChunks()
             throws URISyntaxException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, IOException
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 8e91be789..d9dd9835f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -244,7 +244,8 @@ private Ds3ClientHelpers.Job innerStartReadJob(final List objects,
 
         final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList();
 
-        transferStrategyBuilder.withMasterObjectList(masterObjectList)
+        transferStrategyBuilder
+                .withMasterObjectList(masterObjectList)
                 .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), partialRanges));
 
         return new ReadJobImpl(transferStrategyBuilder);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java
index 3055e9f21..0096b698a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPart.java
@@ -23,26 +23,26 @@
  */
 public class JobPart {
     private final Ds3Client client;
-    private final BulkObject bulkObject;
+    private final BulkObject blob;
 
-    public JobPart(final Ds3Client client, final BulkObject bulkObject) {
+    public JobPart(final Ds3Client client, final BulkObject blob) {
         this.client = client;
-        this.bulkObject = bulkObject;
+        this.blob = blob;
     }
 
     public Ds3Client getClient() {
         return client;
     }
 
-    public BulkObject getBulkObject() {
-        return bulkObject;
+    public BulkObject getBlob() {
+        return blob;
     }
 
     @Override
     public String toString() {
         return "JobPart{" +
                 "client=" + client +
-                ", bulkObject=" + bulkObject +
+                ", blob=" + blob +
                 '}';
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
index 47479f4e2..96a312e35 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
@@ -15,27 +15,114 @@
 
 package com.spectralogic.ds3client.helpers;
 
+import com.google.common.collect.Sets;
 import com.spectralogic.ds3client.models.BulkObject;
 import com.spectralogic.ds3client.models.Objects;
+import com.spectralogic.ds3client.models.PhysicalPlacement;
 
 import java.util.Collection;
+import java.util.Set;
+import java.util.UUID;
 
 public class JobState {
-    final int numBlobsToTransfer;
+    final Set blobs = Sets.newConcurrentHashSet();
 
     public JobState(final Collection chunksThatContainBlobs) {
-        int numBlobsInChunks = 0;
-
         for (final Objects chunk : chunksThatContainBlobs) {
-            for (final BulkObject ignored : chunk.getObjects()) {
-                ++numBlobsInChunks;
+            for (final BulkObject blob : chunk.getObjects()) {
+                blobs.add(new BlobIdentityDecorator(blob));
             }
         }
+    }
+
+    public void blobTransferredOrFailed(final BulkObject blob) {
+        blobs.remove(new BlobIdentityDecorator(blob));
+    }
+
+    public boolean hasMoreBlobsToTransfer() {
+        return blobs.size() > 0;
+    }
 
-        numBlobsToTransfer = numBlobsInChunks;
+    public boolean contains(final BulkObject blob) {
+        return blobs.contains(new BlobIdentityDecorator(blob));
     }
 
-    public int numBlobsToTransfer() {
-        return numBlobsToTransfer;
+    /**
+     * This class is used to know whether we have attempted to transfer a blob.  In getting the master object list from
+     * a call to either getBulkJobSpectraS3 or putBulkJobSpectraS3, the resulting blobs will have their inCache property
+     * marked false.  In the master object list returned from a call to either allocateChunk or
+     * getJobChunksReadyForClientProcessingSpectraS3, the resulting blobs may or may not have the same value for
+     * their inCache property.  Since the BlackPearl may continually send us a blob that will always fail to transfer,
+     * we check to see if a blob in the master object list we get during a transfer is in the master object list we
+     * get when initiating a job.  If not, we can assume that we have previously tried and failed to transfer that
+     * blob and skip it.
+     */
+    private static class BlobIdentityDecorator {
+        private final String bucket;
+
+        private final UUID id;
+
+        private final boolean latest;
+
+        private final long length;
+
+        private final String name;
+
+        private final long offset;
+
+        private final PhysicalPlacement physicalPlacement;
+
+        private final long version;
+
+        private BlobIdentityDecorator(final BulkObject blob) {
+            this.bucket = blob.getBucket();
+            this.id = blob.getId();
+            this.latest = blob.getLatest();
+            this.length = blob.getLength();
+            this.name = blob.getName();
+            this.offset = blob.getOffset();
+            this.physicalPlacement = blob.getPhysicalPlacement();
+            this.version = blob.getVersion();
+        }
+
+        @Override
+        public boolean equals(final Object o) {
+            if (this == o) return true;
+            if (!(o instanceof BlobIdentityDecorator)) return false;
+
+            final BlobIdentityDecorator that = (BlobIdentityDecorator) o;
+
+            if (latest != that.latest) return false;
+            if (length != that.length) return false;
+            if (offset != that.offset) return false;
+            if (version != that.version) return false;
+            if (bucket != null ? !bucket.equals(that.bucket) : that.bucket != null) return false;
+            if (id != null ? !id.equals(that.id) : that.id != null) return false;
+            if (name != null ? !name.equals(that.name) : that.name != null) return false;
+            return physicalPlacement != null ? physicalPlacement.equals(that.physicalPlacement) : that.physicalPlacement == null;
+        }
+
+        @Override
+        public int hashCode() {
+            int result = bucket != null ? bucket.hashCode() : 0;
+            result = 31 * result + (id != null ? id.hashCode() : 0);
+            result = 31 * result + (latest ? 1 : 0);
+            result = 31 * result + (int) (length ^ (length >>> 32));
+            result = 31 * result + (name != null ? name.hashCode() : 0);
+            result = 31 * result + (int) (offset ^ (offset >>> 32));
+            result = 31 * result + (physicalPlacement != null ? physicalPlacement.hashCode() : 0);
+            result = 31 * result + (int) (version ^ (version >>> 32));
+            return result;
+        }
+
+        @Override
+        public String toString() {
+            return "BlobIdentityDecorator{" +
+                    "bucket='" + bucket + '\'' +
+                    ", length=" + length +
+                    ", name='" + name + '\'' +
+                    ", offset=" + offset +
+                    '}';
+        }
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java
index e8c329b0a..a8f6ff9c3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java
@@ -35,8 +35,6 @@
  * A subclass of {@link BlobStrategy} used in get transfers.
  */
 public class GetSequentialBlobStrategy extends AbstractBlobStrategy {
-    private static final Logger LOG = LoggerFactory.getLogger(GetSequentialBlobStrategy.class);
-
     public GetSequentialBlobStrategy(final Ds3Client client,
                                      final MasterObjectList masterObjectList,
                                      final EventDispatcher eventDispatcher,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index d16130c00..0419a075b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -31,9 +31,9 @@
 import java.io.IOException;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import com.spectralogic.ds3client.networking.FailedRequestException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,11 +45,11 @@ abstract class AbstractTransferStrategy implements TransferStrategy {
     private static final Logger LOG = LoggerFactory.getLogger(AbstractTransferStrategy.class);
 
     private final BlobStrategy blobStrategy;
+    private final JobState jobState;
     private final ListeningExecutorService executorService;
     private final EventDispatcher eventDispatcher;
     private final MasterObjectList masterObjectList;
     private final FailureEvent.FailureActivity failureActivity;
-    private final AtomicInteger totalNumBlobsToTransfer;
 
     private final AtomicReference cachedException = new AtomicReference<>();
 
@@ -77,7 +77,7 @@ public AbstractTransferStrategy(final BlobStrategy blobStrategy,
                                     final FailureEvent.FailureActivity failureActivity)
     {
         this.blobStrategy = blobStrategy;
-        totalNumBlobsToTransfer = new AtomicInteger(jobState.numBlobsToTransfer());
+        this.jobState = jobState;
         this.executorService = executorService;
         this.eventDispatcher = eventDispatcher;
         this.masterObjectList = masterObjectList;
@@ -103,20 +103,20 @@ public AbstractTransferStrategy withTransferMethod(final TransferMethod transfer
     public void transfer() throws IOException {
         cachedException.set(null);
 
-        while (totalNumBlobsToTransfer.get() > 0 && ! Thread.currentThread().isInterrupted()) {
+        while (jobState.hasMoreBlobsToTransfer() && ! Thread.currentThread().isInterrupted()) {
             try {
                 final Iterable jobParts = blobStrategy.getWork();
                 final CountDownLatch countDownLatch = new CountDownLatch(Iterables.size(jobParts));
                 transferJobParts(jobParts, countDownLatch);
                 countDownLatch.await();
-            } catch (final Ds3NoMoreRetriesException e) {
+            } catch (final Ds3NoMoreRetriesException | FailedRequestException e) {
+                LOG.info("---> Caught FailedRequestException");
                 emitFailureEvent(makeFailureEvent(failureActivity, e, firstChunk()));
                 throw e;
             } catch (final InterruptedException e) {
                 LOG.info("Error getting entries from work queue.", e);
                 Thread.currentThread().interrupt();
             } catch (final Throwable t) {
-                totalNumBlobsToTransfer.decrementAndGet();
                 emitFailureAndSetCachedException(t);
             }
         }
@@ -143,27 +143,40 @@ private synchronized void maybeSetCachedException(final Throwable t) {
 
     private void transferJobParts(final Iterable jobParts, final CountDownLatch countDownLatch) throws IOException {
         for (final JobPart jobPart : jobParts) {
-            executorService.submit(new Callable() {
-                @Override
-                public Void call() throws Exception {
-                    try {
-                        transferMethod.transferJobPart(jobPart);
-                    } catch (final RuntimeException e) {
-                        emitFailureAndSetCachedException(e);
-                        throw e;
-                    } catch (final Exception e) {
-                        emitFailureAndSetCachedException(e);
-                        throw new RuntimeException(e);
-                    } finally {
-                        totalNumBlobsToTransfer.decrementAndGet();
-                        countDownLatch.countDown();
-                        return null;
+            if ( ! jobState.contains(jobPart.getBlob())) {
+                LOG.info("---> Could not find: {}", jobPart.getBlob());
+                countDownLatch.countDown();
+            } else {
+                executorService.submit(new Callable() {
+                    @Override
+                    public Void call() throws Exception {
+                        try {
+                            transferMethod.transferJobPart(jobPart);
+                        } catch (final RuntimeException e) {
+                            LOG.error("---> Caught RuntimeException", e);
+                            emitFailureAndSetCachedException(e);
+                            throw e;
+                        } catch (final Exception e) {
+                            LOG.error("---> Caught Exception", e);
+                            emitFailureAndSetCachedException(e);
+                            throw new RuntimeException(e);
+                        } finally {
+                            LOG.info("---> Transferred: {}", jobPart.getBlob());
+                            jobState.blobTransferredOrFailed(jobPart.getBlob());
+                            countDownLatch.countDown();
+                            return null;
+                        }
                     }
-                }
-            });
+                });
+            }
         }
     }
 
+    private void handleJobParrtProcessed(final CountDownLatch countDownLatch, final JobPart jobPart) {
+        jobState.blobTransferredOrFailed(jobPart.getBlob());
+        countDownLatch.countDown();
+    }
+
     private void emitFailureEvent(final FailureEvent failureEvent) {
         eventDispatcher.emitFailureEvent(failureEvent);
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java
index 9197ed700..b80ca778c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java
@@ -96,7 +96,7 @@ private TransferState makeInitialTransferState() {
     private synchronized void makeTransferMethodForPartialRetry(final JobPart jobPart, final long numBytesTransferred) {
         final TransferState existingTransferState = transferMethodMap.get(jobPart);
 
-        ImmutableCollection ranges = initializeRanges(jobPart.getBulkObject(), existingTransferState);
+        ImmutableCollection ranges = initializeRanges(jobPart.getBlob(), existingTransferState);
         final Long numBytesToTransfer = initializeNumBytesToTransfer(existingTransferState, ranges);
         ranges = updateRanges(ranges, numBytesTransferred, numBytesToTransfer);
         final long destinationChannelOffset = existingTransferState.getDestinationChannelOffset() + numBytesTransferred;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
index abe6e5c8d..dc8bc3b56 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
@@ -65,7 +65,7 @@ public GetJobPartialBlobTransferMethod(final ChannelStrategy channelStrategy,
      */
     @Override
     public void transferJobPart(final JobPart jobPart) throws IOException {
-        final BulkObject blob = jobPart.getBulkObject();
+        final BulkObject blob = jobPart.getBlob();
 
         final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(blob);
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
index caac43a3c..3a174145d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
@@ -61,11 +61,11 @@ public GetJobTransferMethod(final ChannelStrategy channelStrategy,
      */
     @Override
     public void transferJobPart(final JobPart jobPart) throws IOException {
-        final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(jobPart.getBulkObject());
+        final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(jobPart.getBlob());
 
         final GetObjectResponse getObjectResponse = jobPart.getClient().getObject(makeGetObjectRequest(seekableByteChannel, jobPart));
 
-        final BulkObject blob = jobPart.getBulkObject();
+        final BulkObject blob = jobPart.getBlob();
 
         channelStrategy.releaseChannelForBlob(seekableByteChannel, blob);
 
@@ -77,7 +77,7 @@ public void transferJobPart(final JobPart jobPart) throws IOException {
     }
 
     private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekableByteChannel, final JobPart jobPart) {
-        final BulkObject blob = jobPart.getBulkObject();
+        final BulkObject blob = jobPart.getBlob();
 
         final GetObjectRequest getObjectRequest = new GetObjectRequest(
                 bucketName,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
index 753f5ee9a..f7eb5eb63 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
@@ -71,7 +71,7 @@ public PutJobTransferMethod(final ChannelStrategy channelStrategy,
      */
     @Override
     public void transferJobPart(final JobPart jobPart) throws IOException {
-        final BulkObject blob = jobPart.getBulkObject();
+        final BulkObject blob = jobPart.getBlob();
 
         final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(blob);
 
@@ -84,7 +84,7 @@ public void transferJobPart(final JobPart jobPart) throws IOException {
     }
 
     private PutObjectRequest makePutObjectRequest(final SeekableByteChannel seekableByteChannel, final JobPart jobPart) {
-        final BulkObject blob = jobPart.getBulkObject();
+        final BulkObject blob = jobPart.getBlob();
 
         final PutObjectRequest putObjectRequest = new PutObjectRequest(
                 bucketName,
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index d5627a1f6..d9fba8537 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -112,8 +112,11 @@ public void testReadObjectsWithFailedGet() throws IOException, ParseException {
         final GetBulkJobSpectraS3Response buildBulkGetResponse = buildBulkGetResponse();
         Mockito.when(ds3Client.getBulkJobSpectraS3(hasChunkOrdering(JobChunkClientProcessingOrderGuarantee.NONE))).thenReturn(buildBulkGetResponse);
 
-        final GetJobChunksReadyForClientProcessingSpectraS3Response jobChunksResponse = buildJobChunksResponse2();
-        Mockito.when(ds3Client.getJobChunksReadyForClientProcessingSpectraS3(hasJobId(jobId))).thenReturn(jobChunksResponse);
+        final GetJobChunksReadyForClientProcessingSpectraS3Response jobChunksResponse2 = buildJobChunksResponse2();
+        final GetJobChunksReadyForClientProcessingSpectraS3Response jobChunksResponse3 = buildJobChunksResponse3();
+        Mockito.when(ds3Client.getJobChunksReadyForClientProcessingSpectraS3(hasJobId(jobId)))
+                .thenReturn(jobChunksResponse2)
+                .thenReturn(jobChunksResponse3);
 
         Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "foo", jobId, 6))).thenThrow(new StubException());
         Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "baz", jobId, 6))).then(getObjectAnswer("ntents"));
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java
new file mode 100644
index 000000000..4580f9bac
--- /dev/null
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java
@@ -0,0 +1,145 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
+
+import com.google.common.collect.Sets;
+import com.spectralogic.ds3client.helpers.JobPart;
+import com.spectralogic.ds3client.helpers.JobState;
+import com.spectralogic.ds3client.helpers.events.FailureEvent;
+import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
+import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy;
+import com.spectralogic.ds3client.models.BulkObject;
+import com.spectralogic.ds3client.models.MasterObjectList;
+import com.spectralogic.ds3client.models.Objects;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TransferStrategy_Test {
+    private static final String BLOB1 = "blob1";
+    private static final String BLOB2 = "blob2";
+    private static final String BLOB3 = "blob3";
+    private static final String BLOB4 = "blob4";
+
+    @Test
+    public void testContinuallyGettingBlobThatFailsToTransfer() throws IOException {
+        final Set blobsTransferredSuccessfully = Sets.newConcurrentHashSet();
+        final Set blobsThatFailed = Sets.newConcurrentHashSet();
+        final AtomicInteger numTimesTransferCalled = new AtomicInteger(0);
+
+        new MultiThreadedTransferStrategy(new ContinuallyReturnFailingBlobStrategy(),
+                new JobState(chunksWithBlobs()),
+                10,
+                new EventDispatcherImpl(new SameThreadEventRunner()),
+                masterObjectList(),
+                FailureEvent.FailureActivity.GettingObject)
+                .withTransferMethod(transferMethod(blobsTransferredSuccessfully, blobsThatFailed, numTimesTransferCalled))
+                .transfer();
+
+        assertEquals(1, blobsThatFailed.size());
+        assertTrue(blobsThatFailed.contains(BLOB2));
+        assertEquals(3, blobsTransferredSuccessfully.size());
+        assertTrue(blobsTransferredSuccessfully.contains(BLOB1));
+        assertTrue(blobsTransferredSuccessfully.contains(BLOB3));
+        assertTrue(blobsTransferredSuccessfully.contains(BLOB4));
+        assertEquals(4, numTimesTransferCalled.get());
+    }
+
+    private class ContinuallyReturnFailingBlobStrategy implements BlobStrategy {
+        private final AtomicBoolean getWorkCalledAlready = new AtomicBoolean(false);
+
+        @Override
+        public Iterable getWork() throws IOException, InterruptedException {
+            if (getWorkCalledAlready.compareAndSet(false, true)) {
+                return Arrays.asList(firstChunkWithFailingBlob());
+            }
+
+            return Arrays.asList(remainingChunkWithFailingBlob());
+        }
+
+        private JobPart[] firstChunkWithFailingBlob() {
+            final BulkObject blob1 = new BulkObject(); blob1.setName(BLOB1);
+            final BulkObject blob2 = new BulkObject(); blob2.setName(BLOB2);
+
+            return new JobPart[] {
+                    new JobPart(null, blob1),
+                    new JobPart(null, blob2)
+            };
+        }
+
+        private JobPart[] remainingChunkWithFailingBlob() {
+            final BulkObject blob2 = new BulkObject(); blob2.setName(BLOB2);
+            final BulkObject blob3 = new BulkObject(); blob3.setName(BLOB3);
+            final BulkObject blob4 = new BulkObject(); blob4.setName(BLOB4);
+
+            return new JobPart[] {
+                    new JobPart(null, blob2),
+                    new JobPart(null, blob3),
+                    new JobPart(null, blob4)
+            };
+        }
+    }
+
+    private List chunksWithBlobs() {
+        final BulkObject blob1 = new BulkObject(); blob1.setName(BLOB1);
+        final BulkObject blob2 = new BulkObject(); blob2.setName(BLOB2);
+        final BulkObject blob3 = new BulkObject(); blob3.setName(BLOB3);
+        final BulkObject blob4 = new BulkObject(); blob4.setName(BLOB4);
+
+        final Objects chunk1 = new Objects();
+        chunk1.setObjects(Arrays.asList(blob1, blob2));
+
+        final Objects chunk2 = new Objects();
+        chunk2.setObjects(Arrays.asList(blob3, blob4));
+
+        return Arrays.asList(chunk1, chunk2);
+    }
+
+    private MasterObjectList masterObjectList() {
+        final MasterObjectList masterObjectList = new MasterObjectList();
+        masterObjectList.setObjects(chunksWithBlobs());
+
+        return masterObjectList;
+    }
+
+    private TransferMethod transferMethod(final Set blobsTransferredSuccessfully,
+                                          final Set blobsThatFailed,
+                                          final AtomicInteger numTimesTransferCalled)
+    {
+        return new TransferMethod() {
+            @Override
+            public void transferJobPart(final JobPart jobPart) throws IOException {
+                numTimesTransferCalled.incrementAndGet();
+
+                if (jobPart.getBlob().getName().equals(BLOB2)) {
+                    blobsThatFailed.add(jobPart.getBlob().getName());
+                    throw new IOException(BLOB2);
+                } else {
+                    blobsTransferredSuccessfully.add(jobPart.getBlob().getName());
+                }
+            }
+        };
+    }
+}

From ae8046963664d1bf56e19f9e8b2bbda6efee532c Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 11:04:34 -0600
Subject: [PATCH 038/229] The final frontier.

---
 .../ds3client/integration/Iterators_Test.java |  8 ++--
 .../test/helpers/TempStorageIds.java          |  4 +-
 .../AbstractPaginationResponse.java           |  4 +-
 .../ds3client/helpers/Ds3ClientHelpers.java   | 20 ++++++++-
 .../helpers/Ds3ClientHelpersImpl.java         | 42 +++++++++++++++++--
 .../pagination/GetBucketKeyLoader.java        | 33 +++++++--------
 .../pagination/GetBucketKeyLoaderFactory.java | 17 ++++++--
 .../ds3client/utils/PropertyUtils.java        |  2 +-
 .../helpers/Ds3ClientHelpers_Test.java        | 31 +++++++-------
 9 files changed, 109 insertions(+), 52 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 3bd7e35dd..955397e31 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -84,13 +84,13 @@ public void IterateWithDelimiter() throws IOException, URISyntaxException {
         try {
             HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/");
-            Iterable  fileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
-            List fileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
+            final Iterable  fileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
+            final List fileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
             assertThat(fileSystemKeys.size(), is(4));
 
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/more/");
-            Iterable  anotherFileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
-            List anotherFileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
+            final Iterable  anotherFileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
+            final List anotherFileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
             assertThat(anotherFileSystemKeys.size(), is(5));
         }
         finally {
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageIds.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageIds.java
index 51ed48a2e..6b31ae3dd 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageIds.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageIds.java
@@ -23,8 +23,8 @@
  * These IDs are stored for teardown of testing environment.
  */
 public class TempStorageIds {
-    private UUID storageDomainMemberId;
-    private UUID dataPersistenceRuleId;
+    private final UUID storageDomainMemberId;
+    private final UUID dataPersistenceRuleId;
 
     public TempStorageIds(final UUID storageDomainMemberId, final UUID dataPersistenceRuleId) {
         this.storageDomainMemberId = storageDomainMemberId;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractPaginationResponse.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractPaginationResponse.java
index 84cc9e0c2..f555b2d30 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractPaginationResponse.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractPaginationResponse.java
@@ -19,8 +19,8 @@
 
 public abstract class AbstractPaginationResponse extends AbstractResponse implements PaginationResponse {
 
-    private Integer pagingTruncated;
-    private Integer pagingTotalResultCount;
+    private final Integer pagingTruncated;
+    private final Integer pagingTotalResultCount;
 
     public AbstractPaginationResponse(final Integer pagingTotalResultCount, final Integer pagingTruncated, final String checksum, final ChecksumType.Type checksumType) {
         super(checksum, checksumType);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index a1c8934d9..6e367e43c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -469,6 +469,24 @@ public abstract Iterable listObjects(final String bucket, final String
      */
     public abstract Iterable listObjectsForDirectory(final Path directory) throws IOException;
 
+
+    /**
+     * Defaults to the path delimiter of '/'
+     * Returns an Iterable of {@link FileSystemKey}
+     * @param bucket The bucket
+     * @param keyPrefix Limits the response to keys that begin with the specified prefix
+     */
+    public abstract Iterable remoteListDirectory(final String bucket, final String keyPrefix) throws IOException;
+
+    /**
+     * Defaults to the path delimiter of '/'
+     * Returns an Iterable of {@link FileSystemKey}
+     * @param bucket The bucket
+     * @param keyPrefix Limits the response to keys that begin with the specified prefix
+     * @param nextMarker Specifies the key to start with
+     */
+    public abstract Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker) throws IOException;
+
     /**
      * Defaults to the path delimiter of '/'
      * Returns an Iterable of {@link FileSystemKey}
@@ -476,7 +494,6 @@ public abstract Iterable listObjects(final String bucket, final String
      * @param keyPrefix Limits the response to keys that begin with the specified prefix
      * @param nextMarker Specifies the key to start with when listing objects
      * @param maxKeys Sets the maximum number of keys returned in the response body.
-     * @return {@link ContentPrefix}
      */
     public abstract Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException;
 
@@ -487,7 +504,6 @@ public abstract Iterable listObjects(final String bucket, final String
      * @param delimiter Specifies a path delimiter for the S3 query
      * @param nextMarker Specifies the key to start with when listing objects
      * @param maxKeys Sets the maximum number of keys returned in the response body.
-     * @return {@link ContentPrefix}
      */
     public abstract Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 8b6eedddb..ac9ffdce9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -36,6 +36,7 @@
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;
 import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers;
 import com.spectralogic.ds3client.models.*;
+import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.models.common.Range;
 import com.spectralogic.ds3client.models.bulk.*;
 import com.spectralogic.ds3client.networking.FailedRequestException;
@@ -66,6 +67,32 @@ class Ds3ClientHelpersImpl extends Ds3ClientHelpers {
     private final EventRunner eventRunner;
     private final FileSystemHelper fileSystemHelper;
 
+    private final Function> getFileSystemKeysFunction = new Function>() {
+        @Nullable
+        @Override
+        public ImmutableList apply(@Nullable final ListBucketResult result) {
+            final List contents = result.getObjects();
+            final List prefixes = result.getCommonPrefixes();
+            final ImmutableList.Builder fileSystemKeys = new ImmutableList.Builder<>();
+            for(final CommonPrefixes prefix: prefixes) {
+                fileSystemKeys.add(new FileSystemKey(prefix));
+            }
+            for(final Contents content : contents) {
+                fileSystemKeys.add(new FileSystemKey(content));
+            }
+            return fileSystemKeys.build();
+        }
+    };
+
+    private final Function> getContentsFunction = new Function>() {
+        @Nullable
+        @Override
+        public ImmutableList apply(@Nullable final ListBucketResult input) {
+            return ImmutableList.copyOf(input.getObjects());
+        }
+    };
+
+
     public Ds3ClientHelpersImpl(final Ds3Client client) {
         this(client, TransferStrategyBuilder.DEFAULT_CHUNK_ATTEMPT_RETRY_ATTEMPTS);
     }
@@ -512,7 +539,7 @@ public Iterable listObjects(final String bucket, final String keyPrefi
     @Override
     public Iterable listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys, final int retries) {
 
-        return new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries));
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries, getContentsFunction));
     }
 
     @Override
@@ -528,13 +555,22 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
         return objects.build();
     }
 
+    @Override
+    public Iterable remoteListDirectory(final String bucket, final String keyPrefix) throws IOException {
+        return this.remoteListDirectory(bucket, keyPrefix, null);
+    }
+
+    @Override
+    public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker) throws IOException {
+        return this.remoteListDirectory(bucket, keyPrefix, nextMarker,1000);
+    }
     @Override
     public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException {
-        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, "/", nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
+        return remoteListDirectory(bucket,keyPrefix,nextMarker, "/", maxKeys);
     }
 
     public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException {
-        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES, getFileSystemKeysFunction));
     }
 
     public Iterable addPrefixToDs3ObjectsList(final Iterable objectsList, final String prefix) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index 8032232e5..1a186ef35 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -1,5 +1,6 @@
 package com.spectralogic.ds3client.helpers.pagination;
 
+import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.GetBucketRequest;
@@ -17,7 +18,7 @@
 import java.util.Collections;
 import java.util.List;
 
-public class GetBucketKeyLoader implements LazyIterable.LazyLoader{
+public class GetBucketKeyLoader implements LazyIterable.LazyLoader{
     private static final int DEFAULT_MAX_KEYS = 1000;
 
     private final Ds3Client client;
@@ -26,35 +27,36 @@ public class GetBucketKeyLoader implements LazyIterable.LazyLoader> function;
     private String nextMarker;
     private boolean truncated;
     private boolean endOfInput = false;
 
-    public GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount) {
+    public GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount, final Function> function) {
         this.client = client;
         this.bucket = bucket;
         this.prefix = prefix;
         this.delimiter = delimiter;
         this.nextMarker = nextMarker;
-        this.maxKeys = Math.min(maxKeys,DEFAULT_MAX_KEYS);
+        this.maxKeys = Math.min(maxKeys, DEFAULT_MAX_KEYS);
         this.retryCount = retryCount;
         this.nextMarker = nextMarker;
         this.truncated  = nextMarker != null;
+        this.function = function;
     }
 
     @Override
-    public List getNextValues() {
+    public List getNextValues() {
         if(endOfInput) {
-            return Collections.emptyList();
+            return (List) Collections.emptyList();
         }
         int retryAttempt = 0;
         while(true) {
             final GetBucketRequest request = new GetBucketRequest(bucket);
             request.withMaxKeys(maxKeys);
-            if (prefix != null) request.withPrefix(prefix);
-            if (truncated)  request.withMarker(nextMarker);
-            if (delimiter != null) request.withDelimiter(delimiter);
+            if (prefix != null) { request.withPrefix(prefix); }
+            if (truncated) { request.withMarker(nextMarker); }
+            if (delimiter != null) { request.withDelimiter(delimiter); }
             GetBucketResponse response;
             try {
                 response = this.client.getBucket(request);
@@ -70,21 +72,14 @@ public List getNextValues() {
                 retryAttempt++;
             } else {
                 final ListBucketResult result = response.getListBucketResult();
+
                 truncated = result.getTruncated();
                 this.nextMarker = result.getNextMarker();
                 if (Guard.isStringNullOrEmpty(nextMarker) && !truncated) {
                     endOfInput = true;
                 }
-                final List contents = result.getObjects();
-                final List prefixes = result.getCommonPrefixes();
-                final ImmutableList.Builder fileSystemKeys = new ImmutableList.Builder<>();
-                for(CommonPrefixes prefix: prefixes) {
-                    fileSystemKeys.add(new FileSystemKey(prefix));
-                }
-                for(Contents content : contents) {
-                    fileSystemKeys.add(new FileSystemKey(content));
-                }
-                return fileSystemKeys.build();
+                return function.apply(result);
+
             }
         }
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
index 55abd7aae..9b52bbd64 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -1,10 +1,17 @@
 package com.spectralogic.ds3client.helpers.pagination;
 
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.models.Contents;
 import com.spectralogic.ds3client.models.FileSystemKey;
+import com.spectralogic.ds3client.models.ListBucketResult;
+import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 
-public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFactory {
+import java.util.List;
+
+public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFactory {
 
     private final Ds3Client client;
     private final String bucket;
@@ -13,8 +20,9 @@ public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFactory
     private final String delimiter;
     private final int maxKeys;
     private final int defaultListObjectsRetries;
+    private final Function> function;
 
-    public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries) {
+    public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries, final Function> function) {
         this.client = client;
         this.bucket = bucket;
         this.keyPrefix = keyPrefix;
@@ -22,10 +30,11 @@ public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, fi
         this.nextMarker = nextMarker;
         this.maxKeys = maxKeys;
         this.defaultListObjectsRetries = defaultListObjectsRetries;
+        this.function = function;
     }
 
     @Override
-    public LazyIterable.LazyLoader create() {
-        return new GetBucketKeyLoader(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, defaultListObjectsRetries);
+    public LazyIterable.LazyLoader create() {
+        return new GetBucketKeyLoader<>(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, defaultListObjectsRetries, function);
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
index c9fb83854..c12170f8d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
@@ -29,7 +29,7 @@ public final class PropertyUtils {
     private static final String SDK_VERSION_PROPERTY_NAME = "version";
     private static final String PROPERTIES_FILE_NAME = "ds3_sdk.properties";
 
-    private static AtomicReference versionProperty = new AtomicReference<>("");
+    private static final AtomicReference versionProperty = new AtomicReference<>("");
 
     private PropertyUtils() {}
 
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index f715ed751..09a45b97a 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -35,6 +35,7 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Matchers;
 import org.mockito.Mockito;
 
 import java.io.IOException;
@@ -108,7 +109,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException {
     public void testReadObjectsWithFailedGet() throws IOException, ParseException {
         final Ds3Client ds3Client = mock(Ds3Client.class);
 
-        Mockito.when(ds3Client.newForNode(Mockito.any())).thenReturn(ds3Client);
+        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
 
         final GetBulkJobSpectraS3Response buildBulkGetResponse = buildBulkGetResponse();
         Mockito.when(ds3Client.getBulkJobSpectraS3(hasChunkOrdering(JobChunkClientProcessingOrderGuarantee.NONE))).thenReturn(buildBulkGetResponse);
@@ -143,7 +144,7 @@ public void testWriteObjects() throws IOException, ParseException {
         final Ds3Client ds3Client = buildDs3ClientForBulk();
 
         final PutBulkJobSpectraS3Response bulkPutResponse = buildBulkPutResponse();
-        Mockito.when(ds3Client.putBulkJobSpectraS3(Mockito.any(PutBulkJobSpectraS3Request.class))).thenReturn(bulkPutResponse);
+        Mockito.when(ds3Client.putBulkJobSpectraS3(Matchers.any(PutBulkJobSpectraS3Request.class))).thenReturn(bulkPutResponse);
         
         final AllocateJobChunkSpectraS3Response allocateResponse1 = buildAllocateResponse1();
         final AllocateJobChunkSpectraS3Response allocateResponse2 = buildAllocateResponse2();
@@ -203,11 +204,11 @@ public void testWriteObjectsWithFailedPut() throws IOException, ParseException {
         final ConnectionDetails details = mock(ConnectionDetails.class);
         Mockito.when(details.getEndpoint()).thenReturn("localhost");
 
-        Mockito.when(ds3Client.newForNode(Mockito.any())).thenReturn(ds3Client);
+        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
         Mockito.when(ds3Client.getConnectionDetails()).thenReturn(details);
 
         final PutBulkJobSpectraS3Response buildBulkPutResponse = buildBulkPutResponse();
-        Mockito.when(ds3Client.putBulkJobSpectraS3(Mockito.any(PutBulkJobSpectraS3Request.class))).thenReturn(buildBulkPutResponse);
+        Mockito.when(ds3Client.putBulkJobSpectraS3(Matchers.any(PutBulkJobSpectraS3Request.class))).thenReturn(buildBulkPutResponse);
 
         final AllocateJobChunkSpectraS3Response allocateResponse2 = buildAllocateResponse2();
         final AllocateJobChunkSpectraS3Response allocateResponse3 = buildAllocateResponse3();
@@ -295,7 +296,7 @@ public void testRecoverWriteJob() throws IOException, JobRecoveryException, Pars
         final Ds3Client ds3Client = buildDs3ClientForBulk();
 
         final ModifyJobSpectraS3Response modifyWriteJobResponse = buildModifyWriteJobResponse();
-        Mockito.when(ds3Client.modifyJobSpectraS3(Mockito.any(ModifyJobSpectraS3Request.class))).thenReturn(modifyWriteJobResponse);
+        Mockito.when(ds3Client.modifyJobSpectraS3(Matchers.any(ModifyJobSpectraS3Request.class))).thenReturn(modifyWriteJobResponse);
 
         final Job job = Ds3ClientHelpers.wrap(ds3Client).recoverWriteJob(jobId);
         assertThat(job.getJobId(), is(jobId));
@@ -307,7 +308,7 @@ public void testRecoverWriteJobThrowsJobRecoveryExceptionForWrongRequestType() t
         final Ds3Client ds3Client = buildDs3ClientForBulk();
 
         final ModifyJobSpectraS3Response modifyReadJobResponse = buildModifyReadJobResponse();
-        Mockito.when(ds3Client.modifyJobSpectraS3(Mockito.any(ModifyJobSpectraS3Request.class))).thenReturn(modifyReadJobResponse);
+        Mockito.when(ds3Client.modifyJobSpectraS3(Matchers.any(ModifyJobSpectraS3Request.class))).thenReturn(modifyReadJobResponse);
 
         Ds3ClientHelpers.wrap(ds3Client).recoverWriteJob(jobId);
     }
@@ -317,7 +318,7 @@ public void testRecoverReadJob() throws IOException, JobRecoveryException, Parse
         final Ds3Client ds3Client = buildDs3ClientForBulk();
 
         final ModifyJobSpectraS3Response modifyReadJobResponse = buildModifyReadJobResponse();
-        Mockito.when(ds3Client.modifyJobSpectraS3(Mockito.any(ModifyJobSpectraS3Request.class))).thenReturn(modifyReadJobResponse);
+        Mockito.when(ds3Client.modifyJobSpectraS3(Matchers.any(ModifyJobSpectraS3Request.class))).thenReturn(modifyReadJobResponse);
 
         final Job job = Ds3ClientHelpers.wrap(ds3Client).recoverReadJob(jobId);
         assertThat(job.getJobId(), is(jobId));
@@ -329,7 +330,7 @@ public void testRecoverReadJobThrowsJobRecoveryExceptionForWrongRequestType() th
         final Ds3Client ds3Client = buildDs3ClientForBulk();
 
         final ModifyJobSpectraS3Response modifyWriteJobResponse = buildModifyWriteJobResponse();
-        Mockito.when(ds3Client.modifyJobSpectraS3(Mockito.any(ModifyJobSpectraS3Request.class))).thenReturn(modifyWriteJobResponse);
+        Mockito.when(ds3Client.modifyJobSpectraS3(Matchers.any(ModifyJobSpectraS3Request.class))).thenReturn(modifyWriteJobResponse);
 
         Ds3ClientHelpers.wrap(ds3Client).recoverReadJob(jobId);
     }
@@ -423,7 +424,7 @@ private static Ds3Client buildDs3ClientForBulk() throws IOException,
             .thenReturn(jobChunksResponse2)
             .thenReturn(jobChunksResponse3);
 
-        Mockito.when(ds3Client.newForNode(Mockito.any())).thenReturn(ds3Client);
+        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
         Mockito.when(ds3Client.getConnectionDetails()).thenReturn(details);
         return ds3Client;
     }
@@ -612,7 +613,7 @@ public void testWriteObjectsWithRetryAfter() throws IOException, ParseException
 
         final PutBulkJobSpectraS3Response bulkPutResponse = buildBulkPutResponse();
         Mockito.when(ds3Client
-                .putBulkJobSpectraS3(Mockito.any(PutBulkJobSpectraS3Request.class)))
+                .putBulkJobSpectraS3(Matchers.any(PutBulkJobSpectraS3Request.class)))
                 .thenReturn(bulkPutResponse);
 
         final AllocateJobChunkSpectraS3Response allocateResponse1 = buildAllocateResponse1();
@@ -674,22 +675,22 @@ public SeekableByteChannel buildChannel(final String key) throws IOException {
     public void testEnsureBucketExistsRace() throws IOException {
         final Ds3Client ds3Client = mock(Ds3Client.class);
         final HeadBucketResponse response = buildHeadBucketResponse(HeadBucketResponse.Status.DOESNTEXIST);
-        Mockito.when(ds3Client.headBucket(Mockito.any(HeadBucketRequest.class))).thenReturn(response);
-        Mockito.when(ds3Client.putBucket(Mockito.any(PutBucketRequest.class)))
+        Mockito.when(ds3Client.headBucket(Matchers.any(HeadBucketRequest.class))).thenReturn(response);
+        Mockito.when(ds3Client.putBucket(Matchers.any(PutBucketRequest.class)))
                 .thenThrow(new FailedRequestException(ImmutableList.of(202, 409), 409, new Error(), "Conflict", "5"));
 
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(ds3Client);
 
         helpers.ensureBucketExists("fake_bucket"); // if this throws an exception, then this test should fail
-        verify(ds3Client, atLeastOnce()).putBucket(Mockito.any(PutBucketRequest.class));
+        verify(ds3Client, atLeastOnce()).putBucket(Matchers.any(PutBucketRequest.class));
     }
 
     @Test(expected = FailedRequestException.class)
     public void testEnsureBucketExistsReturnsError() throws IOException {
         final Ds3Client ds3Client = mock(Ds3Client.class);
         final HeadBucketResponse response = buildHeadBucketResponse(HeadBucketResponse.Status.DOESNTEXIST);
-        Mockito.when(ds3Client.headBucket(Mockito.any(HeadBucketRequest.class))).thenReturn(response);
-        Mockito.when(ds3Client.putBucket(Mockito.any(PutBucketRequest.class)))
+        Mockito.when(ds3Client.headBucket(Matchers.any(HeadBucketRequest.class))).thenReturn(response);
+        Mockito.when(ds3Client.putBucket(Matchers.any(PutBucketRequest.class)))
                 .thenThrow(new FailedRequestException(ImmutableList.of(202, 409, 500), 500, new Error(), "Error", "5"));
 
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(ds3Client);

From 6feac6219536072a3c6e218756dd2d114e9155e3 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 7 Jun 2017 11:31:38 -0600
Subject: [PATCH 039/229] Take into account that the black pearl may
 continually send a chunk that contains a blob that will always fail to
 transfer.

---
 .../ds3client/helpers/JobState.java           | 19 ++--
 .../AbstractTransferStrategy.java             | 90 +++++++++++--------
 .../helpers/Ds3ClientHelpers_Test.java        |  7 +-
 3 files changed, 67 insertions(+), 49 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
index 96a312e35..5f9151abd 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
@@ -25,28 +25,35 @@
 import java.util.UUID;
 
 public class JobState {
-    final Set blobs = Sets.newConcurrentHashSet();
+    private final Set blobs = Sets.newConcurrentHashSet();
+
+    private final int numBlobsInJob;
 
     public JobState(final Collection chunksThatContainBlobs) {
+        int numBlobsInChunks = 0;
+
         for (final Objects chunk : chunksThatContainBlobs) {
             for (final BulkObject blob : chunk.getObjects()) {
                 blobs.add(new BlobIdentityDecorator(blob));
+                ++numBlobsInChunks;
             }
         }
-    }
 
-    public void blobTransferredOrFailed(final BulkObject blob) {
-        blobs.remove(new BlobIdentityDecorator(blob));
+        numBlobsInJob = numBlobsInChunks;
     }
 
-    public boolean hasMoreBlobsToTransfer() {
-        return blobs.size() > 0;
+    public boolean blobTransferredOrFailed(final BulkObject blob) {
+        return blobs.remove(new BlobIdentityDecorator(blob));
     }
 
     public boolean contains(final BulkObject blob) {
         return blobs.contains(new BlobIdentityDecorator(blob));
     }
 
+    public int numBlobsInJob() {
+        return numBlobsInJob;
+    }
+
     /**
      * This class is used to know whether we have attempted to transfer a blob.  In getting the master object list from
      * a call to either getBulkJobSpectraS3 or putBulkJobSpectraS3, the resulting blobs will have their inCache property
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index 0419a075b..ffb4d7b84 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -15,6 +15,8 @@
 
 package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
 
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.spectralogic.ds3client.Ds3Client;
@@ -29,14 +31,18 @@
 import com.spectralogic.ds3client.models.Objects;
 
 import java.io.IOException;
+import java.util.NoSuchElementException;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 /**
  * An implementation of {@link TransferStrategy} that provides a default implementation {@code transfer}
  * implementation.
@@ -103,20 +109,27 @@ public AbstractTransferStrategy withTransferMethod(final TransferMethod transfer
     public void transfer() throws IOException {
         cachedException.set(null);
 
-        while (jobState.hasMoreBlobsToTransfer() && ! Thread.currentThread().isInterrupted()) {
+        final AtomicInteger numBlobsRemaining = new AtomicInteger(jobState.numBlobsInJob());
+
+        while ( ! Thread.currentThread().isInterrupted() && numBlobsRemaining.get() > 0) {
             try {
-                final Iterable jobParts = blobStrategy.getWork();
+                final Iterable jobParts = jobPartsNotAlreadyTransferred();
+
+                final int numJobParts = Iterables.size(jobParts);
+
+                if (numJobParts <= 0) {
+                    break;
+                }
+
                 final CountDownLatch countDownLatch = new CountDownLatch(Iterables.size(jobParts));
-                transferJobParts(jobParts, countDownLatch);
+                transferJobParts(jobParts, countDownLatch, numBlobsRemaining);
                 countDownLatch.await();
             } catch (final Ds3NoMoreRetriesException | FailedRequestException e) {
-                LOG.info("---> Caught FailedRequestException");
                 emitFailureEvent(makeFailureEvent(failureActivity, e, firstChunk()));
                 throw e;
-            } catch (final InterruptedException e) {
-                LOG.info("Error getting entries from work queue.", e);
+            } catch (final InterruptedException | NoSuchElementException e) {
                 Thread.currentThread().interrupt();
-            } catch (final Throwable t) {
+            }  catch (final Throwable t) {
                 emitFailureAndSetCachedException(t);
             }
         }
@@ -126,6 +139,17 @@ public void transfer() throws IOException {
         }
     }
 
+    private Iterable jobPartsNotAlreadyTransferred() throws IOException, InterruptedException {
+        return FluentIterable
+                .from(blobStrategy.getWork())
+                .filter(new Predicate() {
+                    @Override
+                    public boolean apply(@Nullable final JobPart jobPart) {
+                        return jobState.contains(jobPart.getBlob());
+                    }
+                });
+    }
+
     private void emitFailureAndSetCachedException(final Throwable t) {
         emitFailureEvent(makeFailureEvent(failureActivity, t, firstChunk()));
         maybeSetCachedException(t);
@@ -141,42 +165,32 @@ private synchronized void maybeSetCachedException(final Throwable t) {
         }
     }
 
-    private void transferJobParts(final Iterable jobParts, final CountDownLatch countDownLatch) throws IOException {
+    private void transferJobParts(final Iterable jobParts,
+                                  final CountDownLatch countDownLatch,
+                                  final AtomicInteger numBlobsTransferred) throws IOException {
         for (final JobPart jobPart : jobParts) {
-            if ( ! jobState.contains(jobPart.getBlob())) {
-                LOG.info("---> Could not find: {}", jobPart.getBlob());
-                countDownLatch.countDown();
-            } else {
-                executorService.submit(new Callable() {
-                    @Override
-                    public Void call() throws Exception {
-                        try {
-                            transferMethod.transferJobPart(jobPart);
-                        } catch (final RuntimeException e) {
-                            LOG.error("---> Caught RuntimeException", e);
-                            emitFailureAndSetCachedException(e);
-                            throw e;
-                        } catch (final Exception e) {
-                            LOG.error("---> Caught Exception", e);
-                            emitFailureAndSetCachedException(e);
-                            throw new RuntimeException(e);
-                        } finally {
-                            LOG.info("---> Transferred: {}", jobPart.getBlob());
-                            jobState.blobTransferredOrFailed(jobPart.getBlob());
-                            countDownLatch.countDown();
-                            return null;
-                        }
+            executorService.submit(new Callable() {
+                @Override
+                public Void call() throws Exception {
+                    try {
+                        transferMethod.transferJobPart(jobPart);
+                    } catch (final RuntimeException e) {
+                        emitFailureAndSetCachedException(e);
+                        throw e;
+                    } catch (final Exception e) {
+                        emitFailureAndSetCachedException(e);
+                        throw new RuntimeException(e);
+                    } finally {
+                        jobState.blobTransferredOrFailed(jobPart.getBlob());
+                        numBlobsTransferred.decrementAndGet();
+                        countDownLatch.countDown();
+                        return null;
                     }
-                });
-            }
+                }
+            });
         }
     }
 
-    private void handleJobParrtProcessed(final CountDownLatch countDownLatch, final JobPart jobPart) {
-        jobState.blobTransferredOrFailed(jobPart.getBlob());
-        countDownLatch.countDown();
-    }
-
     private void emitFailureEvent(final FailureEvent failureEvent) {
         eventDispatcher.emitFailureEvent(failureEvent);
     }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index d9fba8537..d5627a1f6 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -112,11 +112,8 @@ public void testReadObjectsWithFailedGet() throws IOException, ParseException {
         final GetBulkJobSpectraS3Response buildBulkGetResponse = buildBulkGetResponse();
         Mockito.when(ds3Client.getBulkJobSpectraS3(hasChunkOrdering(JobChunkClientProcessingOrderGuarantee.NONE))).thenReturn(buildBulkGetResponse);
 
-        final GetJobChunksReadyForClientProcessingSpectraS3Response jobChunksResponse2 = buildJobChunksResponse2();
-        final GetJobChunksReadyForClientProcessingSpectraS3Response jobChunksResponse3 = buildJobChunksResponse3();
-        Mockito.when(ds3Client.getJobChunksReadyForClientProcessingSpectraS3(hasJobId(jobId)))
-                .thenReturn(jobChunksResponse2)
-                .thenReturn(jobChunksResponse3);
+        final GetJobChunksReadyForClientProcessingSpectraS3Response jobChunksResponse = buildJobChunksResponse2();
+        Mockito.when(ds3Client.getJobChunksReadyForClientProcessingSpectraS3(hasJobId(jobId))).thenReturn(jobChunksResponse);
 
         Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "foo", jobId, 6))).thenThrow(new StubException());
         Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "baz", jobId, 6))).then(getObjectAnswer("ntents"));

From ed330c17970f29160c89eb2d43f3e462247bad9a Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 12:06:05 -0600
Subject: [PATCH 040/229] Corrected a method definition error.

---
 .../ds3client/integration/Iterators_Test.java             | 7 +++++++
 .../ds3client/helpers/Ds3ClientHelpersImpl.java           | 8 +++-----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 955397e31..4306042e5 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -91,6 +91,13 @@ public void IterateWithDelimiter() throws IOException, URISyntaxException {
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/more/");
             final Iterable  anotherFileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
             final List anotherFileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
+            for(final FileSystemKey key : anotherFileSystemKeys) {
+                if(key.isContents()) {
+                    System.out.println("Object: " + key.toContents().getKey());
+                } else {
+                    System.out.println("Prefix: " + key.toCommonPrefixes().getPrefix());
+                }
+            }
             assertThat(anotherFileSystemKeys.size(), is(5));
         }
         finally {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index ac9ffdce9..520df8146 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -71,13 +71,11 @@ class Ds3ClientHelpersImpl extends Ds3ClientHelpers {
         @Nullable
         @Override
         public ImmutableList apply(@Nullable final ListBucketResult result) {
-            final List contents = result.getObjects();
-            final List prefixes = result.getCommonPrefixes();
             final ImmutableList.Builder fileSystemKeys = new ImmutableList.Builder<>();
-            for(final CommonPrefixes prefix: prefixes) {
+            for(final CommonPrefixes prefix: result.getCommonPrefixes()) {
                 fileSystemKeys.add(new FileSystemKey(prefix));
             }
-            for(final Contents content : contents) {
+            for(final Contents content : result.getObjects()) {
                 fileSystemKeys.add(new FileSystemKey(content));
             }
             return fileSystemKeys.build();
@@ -566,7 +564,7 @@ public Iterable remoteListDirectory(final String bucket, final St
     }
     @Override
     public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) throws IOException {
-        return remoteListDirectory(bucket,keyPrefix,nextMarker, "/", maxKeys);
+        return remoteListDirectory(bucket,keyPrefix,"/", keyPrefix, maxKeys);
     }
 
     public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException {

From 7f7ba7e3f26b89e1ca1eb20de32d162d902520de Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 7 Jun 2017 12:17:08 -0600
Subject: [PATCH 041/229] Removing redundant call to Iterables.size()

---
 .../strategy/transferstrategy/AbstractTransferStrategy.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index ffb4d7b84..f42a195a4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -121,7 +121,7 @@ public void transfer() throws IOException {
                     break;
                 }
 
-                final CountDownLatch countDownLatch = new CountDownLatch(Iterables.size(jobParts));
+                final CountDownLatch countDownLatch = new CountDownLatch(numJobParts);
                 transferJobParts(jobParts, countDownLatch, numBlobsRemaining);
                 countDownLatch.await();
             } catch (final Ds3NoMoreRetriesException | FailedRequestException e) {

From 929f739ff4954ad97f33fd9bd0d348528db5b423 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 7 Jun 2017 13:17:33 -0600
Subject: [PATCH 042/229] Returning Objects.hash() from
 BlobIdentityDecorator.hashCode()

---
 .../com/spectralogic/ds3client/helpers/JobState.java   | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
index 5f9151abd..f4abf11d7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
@@ -111,15 +111,7 @@ public boolean equals(final Object o) {
 
         @Override
         public int hashCode() {
-            int result = bucket != null ? bucket.hashCode() : 0;
-            result = 31 * result + (id != null ? id.hashCode() : 0);
-            result = 31 * result + (latest ? 1 : 0);
-            result = 31 * result + (int) (length ^ (length >>> 32));
-            result = 31 * result + (name != null ? name.hashCode() : 0);
-            result = 31 * result + (int) (offset ^ (offset >>> 32));
-            result = 31 * result + (physicalPlacement != null ? physicalPlacement.hashCode() : 0);
-            result = 31 * result + (int) (version ^ (version >>> 32));
-            return result;
+            return java.util.Objects.hash(bucket, id, latest, length, name, offset, physicalPlacement, version);
         }
 
         @Override

From 746bcb1eb06b47053c9d05f1d1a6c9d87db6536a Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 15:19:15 -0600
Subject: [PATCH 043/229] Working on getting rid of copies

---
 .../ds3client/integration/Iterators_Test.java |  36 ++----
 .../ds3client/helpers/Ds3ClientHelpers.java   |   1 -
 .../helpers/Ds3ClientHelpersImpl.java         |  42 ++-----
 .../pagination/GetBucketKeyLoader.java        |  27 +++--
 .../pagination/GetBucketKeyLoaderFactory.java |  52 ++++++++-
 .../helpers/pagination/GetBucketLoader.java   | 103 ------------------
 .../pagination/GetBucketLoaderFactory.java    |  47 --------
 7 files changed, 87 insertions(+), 221 deletions(-)
 delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java
 delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 4306042e5..98198e474 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -17,22 +17,16 @@
 
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.GetBucketRequest;
-import com.spectralogic.ds3client.commands.GetBucketResponse;
-import com.spectralogic.ds3client.commands.PutObjectRequest;
-import com.spectralogic.ds3client.helpers.pagination.GetBucketLoaderFactory;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
 import com.spectralogic.ds3client.helpers.pagination.GetObjectsFullDetailsLoaderFactory;
-import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.models.FileSystemKey;
-import com.spectralogic.ds3client.models.ListBucketResult;
-import com.spectralogic.ds3client.models.common.CommonPrefixes;
-import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
-import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
 import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.FileSystemKey;
 import com.spectralogic.ds3client.networking.FailedRequestException;
+import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -45,9 +39,7 @@
 import java.util.List;
 import java.util.UUID;
 
-import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
-import static com.spectralogic.ds3client.integration.Util.loadBookTestData;
-import static com.spectralogic.ds3client.integration.Util.loadBookTestDataWithPrefix;
+import static com.spectralogic.ds3client.integration.Util.*;
 import static org.hamcrest.CoreMatchers.*;
 import static org.junit.Assert.*;
 
@@ -91,16 +83,8 @@ public void IterateWithDelimiter() throws IOException, URISyntaxException {
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/more/");
             final Iterable  anotherFileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
             final List anotherFileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
-            for(final FileSystemKey key : anotherFileSystemKeys) {
-                if(key.isContents()) {
-                    System.out.println("Object: " + key.toContents().getKey());
-                } else {
-                    System.out.println("Prefix: " + key.toCommonPrefixes().getPrefix());
-                }
-            }
             assertThat(anotherFileSystemKeys.size(), is(5));
-        }
-        finally {
+        } finally {
             deleteAllContents(CLIENT, TEST_ENV_NAME);
         }
     }
@@ -111,7 +95,7 @@ public void emptyGetBucket() throws IOException {
         final String nextMarker = null;
         final int maxKeys = 100;
 
-        emptyTest(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, TEST_DELIMITER, nextMarker, maxKeys, RETRIES));
+        emptyTest(new GetBucketKeyLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, TEST_DELIMITER, nextMarker, maxKeys, RETRIES, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
     @Test
@@ -119,7 +103,7 @@ public void singlePageGetBucket() throws IOException, URISyntaxException {
         final String prefix = "";
         final String nextMarker = null;
         final int maxKeys = 100;
-        paginate(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, TEST_DELIMITER, nextMarker, maxKeys, RETRIES));
+        paginate(new GetBucketKeyLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, TEST_DELIMITER, nextMarker, maxKeys, RETRIES, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
 
@@ -128,13 +112,13 @@ public void multiPageGetBucket() throws IOException, URISyntaxException {
         final String prefix = "";
         final String nextMarker = null;
         final int maxKeys = 2;
-        paginate(new GetBucketLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, TEST_DELIMITER, maxKeys, RETRIES));
+        paginate(new GetBucketKeyLoaderFactory(CLIENT, TEST_ENV_NAME, prefix, nextMarker, TEST_DELIMITER, maxKeys, RETRIES, GetBucketKeyLoaderFactory.contentsFunction));
 
     }
 
     @Test
     public void failedRequestGetBucket() {
-        testFailedRequest(new GetBucketLoaderFactory(CLIENT, "Unknown_Bucket",null, TEST_DELIMITER,null, 1000, 5));
+        testFailedRequest(new GetBucketKeyLoaderFactory(CLIENT, "Unknown_Bucket",null, TEST_DELIMITER,null, 1000, 5, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
    @Test
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 6e367e43c..64108cc1e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -17,7 +17,6 @@
 
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 520df8146..9166e6a6e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -22,23 +22,23 @@
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.HeadBucketRequest;
+import com.spectralogic.ds3client.commands.HeadBucketResponse;
+import com.spectralogic.ds3client.commands.PutBucketRequest;
+import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
-import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
-import com.spectralogic.ds3client.helpers.pagination.GetBucketLoaderFactory;
-import com.spectralogic.ds3client.commands.*;
-import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
+import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;
 import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers;
 import com.spectralogic.ds3client.models.*;
-import com.spectralogic.ds3client.models.common.CommonPrefixes;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.models.common.Range;
-import com.spectralogic.ds3client.models.bulk.*;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import org.slf4j.Logger;
@@ -67,30 +67,6 @@ class Ds3ClientHelpersImpl extends Ds3ClientHelpers {
     private final EventRunner eventRunner;
     private final FileSystemHelper fileSystemHelper;
 
-    private final Function> getFileSystemKeysFunction = new Function>() {
-        @Nullable
-        @Override
-        public ImmutableList apply(@Nullable final ListBucketResult result) {
-            final ImmutableList.Builder fileSystemKeys = new ImmutableList.Builder<>();
-            for(final CommonPrefixes prefix: result.getCommonPrefixes()) {
-                fileSystemKeys.add(new FileSystemKey(prefix));
-            }
-            for(final Contents content : result.getObjects()) {
-                fileSystemKeys.add(new FileSystemKey(content));
-            }
-            return fileSystemKeys.build();
-        }
-    };
-
-    private final Function> getContentsFunction = new Function>() {
-        @Nullable
-        @Override
-        public ImmutableList apply(@Nullable final ListBucketResult input) {
-            return ImmutableList.copyOf(input.getObjects());
-        }
-    };
-
-
     public Ds3ClientHelpersImpl(final Ds3Client client) {
         this(client, TransferStrategyBuilder.DEFAULT_CHUNK_ATTEMPT_RETRY_ATTEMPTS);
     }
@@ -531,13 +507,13 @@ public Iterable listObjects(final String bucket, final String keyPrefi
     @Override
     public Iterable listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys) {
 
-        return new LazyIterable<>(new GetBucketLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES));
+        return this.listObjects(bucket, keyPrefix, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES);
     }
 
     @Override
     public Iterable listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys, final int retries) {
 
-        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries, getContentsFunction));
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
     @Override
@@ -568,7 +544,7 @@ public Iterable remoteListDirectory(final String bucket, final St
     }
 
     public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException {
-        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES, getFileSystemKeysFunction));
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES, GetBucketKeyLoaderFactory.getFileSystemKeysFunction));
     }
 
     public Iterable addPrefixToDs3ObjectsList(final Iterable objectsList, final String prefix) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index 1a186ef35..d574b0572 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -1,14 +1,24 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
 package com.spectralogic.ds3client.helpers.pagination;
 
 import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.GetBucketRequest;
 import com.spectralogic.ds3client.commands.GetBucketResponse;
-import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.models.FileSystemKey;
 import com.spectralogic.ds3client.models.ListBucketResult;
-import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.networking.TooManyRetriesException;
 import com.spectralogic.ds3client.utils.Guard;
@@ -21,18 +31,19 @@
 public class GetBucketKeyLoader implements LazyIterable.LazyLoader{
     private static final int DEFAULT_MAX_KEYS = 1000;
 
+
     private final Ds3Client client;
     private final String bucket;
     private final String prefix;
     private final String delimiter;
     private final int maxKeys;
     private final int retryCount;
-    private final Function> function;
+    private final Function> function;
     private String nextMarker;
     private boolean truncated;
     private boolean endOfInput = false;
 
-    public GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount, final Function> function) {
+    public GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount, final Function> function) {
         this.client = client;
         this.bucket = bucket;
         this.prefix = prefix;
@@ -47,11 +58,11 @@ public GetBucketKeyLoader(final Ds3Client client, final String bucket, final Str
 
     @Override
     public List getNextValues() {
-        if(endOfInput) {
+        if (endOfInput) {
             return (List) Collections.emptyList();
         }
         int retryAttempt = 0;
-        while(true) {
+        while (true) {
             final GetBucketRequest request = new GetBucketRequest(bucket);
             request.withMaxKeys(maxKeys);
             if (prefix != null) { request.withPrefix(prefix); }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
index 9b52bbd64..34378c4e7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -1,7 +1,21 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
 package com.spectralogic.ds3client.helpers.pagination;
 
 import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.FluentIterable;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.models.Contents;
 import com.spectralogic.ds3client.models.FileSystemKey;
@@ -9,10 +23,42 @@
 import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 
+import javax.annotation.Nullable;
 import java.util.List;
 
 public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFactory {
 
+    public static final Function> contentsFunction = new Function>() {
+        @Override
+        public List apply(@Nullable final ListBucketResult input) {
+
+            return FluentIterable.from(input.getObjects()).toList();
+        }
+    };
+
+    public static final Function> getFileSystemKeysFunction = new Function>() {
+        @Nullable
+        @Override
+        public List apply(@Nullable final ListBucketResult result) {
+            final FluentIterable contentIterable = FluentIterable.from(result.getObjects())
+                    .transform(new Function() {
+                        @Override
+                        public FileSystemKey apply(@Nullable final Contents input) {
+                            return new FileSystemKey(input);
+                        }
+                    });
+            return FluentIterable.from(result.getCommonPrefixes())
+                    .transform( new Function() {
+                        @Override
+                        public FileSystemKey apply(@Nullable final CommonPrefixes input) {
+                            return new FileSystemKey(input);
+                        }
+                    })
+                    .append(contentIterable)
+                    .toList();
+        }
+    };
+
     private final Ds3Client client;
     private final String bucket;
     private final String keyPrefix;
@@ -20,9 +66,9 @@ public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFact
     private final String delimiter;
     private final int maxKeys;
     private final int defaultListObjectsRetries;
-    private final Function> function;
+    private final Function> function;
 
-    public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries, final Function> function) {
+    public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries, final Function> function) {
         this.client = client;
         this.bucket = bucket;
         this.keyPrefix = keyPrefix;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java
deleted file mode 100644
index 6943d7c58..000000000
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoader.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * ******************************************************************************
- *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
- *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
- *   this file except in compliance with the License. A copy of the License is located at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *   or in the "license" file accompanying this file.
- *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
- *   specific language governing permissions and limitations under the License.
- * ****************************************************************************
- */
-
-package com.spectralogic.ds3client.helpers.pagination;
-
-import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.GetBucketRequest;
-import com.spectralogic.ds3client.commands.GetBucketResponse;
-import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.models.ListBucketResult;
-import com.spectralogic.ds3client.networking.FailedRequestException;
-import com.spectralogic.ds3client.networking.TooManyRetriesException;
-import com.spectralogic.ds3client.utils.Guard;
-import com.spectralogic.ds3client.utils.collections.LazyIterable;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-public class GetBucketLoader implements LazyIterable.LazyLoader {
-    private static final int DEFAULT_MAX_KEYS = 1000;
-
-    private final Ds3Client client;
-    private final String bucket;
-    private final String prefix;
-    private final String delimiter;
-    private final int maxKeys;
-    private final int retryCount;
-
-    private String nextMarker;
-    private boolean truncated;
-    private boolean endOfInput = false;
-
-    public GetBucketLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount) {
-        this.client = client;
-        this.bucket = bucket;
-        this.prefix = prefix;
-        this.delimiter = delimiter;
-        this.maxKeys = maxKeys;
-        this.retryCount = retryCount;
-
-        this.nextMarker = nextMarker;
-        this.truncated = nextMarker != null;
-    }
-
-    @Override
-    public List getNextValues() {
-        if (endOfInput) {
-            return Collections.emptyList();
-        }
-        int retryAttempt = 0;
-        while(true) {
-            final GetBucketRequest request = new GetBucketRequest(bucket);
-            request.withMaxKeys(Math.min(maxKeys, DEFAULT_MAX_KEYS));
-            if (prefix != null) {
-                request.withPrefix(prefix);
-            }
-            if (truncated) {
-                request.withMarker(nextMarker);
-            }
-
-            if (delimiter != null) {
-                request.withDelimiter(delimiter);
-            }
-
-            final GetBucketResponse response;
-            try {
-                response = this.client.getBucket(request);
-                final ListBucketResult result = response.getListBucketResult();
-
-                truncated = result.getTruncated();
-                this.nextMarker = result.getNextMarker();
-
-                if (Guard.isStringNullOrEmpty(nextMarker) && !truncated) {
-                    endOfInput = true;
-                }
-
-                return result.getObjects();
-            } catch (final FailedRequestException e) {
-              throw new RuntimeException("Failed to get the list of objects due to a failed request", e);
-            } catch (final IOException e) {
-                if (retryAttempt >= retryCount) {
-                    //TODO need a proxied test to validate this retry logic
-                    throw new TooManyRetriesException("Failed to get the next set of objects from the getBucket request after " + retryCount + " retries", e);
-                }
-                retryAttempt++;
-            }
-        }
-
-    }
-}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
deleted file mode 100644
index 7ac05747a..000000000
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * ******************************************************************************
- *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
- *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
- *   this file except in compliance with the License. A copy of the License is located at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *   or in the "license" file accompanying this file.
- *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
- *   specific language governing permissions and limitations under the License.
- * ****************************************************************************
- */
-
-package com.spectralogic.ds3client.helpers.pagination;
-
-import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.utils.collections.LazyIterable;
-
-public class GetBucketLoaderFactory implements LazyIterable.LazyLoaderFactory {
-
-    private final Ds3Client client;
-    private final String bucket;
-    private final String keyPrefix;
-    private final String nextMarker;
-    private final String delimiter;
-    private final int maxKeys;
-    private final int defaultListObjectsRetries;
-
-    public GetBucketLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries) {
-
-        this.client = client;
-        this.bucket = bucket;
-        this.keyPrefix = keyPrefix;
-        this.delimiter = delimiter;
-        this.nextMarker = nextMarker;
-        this.maxKeys = maxKeys;
-        this.defaultListObjectsRetries = defaultListObjectsRetries;
-    }
-
-    @Override
-    public LazyIterable.LazyLoader create() {
-        return new GetBucketLoader(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, defaultListObjectsRetries);
-    }
-}

From 9f7d76ba1a895fddc0d9ec85b515521f6b6599e8 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 15:41:50 -0600
Subject: [PATCH 044/229] Refactored for readability and to try and make the
 methods each have less they are responsible for.

---
 .../pagination/GetBucketKeyLoader.java        | 52 ++++++++++++-------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index d574b0572..00d1ee1cc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -30,8 +30,6 @@
 
 public class GetBucketKeyLoader implements LazyIterable.LazyLoader{
     private static final int DEFAULT_MAX_KEYS = 1000;
-
-
     private final Ds3Client client;
     private final String bucket;
     private final String prefix;
@@ -56,6 +54,36 @@ public GetBucketKeyLoader(final Ds3Client client, final String bucket, final Str
         this.function = function;
     }
 
+    private GetBucketRequest prepRequest() {
+        final GetBucketRequest request = new GetBucketRequest(bucket);
+        request.withMaxKeys(maxKeys);
+        if (prefix != null) {
+            request.withPrefix(prefix);
+        } ;
+        if (truncated) {
+            request.withMarker(nextMarker);
+        } ;
+        if (delimiter != null) {
+            request.withDelimiter(delimiter);
+        }
+        return request;
+    }
+
+    private GetBucketResponse getResponse(GetBucketRequest request, int retryAttempt) {
+        GetBucketResponse response;
+        try {
+            response = this.client.getBucket(request);
+        } catch (final FailedRequestException e) {
+            throw new RuntimeException("Failed to get the list of objects due to a failed request", e);
+        } catch (final IOException e) {
+            if (retryAttempt >= retryCount) {
+                throw new TooManyRetriesException("Failed to get the next set of objects from the getBucketKey request after " + retryCount + " retries", e);
+            }
+            response = null;
+        }
+        return response;
+    }
+
     @Override
     public List getNextValues() {
         if (endOfInput) {
@@ -63,34 +91,18 @@ public List getNextValues() {
         }
         int retryAttempt = 0;
         while (true) {
-            final GetBucketRequest request = new GetBucketRequest(bucket);
-            request.withMaxKeys(maxKeys);
-            if (prefix != null) { request.withPrefix(prefix); }
-            if (truncated) { request.withMarker(nextMarker); }
-            if (delimiter != null) { request.withDelimiter(delimiter); }
-            GetBucketResponse response;
-            try {
-                response = this.client.getBucket(request);
-            } catch (final FailedRequestException e) {
-                throw new RuntimeException("Failed to get the list of objects due to a failed request", e);
-            } catch (final IOException e) {
-               if (retryAttempt >= retryCount) {
-                   throw new TooManyRetriesException("Failed to get the next set of objects from the getBucketKey request after " + retryCount + " retries", e);
-               }
-               response = null;
-            }
+            final GetBucketRequest request = prepRequest();
+            final GetBucketResponse response = getResponse(request, retryAttempt);
             if (response == null) {
                 retryAttempt++;
             } else {
                 final ListBucketResult result = response.getListBucketResult();
-
                 truncated = result.getTruncated();
                 this.nextMarker = result.getNextMarker();
                 if (Guard.isStringNullOrEmpty(nextMarker) && !truncated) {
                     endOfInput = true;
                 }
                 return function.apply(result);
-
             }
         }
     }

From 31dcdc26ba2d520c7006330f7cc416ee6e0b7ed6 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 15:50:54 -0600
Subject: [PATCH 045/229] Continued simplifying of the methods.

---
 .../ds3client/helpers/pagination/GetBucketKeyLoader.java     | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index 00d1ee1cc..e425e9c81 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -30,6 +30,7 @@
 
 public class GetBucketKeyLoader implements LazyIterable.LazyLoader{
     private static final int DEFAULT_MAX_KEYS = 1000;
+    private final List emptyList = Collections.emptyList();
     private final Ds3Client client;
     private final String bucket;
     private final String prefix;
@@ -86,9 +87,7 @@ private GetBucketResponse getResponse(GetBucketRequest request, int retryAttempt
 
     @Override
     public List getNextValues() {
-        if (endOfInput) {
-            return (List) Collections.emptyList();
-        }
+        if (endOfInput) { return emptyList; }
         int retryAttempt = 0;
         while (true) {
             final GetBucketRequest request = prepRequest();

From a0779ea474f3ac51a6ca2129565f1e10240a333b Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 15:55:05 -0600
Subject: [PATCH 046/229] Formatted if statements.

---
 .../java/com/spectralogic/ds3client/models/FileSystemKey.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
index 3a003b8cc..dc13b5f9c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
@@ -50,7 +50,7 @@ public boolean isContents() {
     }
 
     public CommonPrefixes toCommonPrefixes() throws RuntimeException{
-       if(isContents()) {
+       if (isContents()) {
            throw new RuntimeException("Could not create CommonPrefixes");
        }
        final CommonPrefixes commonPrefixes = new CommonPrefixes();
@@ -59,7 +59,7 @@ public CommonPrefixes toCommonPrefixes() throws RuntimeException{
     }
 
     public Contents toContents() throws RuntimeException {
-        if(isPrefix()) {
+        if (isPrefix()) {
             throw new RuntimeException("Could not create Contents");
         }
         final Contents contents = new Contents();

From b6a6b5915c42afc6772aa7e41eace56672bcb2a1 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 15:58:22 -0600
Subject: [PATCH 047/229] Missed a final

---
 .../spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index 09a45b97a..4657985da 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -273,7 +273,7 @@ public void testRemoteListDirectory() throws IOException {
         Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, null))).thenReturn(new StubGetBucketResponse(0));
         Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, "baz"))).thenReturn(new StubGetBucketResponse(1));
         Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, "/foo/baz"))).thenReturn(new StubGetBucketResponse(1));
-        Iterable fileSystemKeys = Ds3ClientHelpers.wrap(ds3Client).remoteListDirectory(MYBUCKET, null, "/", null, 0);
+        final Iterable fileSystemKeys = Ds3ClientHelpers.wrap(ds3Client).remoteListDirectory(MYBUCKET, null, "/", null, 0);
         final List contentsList = Lists.newArrayList(fileSystemKeys);
         assertThat(contentsList.size(), is(3));
     }

From f1e6372ec28cca9c6f5f56d08aeadfa8e80b8022 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 16:01:05 -0600
Subject: [PATCH 048/229] Copyright added.

---
 .../pagination/GetBucketKeyLoaderFactory.java      |  1 +
 .../ds3client/models/FileSystemKey.java            | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
index 34378c4e7..dc01f4395 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -12,6 +12,7 @@
  *   specific language governing permissions and limitations under the License.
  * ****************************************************************************
  */
+
 package com.spectralogic.ds3client.helpers.pagination;
 
 import com.google.common.base.Function;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
index dc13b5f9c..d910c3ce5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
@@ -1,3 +1,17 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
 package com.spectralogic.ds3client.models;
 
 import com.spectralogic.ds3client.models.common.CommonPrefixes;

From 1b4141b4f2e273bf4c2f3d6b778003e8579197f5 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 16:36:57 -0600
Subject: [PATCH 049/229] Working on comments.

---
 .../spectralogic/ds3client/integration/Iterators_Test.java    | 4 ++--
 .../com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java  | 4 +++-
 .../ds3client/helpers/pagination/GetBucketKeyLoader.java      | 2 +-
 .../test/java/com/spectralogic/ds3client/Ds3Client_Test.java  | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 98198e474..66c5eae75 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -72,7 +72,7 @@ public static void teardown() throws IOException {
     }
 
     @Test
-    public void IterateWithDelimiter() throws IOException, URISyntaxException {
+    public void iterateWithDelimiter() throws IOException, URISyntaxException {
         try {
             HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/");
@@ -82,7 +82,7 @@ public void IterateWithDelimiter() throws IOException, URISyntaxException {
 
             loadBookTestDataWithPrefix(CLIENT, TEST_ENV_NAME, "books/more/");
             final Iterable  anotherFileSystemKeysIterator  = Ds3ClientHelpers.wrap(CLIENT).remoteListDirectory(TEST_ENV_NAME, "books/", null, 100);
-            final List anotherFileSystemKeys = Lists.newArrayList(fileSystemKeysIterator);
+            final List anotherFileSystemKeys = Lists.newArrayList(anotherFileSystemKeysIterator);
             assertThat(anotherFileSystemKeys.size(), is(5));
         } finally {
             deleteAllContents(CLIENT, TEST_ENV_NAME);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 64108cc1e..81204cd49 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -470,7 +470,9 @@ public abstract Iterable listObjects(final String bucket, final String
 
 
     /**
-     * Defaults to the path delimiter of '/'
+     * The '/' character is traditionally used as a path delimiter, these methods provide '/' as a default delimiter.
+     * An S3 object like foo/bar/baz with the prefix of foo/ and the delimiter of '/' will act similar to a traditional file system
+     *  proving a common prefix for each 'directory' descending from the current folder of the prefix.
      * Returns an Iterable of {@link FileSystemKey}
      * @param bucket The bucket
      * @param keyPrefix Limits the response to keys that begin with the specified prefix
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index e425e9c81..973b0bd49 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -60,7 +60,7 @@ private GetBucketRequest prepRequest() {
         request.withMaxKeys(maxKeys);
         if (prefix != null) {
             request.withPrefix(prefix);
-        } ;
+        }
         if (truncated) {
             request.withMarker(nextMarker);
         } ;
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 83f554a50..506e5b441 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -926,7 +926,7 @@ public void testGettingDefaultUserAgent() {
         final String userAgent = newClient.getConnectionDetails().getUserAgent();
         final String[] userAgentFields = userAgent.split("-");
 
-        assertTrue(userAgentFields.length >= 2);
+        assertThat(userAgentFields.length >= 2, is(true));
 
         // look for a pattern like 3.4.0, but leave open the possibility of a string like 3.4.0-SNAPSHOT
         final Pattern matchPattern = Pattern.compile("\\d+\\.\\d+\\.\\d+");

From 33889df9ca454a8b4d33fb662bab2e9da02bf434 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Wed, 7 Jun 2017 16:52:12 -0600
Subject: [PATCH 050/229] Added custom runtimeExceptions.

---
 .../pagination/GetBucketKeyLoader.java        |  7 ++++---
 .../ds3client/models/FileSystemKey.java       |  4 ++--
 ...bleToConvertToCommonPrefixesException.java | 21 +++++++++++++++++++
 .../UnabledToConvertToContentsException.java  | 21 +++++++++++++++++++
 .../FailedToGetBucketException.java           | 21 +++++++++++++++++++
 .../helpers/Ds3ClientHelpers_Test.java        |  1 -
 6 files changed, 69 insertions(+), 6 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnableToConvertToCommonPrefixesException.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnabledToConvertToContentsException.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/FailedToGetBucketException.java

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index 973b0bd49..000b79027 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -20,6 +20,7 @@
 import com.spectralogic.ds3client.commands.GetBucketResponse;
 import com.spectralogic.ds3client.models.ListBucketResult;
 import com.spectralogic.ds3client.networking.FailedRequestException;
+import com.spectralogic.ds3client.networking.FailedToGetBucketException;
 import com.spectralogic.ds3client.networking.TooManyRetriesException;
 import com.spectralogic.ds3client.utils.Guard;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
@@ -63,19 +64,19 @@ private GetBucketRequest prepRequest() {
         }
         if (truncated) {
             request.withMarker(nextMarker);
-        } ;
+        }
         if (delimiter != null) {
             request.withDelimiter(delimiter);
         }
         return request;
     }
 
-    private GetBucketResponse getResponse(GetBucketRequest request, int retryAttempt) {
+    private GetBucketResponse getResponse(final GetBucketRequest request, final int retryAttempt) {
         GetBucketResponse response;
         try {
             response = this.client.getBucket(request);
         } catch (final FailedRequestException e) {
-            throw new RuntimeException("Failed to get the list of objects due to a failed request", e);
+            throw new FailedToGetBucketException("Failed to get the list of objects due to a failed request", e);
         } catch (final IOException e) {
             if (retryAttempt >= retryCount) {
                 throw new TooManyRetriesException("Failed to get the next set of objects from the getBucketKey request after " + retryCount + " retries", e);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
index d910c3ce5..bb147ca05 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
@@ -65,7 +65,7 @@ public boolean isContents() {
 
     public CommonPrefixes toCommonPrefixes() throws RuntimeException{
        if (isContents()) {
-           throw new RuntimeException("Could not create CommonPrefixes");
+           throw new UnableToConvertToCommonPrefixesException("Could not create a CommonPrefix");
        }
        final CommonPrefixes commonPrefixes = new CommonPrefixes();
        commonPrefixes.setPrefix(prefix);
@@ -74,7 +74,7 @@ public CommonPrefixes toCommonPrefixes() throws RuntimeException{
 
     public Contents toContents() throws RuntimeException {
         if (isPrefix()) {
-            throw new RuntimeException("Could not create Contents");
+            throw new UnabledToConvertToContentsException("Could not create a Contents");
         }
         final Contents contents = new Contents();
         contents.setETag(eTag);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnableToConvertToCommonPrefixesException.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnableToConvertToCommonPrefixesException.java
new file mode 100644
index 000000000..177b472ea
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnableToConvertToCommonPrefixesException.java
@@ -0,0 +1,21 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.models;
+public class UnableToConvertToCommonPrefixesException extends RuntimeException {
+    public UnableToConvertToCommonPrefixesException(final String s) {
+        super(s);
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnabledToConvertToContentsException.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnabledToConvertToContentsException.java
new file mode 100644
index 000000000..e18a161f0
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnabledToConvertToContentsException.java
@@ -0,0 +1,21 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+package com.spectralogic.ds3client.models;
+
+public class UnabledToConvertToContentsException extends RuntimeException {
+    public UnabledToConvertToContentsException(final String s) {
+        super(s);
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/FailedToGetBucketException.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/FailedToGetBucketException.java
new file mode 100644
index 000000000..27f13edef
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/FailedToGetBucketException.java
@@ -0,0 +1,21 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+package com.spectralogic.ds3client.networking;
+
+public class FailedToGetBucketException extends RuntimeException {
+    public FailedToGetBucketException(final String s, final FailedRequestException e) {
+        super(s, e);
+    }
+}
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index 4657985da..6cb875b61 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -28,7 +28,6 @@
 import com.spectralogic.ds3client.models.Error;
 import com.spectralogic.ds3client.models.Objects;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
-import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.networking.ConnectionDetails;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;

From 841fa626b39adb95f9928630645cc9d0dde60f08 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Thu, 8 Jun 2017 11:39:20 -0600
Subject: [PATCH 051/229] Cleaned up GetBucketKeyLoader to improve
 comprehension.

---
 .../pagination/GetBucketKeyLoader.java        | 65 +++++++------------
 1 file changed, 25 insertions(+), 40 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index 000b79027..d9aee8dec 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -29,7 +29,7 @@
 import java.util.Collections;
 import java.util.List;
 
-public class GetBucketKeyLoader implements LazyIterable.LazyLoader{
+public class GetBucketKeyLoader implements LazyIterable.LazyLoader {
     private static final int DEFAULT_MAX_KEYS = 1000;
     private final List emptyList = Collections.emptyList();
     private final Ds3Client client;
@@ -38,12 +38,12 @@ public class GetBucketKeyLoader implements LazyIterable.LazyLoader{
     private final String delimiter;
     private final int maxKeys;
     private final int retryCount;
-    private final Function> function;
+    private final Function> function;
     private String nextMarker;
     private boolean truncated;
     private boolean endOfInput = false;
 
-    public GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount, final Function> function) {
+    GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount, final Function> function) {
         this.client = client;
         this.bucket = bucket;
         this.prefix = prefix;
@@ -52,58 +52,43 @@ public GetBucketKeyLoader(final Ds3Client client, final String bucket, final Str
         this.maxKeys = Math.min(maxKeys, DEFAULT_MAX_KEYS);
         this.retryCount = retryCount;
         this.nextMarker = nextMarker;
-        this.truncated  = nextMarker != null;
+        this.truncated = nextMarker != null;
         this.function = function;
     }
 
     private GetBucketRequest prepRequest() {
         final GetBucketRequest request = new GetBucketRequest(bucket);
+        if (prefix != null) { request.withPrefix(prefix); }
+        if (truncated) { request.withMarker(nextMarker); }
+        if (delimiter != null) { request.withDelimiter(delimiter); }
         request.withMaxKeys(maxKeys);
-        if (prefix != null) {
-            request.withPrefix(prefix);
-        }
-        if (truncated) {
-            request.withMarker(nextMarker);
-        }
-        if (delimiter != null) {
-            request.withDelimiter(delimiter);
-        }
         return request;
     }
 
-    private GetBucketResponse getResponse(final GetBucketRequest request, final int retryAttempt) {
-        GetBucketResponse response;
-        try {
-            response = this.client.getBucket(request);
-        } catch (final FailedRequestException e) {
-            throw new FailedToGetBucketException("Failed to get the list of objects due to a failed request", e);
-        } catch (final IOException e) {
-            if (retryAttempt >= retryCount) {
-                throw new TooManyRetriesException("Failed to get the next set of objects from the getBucketKey request after " + retryCount + " retries", e);
-            }
-            response = null;
-        }
-        return response;
-    }
-
     @Override
     public List getNextValues() {
         if (endOfInput) { return emptyList; }
         int retryAttempt = 0;
-        while (true) {
+        while (retryCount > retryAttempt) {
             final GetBucketRequest request = prepRequest();
-            final GetBucketResponse response = getResponse(request, retryAttempt);
-            if (response == null) {
+            final ListBucketResult result;
+            final GetBucketResponse response;
+            try {
+                response = this.client.getBucket(request);
+            } catch (final FailedRequestException e) {
+                throw new FailedToGetBucketException("Failed to get the list of objects due to a failed request", e);
+            } catch (final IOException e) {
                 retryAttempt++;
-            } else {
-                final ListBucketResult result = response.getListBucketResult();
-                truncated = result.getTruncated();
-                this.nextMarker = result.getNextMarker();
-                if (Guard.isStringNullOrEmpty(nextMarker) && !truncated) {
-                    endOfInput = true;
-                }
-                return function.apply(result);
+                continue;
+            }
+            result = response.getListBucketResult();
+            this.truncated = result.getTruncated();
+            this.nextMarker = result.getNextMarker();
+            if (Guard.isStringNullOrEmpty(nextMarker) && !truncated) {
+                endOfInput = true;
             }
+            return function.apply(result);
         }
+        throw new TooManyRetriesException("Failed to get the next set of objects from the getBucketKey request after " + retryCount + " retries");
     }
-}
+}
\ No newline at end of file

From e57af9bc2b12d66f149b9d46fa6b790ff93ea88b Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Fri, 9 Jun 2017 09:33:26 -0600
Subject: [PATCH 052/229] Converted LazyLoader to user an Iterable instead of a
 List.

This allows the old method of having getNextValues() to continue to work
but also allows for reducing the amount of times data is copied.
---
 .../pagination/GetBucketKeyLoader.java        |  6 ++---
 .../pagination/GetBucketKeyLoaderFactory.java | 17 +++++++-------
 .../utils/collections/LazyIterable.java       | 22 ++++++-------------
 3 files changed, 18 insertions(+), 27 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
index d9aee8dec..ddabea3c3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoader.java
@@ -38,12 +38,12 @@ public class GetBucketKeyLoader implements LazyIterable.LazyLoader {
     private final String delimiter;
     private final int maxKeys;
     private final int retryCount;
-    private final Function> function;
+    private final Function> function;
     private String nextMarker;
     private boolean truncated;
     private boolean endOfInput = false;
 
-    GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount, final Function> function) {
+    GetBucketKeyLoader(final Ds3Client client, final String bucket, final String prefix, final String delimiter, final String nextMarker, final int maxKeys, final int retryCount, final Function> function) {
         this.client = client;
         this.bucket = bucket;
         this.prefix = prefix;
@@ -66,7 +66,7 @@ private GetBucketRequest prepRequest() {
     }
 
     @Override
-    public List getNextValues() {
+    public Iterable getNextValues() {
         if (endOfInput) { return emptyList; }
         int retryAttempt = 0;
         while (retryCount > retryAttempt) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
index dc01f4395..c0b4cb3b6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -29,18 +29,18 @@
 
 public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFactory {
 
-    public static final Function> contentsFunction = new Function>() {
+    public static final Function> contentsFunction = new Function>() {
         @Override
-        public List apply(@Nullable final ListBucketResult input) {
+        public Iterable apply(@Nullable final ListBucketResult input) {
 
-            return FluentIterable.from(input.getObjects()).toList();
+            return FluentIterable.from(input.getObjects());
         }
     };
 
-    public static final Function> getFileSystemKeysFunction = new Function>() {
+    public static final Function> getFileSystemKeysFunction = new Function>() {
         @Nullable
         @Override
-        public List apply(@Nullable final ListBucketResult result) {
+        public Iterable apply(@Nullable final ListBucketResult result) {
             final FluentIterable contentIterable = FluentIterable.from(result.getObjects())
                     .transform(new Function() {
                         @Override
@@ -55,8 +55,7 @@ public FileSystemKey apply(@Nullable final CommonPrefixes input) {
                             return new FileSystemKey(input);
                         }
                     })
-                    .append(contentIterable)
-                    .toList();
+                    .append(contentIterable);
         }
     };
 
@@ -67,9 +66,9 @@ public FileSystemKey apply(@Nullable final CommonPrefixes input) {
     private final String delimiter;
     private final int maxKeys;
     private final int defaultListObjectsRetries;
-    private final Function> function;
+    private final Function> function;
 
-    public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries, final Function> function) {
+    public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries, final Function> function) {
         this.client = client;
         this.bucket = bucket;
         this.keyPrefix = keyPrefix;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/LazyIterable.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/LazyIterable.java
index fc0d403f7..3eb6565dd 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/LazyIterable.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/LazyIterable.java
@@ -15,10 +15,7 @@
 
 package com.spectralogic.ds3client.utils.collections;
 
-import com.spectralogic.ds3client.utils.Guard;
-
 import java.util.Iterator;
-import java.util.List;
 import java.util.NoSuchElementException;
 
 /**
@@ -59,15 +56,14 @@ public interface LazyLoaderFactory {
      * @param  The model that is returned by the iterator
      */
     public interface LazyLoader {
-        List getNextValues();
+        Iterable getNextValues();
     }
 
     private class LazyObjectIterator implements Iterator {
 
         private final LazyLoader iterableLoader;
 
-        private List cache = null;
-        private int cachePointer;
+        private Iterator cache = null;
         private boolean endOfContent = false;
 
         private LazyObjectIterator(final LazyLoader iterableLoader) {
@@ -80,7 +76,7 @@ public boolean hasNext() {
                 return false;
             }
 
-            if (cache == null || cachePointer == cache.size()) {
+            if (cache == null || !cache.hasNext()) {
                 loadCache();
             }
             return !endOfContent;
@@ -91,20 +87,16 @@ public E next() {
             if (endOfContent) {
                 throw new NoSuchElementException("No more content");
             }
-            if (cache == null || cachePointer >= cache.size()) {
+            if (cache == null || !cache.hasNext()) {
                 loadCache();
             }
-            final E contents = cache.get(cachePointer);
-            this.cachePointer++;
-            return contents;
+            return cache.next();
         }
 
         private void loadCache() {
-            this.cache = iterableLoader.getNextValues();
-            if (Guard.isNullOrEmpty(this.cache)) {
+            this.cache = iterableLoader.getNextValues().iterator();
+            if (!cache.hasNext()) {
                 endOfContent = true;
-            } else {
-                this.cachePointer = 0;
             }
         }
 

From d8264af003beacf2041590f0fe90b704871f84c3 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Fri, 9 Jun 2017 10:01:40 -0600
Subject: [PATCH 053/229] Removed unused import.

---
 .../ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
index c0b4cb3b6..f2874467b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -23,9 +23,7 @@
 import com.spectralogic.ds3client.models.ListBucketResult;
 import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
-
 import javax.annotation.Nullable;
-import java.util.List;
 
 public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFactory {
 

From b3a9014eaae12e0ef6d9cbe18931874f78445c2b Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Fri, 9 Jun 2017 10:17:24 -0600
Subject: [PATCH 054/229] Removed unused constructor.

---
 .../helpers/pagination/GetBucketKeyLoaderFactory.java          | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
index f2874467b..10687fc52 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -30,8 +30,7 @@ public class GetBucketKeyLoaderFactory implements LazyIterable.LazyLoaderFact
     public static final Function> contentsFunction = new Function>() {
         @Override
         public Iterable apply(@Nullable final ListBucketResult input) {
-
-            return FluentIterable.from(input.getObjects());
+            return input.getObjects();
         }
     };
 

From d0e0eec5567d2a38a839ad76805f487c2a91b8e4 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Fri, 16 Jun 2017 11:58:31 -0600
Subject: [PATCH 055/229] Update build.gradle

Changing version number to prepare for release
---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 01e123e56..89efe81b0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -15,7 +15,7 @@
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '3.4.1-SNAPSHOT'
+    version = '3.4.1'
 }
 
 subprojects {

From ce6959e43156d629d9202b8923314f10e2540be1 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 16 Jun 2017 13:48:14 -0600
Subject: [PATCH 056/229] Moved some helper files into proper folders

---
 .../ds3client/integration/Iterators_Test.java             | 2 +-
 .../UnableToConvertToCommonPrefixesException.java         | 2 +-
 .../UnableToConvertToContentsException.java}              | 6 +++---
 .../spectralogic/ds3client/helpers/Ds3ClientHelpers.java  | 2 +-
 .../ds3client/helpers/Ds3ClientHelpersImpl.java           | 1 +
 .../{models => helpers/pagination}/FileSystemKey.java     | 8 ++++++--
 .../helpers/pagination/GetBucketKeyLoaderFactory.java     | 1 -
 .../ds3client/helpers/Ds3ClientHelpers_Test.java          | 1 +
 8 files changed, 14 insertions(+), 9 deletions(-)
 rename ds3-sdk/src/main/java/com/spectralogic/ds3client/{models => exceptions}/UnableToConvertToCommonPrefixesException.java (95%)
 rename ds3-sdk/src/main/java/com/spectralogic/ds3client/{models/UnabledToConvertToContentsException.java => exceptions/UnableToConvertToContentsException.java} (80%)
 rename ds3-sdk/src/main/java/com/spectralogic/ds3client/{models => helpers/pagination}/FileSystemKey.java (88%)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index 66c5eae75..f607395f5 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -24,7 +24,7 @@
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.models.FileSystemKey;
+import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import org.junit.AfterClass;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnableToConvertToCommonPrefixesException.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/UnableToConvertToCommonPrefixesException.java
similarity index 95%
rename from ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnableToConvertToCommonPrefixesException.java
rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/UnableToConvertToCommonPrefixesException.java
index 177b472ea..7fa52ec8b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnableToConvertToCommonPrefixesException.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/UnableToConvertToCommonPrefixesException.java
@@ -13,7 +13,7 @@
  * ****************************************************************************
  */
 
-package com.spectralogic.ds3client.models;
+package com.spectralogic.ds3client.exceptions;
 public class UnableToConvertToCommonPrefixesException extends RuntimeException {
     public UnableToConvertToCommonPrefixesException(final String s) {
         super(s);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnabledToConvertToContentsException.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/UnableToConvertToContentsException.java
similarity index 80%
rename from ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnabledToConvertToContentsException.java
rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/UnableToConvertToContentsException.java
index e18a161f0..37f69694c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/UnabledToConvertToContentsException.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/UnableToConvertToContentsException.java
@@ -12,10 +12,10 @@
  *   specific language governing permissions and limitations under the License.
  * ****************************************************************************
  */
-package com.spectralogic.ds3client.models;
+package com.spectralogic.ds3client.exceptions;
 
-public class UnabledToConvertToContentsException extends RuntimeException {
-    public UnabledToConvertToContentsException(final String s) {
+public class UnableToConvertToContentsException extends RuntimeException {
+    public UnableToConvertToContentsException(final String s) {
         super(s);
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 81204cd49..0e6072830 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -22,7 +22,7 @@
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.models.FileSystemKey;
+import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.utils.Predicate;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index fac326c36..3d08bf8e7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -30,6 +30,7 @@
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
+import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
 import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/FileSystemKey.java
similarity index 88%
rename from ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/FileSystemKey.java
index bb147ca05..d388f711d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FileSystemKey.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/FileSystemKey.java
@@ -12,8 +12,12 @@
  *   specific language governing permissions and limitations under the License.
  * ****************************************************************************
  */
-package com.spectralogic.ds3client.models;
+package com.spectralogic.ds3client.helpers.pagination;
 
+import com.spectralogic.ds3client.exceptions.UnableToConvertToCommonPrefixesException;
+import com.spectralogic.ds3client.exceptions.UnableToConvertToContentsException;
+import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.User;
 import com.spectralogic.ds3client.models.common.CommonPrefixes;
 
 import java.util.Date;
@@ -74,7 +78,7 @@ public CommonPrefixes toCommonPrefixes() throws RuntimeException{
 
     public Contents toContents() throws RuntimeException {
         if (isPrefix()) {
-            throw new UnabledToConvertToContentsException("Could not create a Contents");
+            throw new UnableToConvertToContentsException("Could not create a Contents");
         }
         final Contents contents = new Contents();
         contents.setETag(eTag);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
index 10687fc52..7fb40d110 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -19,7 +19,6 @@
 import com.google.common.collect.FluentIterable;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.models.FileSystemKey;
 import com.spectralogic.ds3client.models.ListBucketResult;
 import com.spectralogic.ds3client.models.common.CommonPrefixes;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index 6cb875b61..abe990803 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -24,6 +24,7 @@
 import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.Job;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder;
+import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
 import com.spectralogic.ds3client.models.*;
 import com.spectralogic.ds3client.models.Error;
 import com.spectralogic.ds3client.models.Objects;

From 2b15152d135856af7092b9f93d13f9b08d9e5e88 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 16 Jun 2017 13:50:07 -0600
Subject: [PATCH 057/229] Updated to 3.5.3 api

---
 ...teDataPersistenceRuleSpectraS3Request.java | 19 ++++++++++-----
 ...etDataPersistenceRuleSpectraS3Request.java | 19 ++++++++++-----
 ...fyDataPersistenceRuleSpectraS3Request.java | 19 ++++++++++-----
 .../RawImportAllTapesSpectraS3Request.java    | 22 +++++++++++++++++
 .../RawImportTapeSpectraS3Request.java        | 23 +++++++++++++++++-
 .../ds3client/models/ActiveJob.java           | 12 ++++++++++
 .../ds3client/models/TapeDrive.java           | 24 +++++++++++++++++++
 7 files changed, 119 insertions(+), 19 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteDataPersistenceRuleSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteDataPersistenceRuleSpectraS3Request.java
index b12176b72..109695f23 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteDataPersistenceRuleSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteDataPersistenceRuleSpectraS3Request.java
@@ -18,19 +18,26 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
 public class DeleteDataPersistenceRuleSpectraS3Request extends AbstractRequest {
 
     // Variables
     
-    private final String dataPersistenceRule;
+    private final String dataPersistenceRuleId;
 
     // Constructor
     
     
-    public DeleteDataPersistenceRuleSpectraS3Request(final String dataPersistenceRule) {
-        this.dataPersistenceRule = dataPersistenceRule;
+    public DeleteDataPersistenceRuleSpectraS3Request(final UUID dataPersistenceRuleId) {
+        this.dataPersistenceRuleId = dataPersistenceRuleId.toString();
+        
+    }
+
+    
+    public DeleteDataPersistenceRuleSpectraS3Request(final String dataPersistenceRuleId) {
+        this.dataPersistenceRuleId = dataPersistenceRuleId;
         
     }
 
@@ -42,11 +49,11 @@ public HttpVerb getVerb() {
 
     @Override
     public String getPath() {
-        return "/_rest_/data_persistence_rule/" + dataPersistenceRule;
+        return "/_rest_/data_persistence_rule/" + dataPersistenceRuleId;
     }
     
-    public String getDataPersistenceRule() {
-        return this.dataPersistenceRule;
+    public String getDataPersistenceRuleId() {
+        return this.dataPersistenceRuleId;
     }
 
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetDataPersistenceRuleSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetDataPersistenceRuleSpectraS3Request.java
index e0152754e..fdc3fd364 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetDataPersistenceRuleSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetDataPersistenceRuleSpectraS3Request.java
@@ -18,19 +18,26 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
 public class GetDataPersistenceRuleSpectraS3Request extends AbstractRequest {
 
     // Variables
     
-    private final String dataPersistenceRule;
+    private final String dataPersistenceRuleId;
 
     // Constructor
     
     
-    public GetDataPersistenceRuleSpectraS3Request(final String dataPersistenceRule) {
-        this.dataPersistenceRule = dataPersistenceRule;
+    public GetDataPersistenceRuleSpectraS3Request(final UUID dataPersistenceRuleId) {
+        this.dataPersistenceRuleId = dataPersistenceRuleId.toString();
+        
+    }
+
+    
+    public GetDataPersistenceRuleSpectraS3Request(final String dataPersistenceRuleId) {
+        this.dataPersistenceRuleId = dataPersistenceRuleId;
         
     }
 
@@ -42,11 +49,11 @@ public HttpVerb getVerb() {
 
     @Override
     public String getPath() {
-        return "/_rest_/data_persistence_rule/" + dataPersistenceRule;
+        return "/_rest_/data_persistence_rule/" + dataPersistenceRuleId;
     }
     
-    public String getDataPersistenceRule() {
-        return this.dataPersistenceRule;
+    public String getDataPersistenceRuleId() {
+        return this.dataPersistenceRuleId;
     }
 
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPersistenceRuleSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPersistenceRuleSpectraS3Request.java
index f82a0d730..920d1bc14 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPersistenceRuleSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPersistenceRuleSpectraS3Request.java
@@ -21,13 +21,14 @@
 import com.spectralogic.ds3client.models.DataIsolationLevel;
 import java.lang.Integer;
 import com.spectralogic.ds3client.models.DataPersistenceRuleType;
+import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
 public class ModifyDataPersistenceRuleSpectraS3Request extends AbstractRequest {
 
     // Variables
     
-    private final String dataPersistenceRule;
+    private final String dataPersistenceRuleId;
 
     private DataIsolationLevel isolationLevel;
 
@@ -38,8 +39,14 @@ public class ModifyDataPersistenceRuleSpectraS3Request extends AbstractRequest {
     // Constructor
     
     
-    public ModifyDataPersistenceRuleSpectraS3Request(final String dataPersistenceRule) {
-        this.dataPersistenceRule = dataPersistenceRule;
+    public ModifyDataPersistenceRuleSpectraS3Request(final UUID dataPersistenceRuleId) {
+        this.dataPersistenceRuleId = dataPersistenceRuleId.toString();
+        
+    }
+
+    
+    public ModifyDataPersistenceRuleSpectraS3Request(final String dataPersistenceRuleId) {
+        this.dataPersistenceRuleId = dataPersistenceRuleId;
         
     }
 
@@ -72,11 +79,11 @@ public HttpVerb getVerb() {
 
     @Override
     public String getPath() {
-        return "/_rest_/data_persistence_rule/" + dataPersistenceRule;
+        return "/_rest_/data_persistence_rule/" + dataPersistenceRuleId;
     }
     
-    public String getDataPersistenceRule() {
-        return this.dataPersistenceRule;
+    public String getDataPersistenceRuleId() {
+        return this.dataPersistenceRuleId;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportAllTapesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportAllTapesSpectraS3Request.java
index b9c45ab9a..d009505cb 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportAllTapesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportAllTapesSpectraS3Request.java
@@ -19,6 +19,7 @@
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
+import java.util.UUID;
 import com.spectralogic.ds3client.models.Priority;
 
 public class RawImportAllTapesSpectraS3Request extends AbstractRequest {
@@ -27,6 +28,8 @@ public class RawImportAllTapesSpectraS3Request extends AbstractRequest {
     
     private final String bucketId;
 
+    private String storageDomainId;
+
     private Priority taskPriority;
 
     // Constructor
@@ -40,6 +43,20 @@ public RawImportAllTapesSpectraS3Request(final String bucketId) {
         this.getQueryParams().put("bucket_id", bucketId);
     }
 
+    public RawImportAllTapesSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
+        this.storageDomainId = storageDomainId.toString();
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+        return this;
+    }
+
+
+    public RawImportAllTapesSpectraS3Request withStorageDomainId(final String storageDomainId) {
+        this.storageDomainId = storageDomainId;
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+        return this;
+    }
+
+
     public RawImportAllTapesSpectraS3Request withTaskPriority(final Priority taskPriority) {
         this.taskPriority = taskPriority;
         this.updateQueryParam("task_priority", taskPriority);
@@ -63,6 +80,11 @@ public String getBucketId() {
     }
 
 
+    public String getStorageDomainId() {
+        return this.storageDomainId;
+    }
+
+
     public Priority getTaskPriority() {
         return this.taskPriority;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportTapeSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportTapeSpectraS3Request.java
index b4f8fcd13..80603dc05 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportTapeSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportTapeSpectraS3Request.java
@@ -19,8 +19,8 @@
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
-import com.spectralogic.ds3client.models.Priority;
 import java.util.UUID;
+import com.spectralogic.ds3client.models.Priority;
 
 public class RawImportTapeSpectraS3Request extends AbstractRequest {
 
@@ -30,6 +30,8 @@ public class RawImportTapeSpectraS3Request extends AbstractRequest {
 
     private final String bucketId;
 
+    private String storageDomainId;
+
     private Priority taskPriority;
 
     // Constructor
@@ -54,6 +56,20 @@ public RawImportTapeSpectraS3Request(final String bucketId, final String tapeId)
         this.getQueryParams().put("bucket_id", bucketId);
     }
 
+    public RawImportTapeSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
+        this.storageDomainId = storageDomainId.toString();
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+        return this;
+    }
+
+
+    public RawImportTapeSpectraS3Request withStorageDomainId(final String storageDomainId) {
+        this.storageDomainId = storageDomainId;
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+        return this;
+    }
+
+
     public RawImportTapeSpectraS3Request withTaskPriority(final Priority taskPriority) {
         this.taskPriority = taskPriority;
         this.updateQueryParam("task_priority", taskPriority);
@@ -82,6 +98,11 @@ public String getBucketId() {
     }
 
 
+    public String getStorageDomainId() {
+        return this.storageDomainId;
+    }
+
+
     public Priority getTaskPriority() {
         return this.taskPriority;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
index 0040298c8..72c7af5b8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
@@ -44,6 +44,9 @@ public class ActiveJob {
     @JsonProperty("CreatedAt")
     private Date createdAt;
 
+    @JsonProperty("DeadJobCleanupAllowed")
+    private boolean deadJobCleanupAllowed;
+
     @JsonProperty("ErrorMessage")
     private String errorMessage;
 
@@ -150,6 +153,15 @@ public void setCreatedAt(final Date createdAt) {
     }
 
 
+    public boolean getDeadJobCleanupAllowed() {
+        return this.deadJobCleanupAllowed;
+    }
+
+    public void setDeadJobCleanupAllowed(final boolean deadJobCleanupAllowed) {
+        this.deadJobCleanupAllowed = deadJobCleanupAllowed;
+    }
+
+
     public String getErrorMessage() {
         return this.errorMessage;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
index 507acd7db..74911ed31 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
@@ -26,6 +26,9 @@
 public class TapeDrive {
 
     // Variables
+    @JsonProperty("CleaningRequired")
+    private boolean cleaningRequired;
+
     @JsonProperty("ErrorMessage")
     private String errorMessage;
 
@@ -38,6 +41,9 @@ public class TapeDrive {
     @JsonProperty("LastCleaned")
     private Date lastCleaned;
 
+    @JsonProperty("MfgSerialNumber")
+    private String mfgSerialNumber;
+
     @JsonProperty("PartitionId")
     private UUID partitionId;
 
@@ -60,6 +66,15 @@ public TapeDrive() {
 
     // Getters and Setters
     
+    public boolean getCleaningRequired() {
+        return this.cleaningRequired;
+    }
+
+    public void setCleaningRequired(final boolean cleaningRequired) {
+        this.cleaningRequired = cleaningRequired;
+    }
+
+
     public String getErrorMessage() {
         return this.errorMessage;
     }
@@ -96,6 +111,15 @@ public void setLastCleaned(final Date lastCleaned) {
     }
 
 
+    public String getMfgSerialNumber() {
+        return this.mfgSerialNumber;
+    }
+
+    public void setMfgSerialNumber(final String mfgSerialNumber) {
+        this.mfgSerialNumber = mfgSerialNumber;
+    }
+
+
     public UUID getPartitionId() {
         return this.partitionId;
     }

From 32ebe5a779272fc30162934b7b1a3295f447b25c Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 19 Jun 2017 13:34:55 -0600
Subject: [PATCH 058/229] Updated setting query params to use UpdateQueryParam
 which escapes values

---
 .../commands/AbortMultiPartUploadRequest.java |  6 +++--
 .../CompleteMultiPartUploadRequest.java       |  6 +++--
 .../ds3client/commands/GetObjectRequest.java  | 24 ++++++++++++-------
 .../ListMultiPartUploadPartsRequest.java      |  6 +++--
 .../PutMultiPartUploadPartRequest.java        | 24 ++++++++++++-------
 .../ds3client/commands/PutObjectRequest.java  | 24 ++++++++++++-------
 ...rageDomainToDs3TargetSpectraS3Request.java |  6 +++--
 .../DelegateCreateUserSpectraS3Request.java   |  3 ++-
 ...leteFolderRecursivelySpectraS3Request.java |  3 ++-
 ...ectStorageDomainBlobsSpectraS3Request.java | 12 ++++++----
 .../EjectStorageDomainSpectraS3Request.java   |  6 +++--
 ...BucketCapacitySummarySpectraS3Request.java | 12 ++++++----
 ...dyForClientProcessingSpectraS3Request.java |  6 +++--
 .../GetObjectDetailsSpectraS3Request.java     |  3 ++-
 ...DomainCapacitySummarySpectraS3Request.java |  6 +++--
 .../ImportAzureTargetSpectraS3Request.java    |  3 ++-
 .../ImportS3TargetSpectraS3Request.java       |  3 ++-
 ...ModifyAllAzureTargetsSpectraS3Request.java |  3 ++-
 .../ModifyAllDs3TargetsSpectraS3Request.java  |  3 ++-
 .../ModifyAllPoolsSpectraS3Request.java       |  3 ++-
 .../ModifyAllS3TargetsSpectraS3Request.java   |  3 ++-
 ...difyAllTapePartitionsSpectraS3Request.java |  3 ++-
 ...reDataReplicationRuleSpectraS3Request.java | 18 +++++++++-----
 ...AzureTargetBucketNameSpectraS3Request.java | 18 +++++++++-----
 ...ificationRegistrationSpectraS3Request.java |  3 ++-
 ...eTargetReadPreferenceSpectraS3Request.java | 18 +++++++++-----
 .../PutBucketAclForGroupSpectraS3Request.java | 18 +++++++++-----
 .../PutBucketAclForUserSpectraS3Request.java  | 18 +++++++++-----
 .../spectrads3/PutBucketSpectraS3Request.java |  3 ++-
 ...utDataPersistenceRuleSpectraS3Request.java | 24 ++++++++++++-------
 ...DataPolicyAclForGroupSpectraS3Request.java | 12 ++++++----
 ...tDataPolicyAclForUserSpectraS3Request.java | 12 ++++++----
 .../PutDataPolicySpectraS3Request.java        |  3 ++-
 ...s3DataReplicationRuleSpectraS3Request.java | 18 +++++++++-----
 ...3TargetReadPreferenceSpectraS3Request.java | 18 +++++++++-----
 ...obalBucketAclForGroupSpectraS3Request.java | 12 ++++++----
 ...lobalBucketAclForUserSpectraS3Request.java | 12 ++++++----
 ...DataPolicyAclForGroupSpectraS3Request.java |  6 +++--
 ...lDataPolicyAclForUserSpectraS3Request.java |  6 +++--
 .../PutGroupGroupMemberSpectraS3Request.java  | 12 ++++++----
 .../spectrads3/PutGroupSpectraS3Request.java  |  3 ++-
 .../PutPoolPartitionSpectraS3Request.java     |  6 +++--
 ...olStorageDomainMemberSpectraS3Request.java | 12 ++++++----
 ...S3DataReplicationRuleSpectraS3Request.java | 18 +++++++++-----
 ...PutS3TargetBucketNameSpectraS3Request.java | 18 +++++++++-----
 ...ificationRegistrationSpectraS3Request.java |  3 ++-
 ...3TargetReadPreferenceSpectraS3Request.java | 18 +++++++++-----
 .../PutStorageDomainSpectraS3Request.java     |  3 ++-
 ...tTapeDensityDirectiveSpectraS3Request.java | 18 +++++++++-----
 ...peStorageDomainMemberSpectraS3Request.java | 18 +++++++++-----
 .../PutUserGroupMemberSpectraS3Request.java   | 12 ++++++----
 .../RawImportAllTapesSpectraS3Request.java    |  3 ++-
 .../RawImportTapeSpectraS3Request.java        |  6 +++--
 .../RegisterAzureTargetSpectraS3Request.java  |  9 ++++---
 .../RegisterDs3TargetSpectraS3Request.java    | 12 ++++++----
 .../RegisterS3TargetSpectraS3Request.java     |  9 ++++---
 56 files changed, 378 insertions(+), 189 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/AbortMultiPartUploadRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/AbortMultiPartUploadRequest.java
index 2c4c8d79f..e4768b494 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/AbortMultiPartUploadRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/AbortMultiPartUploadRequest.java
@@ -39,7 +39,8 @@ public AbortMultiPartUploadRequest(final String bucketName, final String objectN
         this.objectName = objectName;
         this.uploadId = uploadId.toString();
         
-        this.getQueryParams().put("upload_id", uploadId.toString());
+        this.updateQueryParam("upload_id", uploadId);
+
     }
 
     
@@ -48,7 +49,8 @@ public AbortMultiPartUploadRequest(final String bucketName, final String objectN
         this.objectName = objectName;
         this.uploadId = uploadId;
         
-        this.getQueryParams().put("upload_id", UrlEscapers.urlFragmentEscaper().escape(uploadId).replace("+", "%2B"));
+        this.updateQueryParam("upload_id", uploadId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/CompleteMultiPartUploadRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/CompleteMultiPartUploadRequest.java
index b4f5164a2..272609a65 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/CompleteMultiPartUploadRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/CompleteMultiPartUploadRequest.java
@@ -48,7 +48,8 @@ public CompleteMultiPartUploadRequest(final String bucketName, final String obje
         this.uploadId = uploadId.toString();
         this.requestPayload = requestPayload;
         
-        this.getQueryParams().put("upload_id", uploadId.toString());
+        this.updateQueryParam("upload_id", uploadId);
+
     }
 
     
@@ -58,7 +59,8 @@ public CompleteMultiPartUploadRequest(final String bucketName, final String obje
         this.uploadId = uploadId;
         this.requestPayload = requestPayload;
         
-        this.getQueryParams().put("upload_id", UrlEscapers.urlFragmentEscaper().escape(uploadId).replace("+", "%2B"));
+        this.updateQueryParam("upload_id", uploadId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
index d7332cd64..9d1f17ca0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
@@ -68,8 +68,10 @@ public GetObjectRequest(final String bucketName, final String objectName, final
         this.offset = offset;
         this.channel = channel;
         
-        this.getQueryParams().put("job", job.toString());
-        this.getQueryParams().put("offset", Long.toString(offset));
+        this.updateQueryParam("job", job);
+
+        this.updateQueryParam("offset", offset);
+
 
     }
 
@@ -81,8 +83,10 @@ public GetObjectRequest(final String bucketName, final String objectName, final
         this.offset = offset;
         this.channel = channel;
         
-        this.getQueryParams().put("job", UrlEscapers.urlFragmentEscaper().escape(job).replace("+", "%2B"));
-        this.getQueryParams().put("offset", Long.toString(offset));
+        this.updateQueryParam("job", job);
+
+        this.updateQueryParam("offset", offset);
+
 
     }
 
@@ -94,8 +98,10 @@ public GetObjectRequest(final String bucketName, final String objectName, final
         this.offset = offset;
         this.channel = Channels.newChannel(stream);
         
-        this.getQueryParams().put("job", job.toString());
-        this.getQueryParams().put("offset", Long.toString(offset));
+        this.updateQueryParam("job", job);
+
+        this.updateQueryParam("offset", offset);
+
 
     }
 
@@ -107,8 +113,10 @@ public GetObjectRequest(final String bucketName, final String objectName, final
         this.offset = offset;
         this.channel = Channels.newChannel(stream);
         
-        this.getQueryParams().put("job", UrlEscapers.urlFragmentEscaper().escape(job).replace("+", "%2B"));
-        this.getQueryParams().put("offset", Long.toString(offset));
+        this.updateQueryParam("job", job);
+
+        this.updateQueryParam("offset", offset);
+
 
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/ListMultiPartUploadPartsRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/ListMultiPartUploadPartsRequest.java
index 8b2d97ffe..bb75516b9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/ListMultiPartUploadPartsRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/ListMultiPartUploadPartsRequest.java
@@ -44,7 +44,8 @@ public ListMultiPartUploadPartsRequest(final String bucketName, final String obj
         this.objectName = objectName;
         this.uploadId = uploadId.toString();
         
-        this.getQueryParams().put("upload_id", uploadId.toString());
+        this.updateQueryParam("upload_id", uploadId);
+
     }
 
     
@@ -53,7 +54,8 @@ public ListMultiPartUploadPartsRequest(final String bucketName, final String obj
         this.objectName = objectName;
         this.uploadId = uploadId;
         
-        this.getQueryParams().put("upload_id", UrlEscapers.urlFragmentEscaper().escape(uploadId).replace("+", "%2B"));
+        this.updateQueryParam("upload_id", uploadId);
+
     }
 
     public ListMultiPartUploadPartsRequest withMaxParts(final int maxParts) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest.java
index 09ed0d4f4..e3340d670 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest.java
@@ -53,8 +53,10 @@ public PutMultiPartUploadPartRequest(final String bucketName, final String objec
         this.size = size;
         this.channel = channel;
         
-        this.getQueryParams().put("part_number", Integer.toString(partNumber));
-        this.getQueryParams().put("upload_id", uploadId.toString());
+        this.updateQueryParam("part_number", partNumber);
+
+        this.updateQueryParam("upload_id", uploadId);
+
         this.stream = new SeekableByteChannelInputStream(channel);
     }
 
@@ -67,8 +69,10 @@ public PutMultiPartUploadPartRequest(final String bucketName, final String objec
         this.size = size;
         this.channel = channel;
         
-        this.getQueryParams().put("part_number", Integer.toString(partNumber));
-        this.getQueryParams().put("upload_id", UrlEscapers.urlFragmentEscaper().escape(uploadId).replace("+", "%2B"));
+        this.updateQueryParam("part_number", partNumber);
+
+        this.updateQueryParam("upload_id", uploadId);
+
         this.stream = new SeekableByteChannelInputStream(channel);
     }
 
@@ -81,8 +85,10 @@ public PutMultiPartUploadPartRequest(final String bucketName, final String objec
         this.size = size;
         this.stream = stream;
         
-        this.getQueryParams().put("part_number", Integer.toString(partNumber));
-        this.getQueryParams().put("upload_id", uploadId.toString());
+        this.updateQueryParam("part_number", partNumber);
+
+        this.updateQueryParam("upload_id", uploadId);
+
     }
 
     
@@ -94,8 +100,10 @@ public PutMultiPartUploadPartRequest(final String bucketName, final String objec
         this.size = size;
         this.stream = stream;
         
-        this.getQueryParams().put("part_number", Integer.toString(partNumber));
-        this.getQueryParams().put("upload_id", UrlEscapers.urlFragmentEscaper().escape(uploadId).replace("+", "%2B"));
+        this.updateQueryParam("part_number", partNumber);
+
+        this.updateQueryParam("upload_id", uploadId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutObjectRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutObjectRequest.java
index 7ebc8d945..db69c7c4a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutObjectRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutObjectRequest.java
@@ -73,8 +73,10 @@ public PutObjectRequest(final String bucketName, final String objectName, final
         this.channel = channel;
         this.stream = new SeekableByteChannelInputStream(channel);
         
-        this.getQueryParams().put("job", job.toString());
-        this.getQueryParams().put("offset", Long.toString(offset));
+        this.updateQueryParam("job", job);
+
+        this.updateQueryParam("offset", offset);
+
 
     }
 
@@ -88,8 +90,10 @@ public PutObjectRequest(final String bucketName, final String objectName, final
         this.channel = channel;
         this.stream = new SeekableByteChannelInputStream(channel);
         
-        this.getQueryParams().put("job", UrlEscapers.urlFragmentEscaper().escape(job).replace("+", "%2B"));
-        this.getQueryParams().put("offset", Long.toString(offset));
+        this.updateQueryParam("job", job);
+
+        this.updateQueryParam("offset", offset);
+
 
     }
 
@@ -102,8 +106,10 @@ public PutObjectRequest(final String bucketName, final String objectName, final
         this.offset = offset;
         this.stream = stream;
         
-        this.getQueryParams().put("job", job.toString());
-        this.getQueryParams().put("offset", Long.toString(offset));
+        this.updateQueryParam("job", job);
+
+        this.updateQueryParam("offset", offset);
+
 
     }
 
@@ -116,8 +122,10 @@ public PutObjectRequest(final String bucketName, final String objectName, final
         this.offset = offset;
         this.stream = stream;
         
-        this.getQueryParams().put("job", UrlEscapers.urlFragmentEscaper().escape(job).replace("+", "%2B"));
-        this.getQueryParams().put("offset", Long.toString(offset));
+        this.updateQueryParam("job", job);
+
+        this.updateQueryParam("offset", offset);
+
 
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ConvertStorageDomainToDs3TargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ConvertStorageDomainToDs3TargetSpectraS3Request.java
index 9d398f61a..7350b3ac3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ConvertStorageDomainToDs3TargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ConvertStorageDomainToDs3TargetSpectraS3Request.java
@@ -36,7 +36,8 @@ public ConvertStorageDomainToDs3TargetSpectraS3Request(final UUID convertToDs3Ta
         this.storageDomain = storageDomain;
         this.convertToDs3Target = convertToDs3Target.toString();
         
-        this.getQueryParams().put("convert_to_ds3_target", convertToDs3Target.toString());
+        this.updateQueryParam("convert_to_ds3_target", convertToDs3Target);
+
     }
 
     
@@ -44,7 +45,8 @@ public ConvertStorageDomainToDs3TargetSpectraS3Request(final String convertToDs3
         this.storageDomain = storageDomain;
         this.convertToDs3Target = convertToDs3Target;
         
-        this.getQueryParams().put("convert_to_ds3_target", UrlEscapers.urlFragmentEscaper().escape(convertToDs3Target).replace("+", "%2B"));
+        this.updateQueryParam("convert_to_ds3_target", convertToDs3Target);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DelegateCreateUserSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DelegateCreateUserSpectraS3Request.java
index 567de8e16..ce582887e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DelegateCreateUserSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DelegateCreateUserSpectraS3Request.java
@@ -37,7 +37,8 @@ public class DelegateCreateUserSpectraS3Request extends AbstractRequest {
     public DelegateCreateUserSpectraS3Request(final String name) {
         this.name = name;
         
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
+        this.updateQueryParam("name", name);
+
     }
 
     public DelegateCreateUserSpectraS3Request withId(final UUID id) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteFolderRecursivelySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteFolderRecursivelySpectraS3Request.java
index f07f093b4..d511ad6ca 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteFolderRecursivelySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteFolderRecursivelySpectraS3Request.java
@@ -37,7 +37,8 @@ public DeleteFolderRecursivelySpectraS3Request(final String bucketId, final Stri
         this.folder = folder;
         this.bucketId = bucketId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
+        this.updateQueryParam("bucket_id", bucketId);
+
         this.getQueryParams().put("recursive", null);
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
index 218424b33..5e3909fcc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
@@ -56,8 +56,10 @@ public EjectStorageDomainBlobsSpectraS3Request(final String bucketId, final List
         this.getQueryParams().put("operation", "eject");
 
         this.getQueryParams().put("blobs", null);
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("storage_domain_id", storageDomainId.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     
@@ -69,8 +71,10 @@ public EjectStorageDomainBlobsSpectraS3Request(final String bucketId, final List
         this.getQueryParams().put("operation", "eject");
 
         this.getQueryParams().put("blobs", null);
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("storage_domain_id", UrlEscapers.urlFragmentEscaper().escape(storageDomainId).replace("+", "%2B"));
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     public EjectStorageDomainBlobsSpectraS3Request withEjectLabel(final String ejectLabel) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainSpectraS3Request.java
index fb2e3f966..5acdc03ab 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainSpectraS3Request.java
@@ -41,7 +41,8 @@ public EjectStorageDomainSpectraS3Request(final UUID storageDomainId) {
         
         this.getQueryParams().put("operation", "eject");
 
-        this.getQueryParams().put("storage_domain_id", storageDomainId.toString());
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     
@@ -50,7 +51,8 @@ public EjectStorageDomainSpectraS3Request(final String storageDomainId) {
         
         this.getQueryParams().put("operation", "eject");
 
-        this.getQueryParams().put("storage_domain_id", UrlEscapers.urlFragmentEscaper().escape(storageDomainId).replace("+", "%2B"));
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     public EjectStorageDomainSpectraS3Request withBucketId(final String bucketId) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBucketCapacitySummarySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBucketCapacitySummarySpectraS3Request.java
index 876798b94..4a0b74fe0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBucketCapacitySummarySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBucketCapacitySummarySpectraS3Request.java
@@ -51,8 +51,10 @@ public GetBucketCapacitySummarySpectraS3Request(final String bucketId, final UUI
         this.bucketId = bucketId;
         this.storageDomainId = storageDomainId.toString();
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("storage_domain_id", storageDomainId.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     
@@ -60,8 +62,10 @@ public GetBucketCapacitySummarySpectraS3Request(final String bucketId, final Str
         this.bucketId = bucketId;
         this.storageDomainId = storageDomainId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("storage_domain_id", UrlEscapers.urlFragmentEscaper().escape(storageDomainId).replace("+", "%2B"));
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     public GetBucketCapacitySummarySpectraS3Request withPoolHealth(final PoolHealth poolHealth) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetJobChunksReadyForClientProcessingSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetJobChunksReadyForClientProcessingSpectraS3Request.java
index b43559332..a88098379 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetJobChunksReadyForClientProcessingSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetJobChunksReadyForClientProcessingSpectraS3Request.java
@@ -37,14 +37,16 @@ public class GetJobChunksReadyForClientProcessingSpectraS3Request extends Abstra
     public GetJobChunksReadyForClientProcessingSpectraS3Request(final UUID job) {
         this.job = job.toString();
         
-        this.getQueryParams().put("job", job.toString());
+        this.updateQueryParam("job", job);
+
     }
 
     
     public GetJobChunksReadyForClientProcessingSpectraS3Request(final String job) {
         this.job = job;
         
-        this.getQueryParams().put("job", UrlEscapers.urlFragmentEscaper().escape(job).replace("+", "%2B"));
+        this.updateQueryParam("job", job);
+
     }
 
     public GetJobChunksReadyForClientProcessingSpectraS3Request withJobChunk(final UUID jobChunk) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectDetailsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectDetailsSpectraS3Request.java
index f87b7332d..5f4b25716 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectDetailsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectDetailsSpectraS3Request.java
@@ -35,7 +35,8 @@ public GetObjectDetailsSpectraS3Request(final String objectName, final String bu
         this.objectName = objectName;
         this.bucketId = bucketId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
+        this.updateQueryParam("bucket_id", bucketId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainCapacitySummarySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainCapacitySummarySpectraS3Request.java
index ebc8b972b..939d0881b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainCapacitySummarySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainCapacitySummarySpectraS3Request.java
@@ -48,14 +48,16 @@ public class GetStorageDomainCapacitySummarySpectraS3Request extends AbstractReq
     public GetStorageDomainCapacitySummarySpectraS3Request(final UUID storageDomainId) {
         this.storageDomainId = storageDomainId.toString();
         
-        this.getQueryParams().put("storage_domain_id", storageDomainId.toString());
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     
     public GetStorageDomainCapacitySummarySpectraS3Request(final String storageDomainId) {
         this.storageDomainId = storageDomainId;
         
-        this.getQueryParams().put("storage_domain_id", UrlEscapers.urlFragmentEscaper().escape(storageDomainId).replace("+", "%2B"));
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     public GetStorageDomainCapacitySummarySpectraS3Request withPoolHealth(final PoolHealth poolHealth) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAzureTargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAzureTargetSpectraS3Request.java
index f7167b6f4..55c99503d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAzureTargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAzureTargetSpectraS3Request.java
@@ -48,7 +48,8 @@ public ImportAzureTargetSpectraS3Request(final String azureTarget, final String
         
         this.getQueryParams().put("operation", "import");
 
-        this.getQueryParams().put("cloud_bucket_name", UrlEscapers.urlFragmentEscaper().escape(cloudBucketName).replace("+", "%2B"));
+        this.updateQueryParam("cloud_bucket_name", cloudBucketName);
+
     }
 
     public ImportAzureTargetSpectraS3Request withConflictResolutionMode(final ImportConflictResolutionMode conflictResolutionMode) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportS3TargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportS3TargetSpectraS3Request.java
index 1d9e4e886..8cdf9878e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportS3TargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportS3TargetSpectraS3Request.java
@@ -48,7 +48,8 @@ public ImportS3TargetSpectraS3Request(final String cloudBucketName, final String
         
         this.getQueryParams().put("operation", "import");
 
-        this.getQueryParams().put("cloud_bucket_name", UrlEscapers.urlFragmentEscaper().escape(cloudBucketName).replace("+", "%2B"));
+        this.updateQueryParam("cloud_bucket_name", cloudBucketName);
+
     }
 
     public ImportS3TargetSpectraS3Request withConflictResolutionMode(final ImportConflictResolutionMode conflictResolutionMode) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllAzureTargetsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllAzureTargetsSpectraS3Request.java
index 47ebe6827..c24cc0499 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllAzureTargetsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllAzureTargetsSpectraS3Request.java
@@ -32,7 +32,8 @@ public class ModifyAllAzureTargetsSpectraS3Request extends AbstractRequest {
     public ModifyAllAzureTargetsSpectraS3Request(final Quiesced quiesced) {
         this.quiesced = quiesced;
         
-        this.getQueryParams().put("quiesced", quiesced.toString());
+        this.updateQueryParam("quiesced", quiesced);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllDs3TargetsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllDs3TargetsSpectraS3Request.java
index c3acd9f07..896363c3c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllDs3TargetsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllDs3TargetsSpectraS3Request.java
@@ -32,7 +32,8 @@ public class ModifyAllDs3TargetsSpectraS3Request extends AbstractRequest {
     public ModifyAllDs3TargetsSpectraS3Request(final Quiesced quiesced) {
         this.quiesced = quiesced;
         
-        this.getQueryParams().put("quiesced", quiesced.toString());
+        this.updateQueryParam("quiesced", quiesced);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllPoolsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllPoolsSpectraS3Request.java
index 6046b865b..6d3987470 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllPoolsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllPoolsSpectraS3Request.java
@@ -32,7 +32,8 @@ public class ModifyAllPoolsSpectraS3Request extends AbstractRequest {
     public ModifyAllPoolsSpectraS3Request(final Quiesced quiesced) {
         this.quiesced = quiesced;
         
-        this.getQueryParams().put("quiesced", quiesced.toString());
+        this.updateQueryParam("quiesced", quiesced);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllS3TargetsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllS3TargetsSpectraS3Request.java
index 2b5b8b8b5..6f96ab761 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllS3TargetsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllS3TargetsSpectraS3Request.java
@@ -32,7 +32,8 @@ public class ModifyAllS3TargetsSpectraS3Request extends AbstractRequest {
     public ModifyAllS3TargetsSpectraS3Request(final Quiesced quiesced) {
         this.quiesced = quiesced;
         
-        this.getQueryParams().put("quiesced", quiesced.toString());
+        this.updateQueryParam("quiesced", quiesced);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllTapePartitionsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllTapePartitionsSpectraS3Request.java
index b5f209c3f..2c654a86c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllTapePartitionsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyAllTapePartitionsSpectraS3Request.java
@@ -32,7 +32,8 @@ public class ModifyAllTapePartitionsSpectraS3Request extends AbstractRequest {
     public ModifyAllTapePartitionsSpectraS3Request(final Quiesced quiesced) {
         this.quiesced = quiesced;
         
-        this.getQueryParams().put("quiesced", quiesced.toString());
+        this.updateQueryParam("quiesced", quiesced);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureDataReplicationRuleSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureDataReplicationRuleSpectraS3Request.java
index 95d9fc6af..2c514b463 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureDataReplicationRuleSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureDataReplicationRuleSpectraS3Request.java
@@ -44,9 +44,12 @@ public PutAzureDataReplicationRuleSpectraS3Request(final UUID dataPolicyId, fina
         this.targetId = targetId.toString();
         this.type = type;
         
-        this.getQueryParams().put("data_policy_id", dataPolicyId.toString());
-        this.getQueryParams().put("target_id", targetId.toString());
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("target_id", targetId);
+
+        this.updateQueryParam("type", type);
+
     }
 
     
@@ -55,9 +58,12 @@ public PutAzureDataReplicationRuleSpectraS3Request(final String dataPolicyId, fi
         this.targetId = targetId;
         this.type = type;
         
-        this.getQueryParams().put("data_policy_id", UrlEscapers.urlFragmentEscaper().escape(dataPolicyId).replace("+", "%2B"));
-        this.getQueryParams().put("target_id", UrlEscapers.urlFragmentEscaper().escape(targetId).replace("+", "%2B"));
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("target_id", targetId);
+
+        this.updateQueryParam("type", type);
+
     }
 
     public PutAzureDataReplicationRuleSpectraS3Request withMaxBlobPartSizeInBytes(final long maxBlobPartSizeInBytes) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetBucketNameSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetBucketNameSpectraS3Request.java
index d57343e36..f6f30a3e9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetBucketNameSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetBucketNameSpectraS3Request.java
@@ -39,9 +39,12 @@ public PutAzureTargetBucketNameSpectraS3Request(final String bucketId, final Str
         this.name = name;
         this.targetId = targetId.toString();
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
-        this.getQueryParams().put("target_id", targetId.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("name", name);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
     
@@ -50,9 +53,12 @@ public PutAzureTargetBucketNameSpectraS3Request(final String bucketId, final Str
         this.name = name;
         this.targetId = targetId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
-        this.getQueryParams().put("target_id", UrlEscapers.urlFragmentEscaper().escape(targetId).replace("+", "%2B"));
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("name", name);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetFailureNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetFailureNotificationRegistrationSpectraS3Request.java
index a3e631214..d0d501d26 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetFailureNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetFailureNotificationRegistrationSpectraS3Request.java
@@ -41,7 +41,8 @@ public class PutAzureTargetFailureNotificationRegistrationSpectraS3Request exten
     public PutAzureTargetFailureNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         this.notificationEndPoint = notificationEndPoint;
         
-        this.getQueryParams().put("notification_end_point", UrlEscapers.urlFragmentEscaper().escape(notificationEndPoint).replace("+", "%2B"));
+        this.updateQueryParam("notification_end_point", notificationEndPoint);
+
     }
 
     public PutAzureTargetFailureNotificationRegistrationSpectraS3Request withFormat(final HttpResponseFormatType format) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetReadPreferenceSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetReadPreferenceSpectraS3Request.java
index 35c8c9964..34eb768cc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetReadPreferenceSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutAzureTargetReadPreferenceSpectraS3Request.java
@@ -40,9 +40,12 @@ public PutAzureTargetReadPreferenceSpectraS3Request(final String bucketId, final
         this.readPreference = readPreference;
         this.targetId = targetId.toString();
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("read_preference", readPreference.toString());
-        this.getQueryParams().put("target_id", targetId.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("read_preference", readPreference);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
     
@@ -51,9 +54,12 @@ public PutAzureTargetReadPreferenceSpectraS3Request(final String bucketId, final
         this.readPreference = readPreference;
         this.targetId = targetId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("read_preference", readPreference.toString());
-        this.getQueryParams().put("target_id", UrlEscapers.urlFragmentEscaper().escape(targetId).replace("+", "%2B"));
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("read_preference", readPreference);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketAclForGroupSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketAclForGroupSpectraS3Request.java
index aa2e5a500..26b39f0b0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketAclForGroupSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketAclForGroupSpectraS3Request.java
@@ -40,9 +40,12 @@ public PutBucketAclForGroupSpectraS3Request(final String bucketId, final UUID gr
         this.groupId = groupId.toString();
         this.permission = permission;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("group_id", groupId.toString());
-        this.getQueryParams().put("permission", permission.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("group_id", groupId);
+
+        this.updateQueryParam("permission", permission);
+
     }
 
     
@@ -51,9 +54,12 @@ public PutBucketAclForGroupSpectraS3Request(final String bucketId, final String
         this.groupId = groupId;
         this.permission = permission;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("group_id", UrlEscapers.urlFragmentEscaper().escape(groupId).replace("+", "%2B"));
-        this.getQueryParams().put("permission", permission.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("group_id", groupId);
+
+        this.updateQueryParam("permission", permission);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketAclForUserSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketAclForUserSpectraS3Request.java
index cd16c9c5b..7200a3238 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketAclForUserSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketAclForUserSpectraS3Request.java
@@ -40,9 +40,12 @@ public PutBucketAclForUserSpectraS3Request(final String bucketId, final BucketAc
         this.permission = permission;
         this.userId = userId.toString();
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("permission", permission.toString());
-        this.getQueryParams().put("user_id", userId.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("permission", permission);
+
+        this.updateQueryParam("user_id", userId);
+
     }
 
     
@@ -51,9 +54,12 @@ public PutBucketAclForUserSpectraS3Request(final String bucketId, final BucketAc
         this.permission = permission;
         this.userId = userId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("permission", permission.toString());
-        this.getQueryParams().put("user_id", UrlEscapers.urlFragmentEscaper().escape(userId).replace("+", "%2B"));
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("permission", permission);
+
+        this.updateQueryParam("user_id", userId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketSpectraS3Request.java
index 06d73b38d..d90e8c47b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBucketSpectraS3Request.java
@@ -39,7 +39,8 @@ public class PutBucketSpectraS3Request extends AbstractRequest {
     public PutBucketSpectraS3Request(final String name) {
         this.name = name;
         
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
+        this.updateQueryParam("name", name);
+
     }
 
     public PutBucketSpectraS3Request withDataPolicyId(final UUID dataPolicyId) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPersistenceRuleSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPersistenceRuleSpectraS3Request.java
index 56ff347db..3af704350 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPersistenceRuleSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPersistenceRuleSpectraS3Request.java
@@ -47,10 +47,14 @@ public PutDataPersistenceRuleSpectraS3Request(final UUID dataPolicyId, final Dat
         this.storageDomainId = storageDomainId.toString();
         this.type = type;
         
-        this.getQueryParams().put("data_policy_id", dataPolicyId.toString());
-        this.getQueryParams().put("isolation_level", isolationLevel.toString());
-        this.getQueryParams().put("storage_domain_id", storageDomainId.toString());
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("isolation_level", isolationLevel);
+
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
+        this.updateQueryParam("type", type);
+
     }
 
     
@@ -60,10 +64,14 @@ public PutDataPersistenceRuleSpectraS3Request(final String dataPolicyId, final D
         this.storageDomainId = storageDomainId;
         this.type = type;
         
-        this.getQueryParams().put("data_policy_id", UrlEscapers.urlFragmentEscaper().escape(dataPolicyId).replace("+", "%2B"));
-        this.getQueryParams().put("isolation_level", isolationLevel.toString());
-        this.getQueryParams().put("storage_domain_id", UrlEscapers.urlFragmentEscaper().escape(storageDomainId).replace("+", "%2B"));
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("isolation_level", isolationLevel);
+
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
+        this.updateQueryParam("type", type);
+
     }
 
     public PutDataPersistenceRuleSpectraS3Request withMinimumDaysToRetain(final Integer minimumDaysToRetain) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicyAclForGroupSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicyAclForGroupSpectraS3Request.java
index 756961f88..571d567ee 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicyAclForGroupSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicyAclForGroupSpectraS3Request.java
@@ -36,8 +36,10 @@ public PutDataPolicyAclForGroupSpectraS3Request(final UUID dataPolicyId, final U
         this.dataPolicyId = dataPolicyId.toString();
         this.groupId = groupId.toString();
         
-        this.getQueryParams().put("data_policy_id", dataPolicyId.toString());
-        this.getQueryParams().put("group_id", groupId.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("group_id", groupId);
+
     }
 
     
@@ -45,8 +47,10 @@ public PutDataPolicyAclForGroupSpectraS3Request(final String dataPolicyId, final
         this.dataPolicyId = dataPolicyId;
         this.groupId = groupId;
         
-        this.getQueryParams().put("data_policy_id", UrlEscapers.urlFragmentEscaper().escape(dataPolicyId).replace("+", "%2B"));
-        this.getQueryParams().put("group_id", UrlEscapers.urlFragmentEscaper().escape(groupId).replace("+", "%2B"));
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("group_id", groupId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicyAclForUserSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicyAclForUserSpectraS3Request.java
index dfc89df3d..5c5f309a4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicyAclForUserSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicyAclForUserSpectraS3Request.java
@@ -36,8 +36,10 @@ public PutDataPolicyAclForUserSpectraS3Request(final UUID dataPolicyId, final UU
         this.dataPolicyId = dataPolicyId.toString();
         this.userId = userId.toString();
         
-        this.getQueryParams().put("data_policy_id", dataPolicyId.toString());
-        this.getQueryParams().put("user_id", userId.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("user_id", userId);
+
     }
 
     
@@ -45,8 +47,10 @@ public PutDataPolicyAclForUserSpectraS3Request(final String dataPolicyId, final
         this.dataPolicyId = dataPolicyId;
         this.userId = userId;
         
-        this.getQueryParams().put("data_policy_id", UrlEscapers.urlFragmentEscaper().escape(dataPolicyId).replace("+", "%2B"));
-        this.getQueryParams().put("user_id", UrlEscapers.urlFragmentEscaper().escape(userId).replace("+", "%2B"));
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("user_id", userId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicySpectraS3Request.java
index 67990f327..0626a9a40 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicySpectraS3Request.java
@@ -60,7 +60,8 @@ public class PutDataPolicySpectraS3Request extends AbstractRequest {
     public PutDataPolicySpectraS3Request(final String name) {
         this.name = name;
         
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
+        this.updateQueryParam("name", name);
+
     }
 
     public PutDataPolicySpectraS3Request withAlwaysForcePutJobCreation(final boolean alwaysForcePutJobCreation) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDs3DataReplicationRuleSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDs3DataReplicationRuleSpectraS3Request.java
index 550666b39..7e269c9f8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDs3DataReplicationRuleSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDs3DataReplicationRuleSpectraS3Request.java
@@ -44,9 +44,12 @@ public PutDs3DataReplicationRuleSpectraS3Request(final UUID dataPolicyId, final
         this.targetId = targetId.toString();
         this.type = type;
         
-        this.getQueryParams().put("data_policy_id", dataPolicyId.toString());
-        this.getQueryParams().put("target_id", targetId.toString());
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("target_id", targetId);
+
+        this.updateQueryParam("type", type);
+
     }
 
     
@@ -55,9 +58,12 @@ public PutDs3DataReplicationRuleSpectraS3Request(final String dataPolicyId, fina
         this.targetId = targetId;
         this.type = type;
         
-        this.getQueryParams().put("data_policy_id", UrlEscapers.urlFragmentEscaper().escape(dataPolicyId).replace("+", "%2B"));
-        this.getQueryParams().put("target_id", UrlEscapers.urlFragmentEscaper().escape(targetId).replace("+", "%2B"));
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("target_id", targetId);
+
+        this.updateQueryParam("type", type);
+
     }
 
     public PutDs3DataReplicationRuleSpectraS3Request withReplicateDeletes(final boolean replicateDeletes) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDs3TargetReadPreferenceSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDs3TargetReadPreferenceSpectraS3Request.java
index f5f133c5d..3e20b4a97 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDs3TargetReadPreferenceSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDs3TargetReadPreferenceSpectraS3Request.java
@@ -40,9 +40,12 @@ public PutDs3TargetReadPreferenceSpectraS3Request(final String bucketId, final T
         this.readPreference = readPreference;
         this.targetId = targetId.toString();
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("read_preference", readPreference.toString());
-        this.getQueryParams().put("target_id", targetId.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("read_preference", readPreference);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
     
@@ -51,9 +54,12 @@ public PutDs3TargetReadPreferenceSpectraS3Request(final String bucketId, final T
         this.readPreference = readPreference;
         this.targetId = targetId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("read_preference", readPreference.toString());
-        this.getQueryParams().put("target_id", UrlEscapers.urlFragmentEscaper().escape(targetId).replace("+", "%2B"));
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("read_preference", readPreference);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalBucketAclForGroupSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalBucketAclForGroupSpectraS3Request.java
index e9e00177f..2b3618f32 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalBucketAclForGroupSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalBucketAclForGroupSpectraS3Request.java
@@ -37,8 +37,10 @@ public PutGlobalBucketAclForGroupSpectraS3Request(final UUID groupId, final Buck
         this.groupId = groupId.toString();
         this.permission = permission;
         
-        this.getQueryParams().put("group_id", groupId.toString());
-        this.getQueryParams().put("permission", permission.toString());
+        this.updateQueryParam("group_id", groupId);
+
+        this.updateQueryParam("permission", permission);
+
     }
 
     
@@ -46,8 +48,10 @@ public PutGlobalBucketAclForGroupSpectraS3Request(final String groupId, final Bu
         this.groupId = groupId;
         this.permission = permission;
         
-        this.getQueryParams().put("group_id", UrlEscapers.urlFragmentEscaper().escape(groupId).replace("+", "%2B"));
-        this.getQueryParams().put("permission", permission.toString());
+        this.updateQueryParam("group_id", groupId);
+
+        this.updateQueryParam("permission", permission);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalBucketAclForUserSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalBucketAclForUserSpectraS3Request.java
index 8d8d48130..5bb0e15b9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalBucketAclForUserSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalBucketAclForUserSpectraS3Request.java
@@ -37,8 +37,10 @@ public PutGlobalBucketAclForUserSpectraS3Request(final BucketAclPermission permi
         this.permission = permission;
         this.userId = userId.toString();
         
-        this.getQueryParams().put("permission", permission.toString());
-        this.getQueryParams().put("user_id", userId.toString());
+        this.updateQueryParam("permission", permission);
+
+        this.updateQueryParam("user_id", userId);
+
     }
 
     
@@ -46,8 +48,10 @@ public PutGlobalBucketAclForUserSpectraS3Request(final BucketAclPermission permi
         this.permission = permission;
         this.userId = userId;
         
-        this.getQueryParams().put("permission", permission.toString());
-        this.getQueryParams().put("user_id", UrlEscapers.urlFragmentEscaper().escape(userId).replace("+", "%2B"));
+        this.updateQueryParam("permission", permission);
+
+        this.updateQueryParam("user_id", userId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalDataPolicyAclForGroupSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalDataPolicyAclForGroupSpectraS3Request.java
index b276214a2..119dd5168 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalDataPolicyAclForGroupSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalDataPolicyAclForGroupSpectraS3Request.java
@@ -33,14 +33,16 @@ public class PutGlobalDataPolicyAclForGroupSpectraS3Request extends AbstractRequ
     public PutGlobalDataPolicyAclForGroupSpectraS3Request(final UUID groupId) {
         this.groupId = groupId.toString();
         
-        this.getQueryParams().put("group_id", groupId.toString());
+        this.updateQueryParam("group_id", groupId);
+
     }
 
     
     public PutGlobalDataPolicyAclForGroupSpectraS3Request(final String groupId) {
         this.groupId = groupId;
         
-        this.getQueryParams().put("group_id", UrlEscapers.urlFragmentEscaper().escape(groupId).replace("+", "%2B"));
+        this.updateQueryParam("group_id", groupId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalDataPolicyAclForUserSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalDataPolicyAclForUserSpectraS3Request.java
index d42d4016f..a2cee3499 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalDataPolicyAclForUserSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGlobalDataPolicyAclForUserSpectraS3Request.java
@@ -33,14 +33,16 @@ public class PutGlobalDataPolicyAclForUserSpectraS3Request extends AbstractReque
     public PutGlobalDataPolicyAclForUserSpectraS3Request(final UUID userId) {
         this.userId = userId.toString();
         
-        this.getQueryParams().put("user_id", userId.toString());
+        this.updateQueryParam("user_id", userId);
+
     }
 
     
     public PutGlobalDataPolicyAclForUserSpectraS3Request(final String userId) {
         this.userId = userId;
         
-        this.getQueryParams().put("user_id", UrlEscapers.urlFragmentEscaper().escape(userId).replace("+", "%2B"));
+        this.updateQueryParam("user_id", userId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGroupGroupMemberSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGroupGroupMemberSpectraS3Request.java
index 1e736e36f..6ab497d95 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGroupGroupMemberSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGroupGroupMemberSpectraS3Request.java
@@ -36,8 +36,10 @@ public PutGroupGroupMemberSpectraS3Request(final UUID groupId, final UUID member
         this.groupId = groupId.toString();
         this.memberGroupId = memberGroupId.toString();
         
-        this.getQueryParams().put("group_id", groupId.toString());
-        this.getQueryParams().put("member_group_id", memberGroupId.toString());
+        this.updateQueryParam("group_id", groupId);
+
+        this.updateQueryParam("member_group_id", memberGroupId);
+
     }
 
     
@@ -45,8 +47,10 @@ public PutGroupGroupMemberSpectraS3Request(final String groupId, final String me
         this.groupId = groupId;
         this.memberGroupId = memberGroupId;
         
-        this.getQueryParams().put("group_id", UrlEscapers.urlFragmentEscaper().escape(groupId).replace("+", "%2B"));
-        this.getQueryParams().put("member_group_id", UrlEscapers.urlFragmentEscaper().escape(memberGroupId).replace("+", "%2B"));
+        this.updateQueryParam("group_id", groupId);
+
+        this.updateQueryParam("member_group_id", memberGroupId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGroupSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGroupSpectraS3Request.java
index 2e734eb82..a716f9cec 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGroupSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutGroupSpectraS3Request.java
@@ -32,7 +32,8 @@ public class PutGroupSpectraS3Request extends AbstractRequest {
     public PutGroupSpectraS3Request(final String name) {
         this.name = name;
         
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
+        this.updateQueryParam("name", name);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutPoolPartitionSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutPoolPartitionSpectraS3Request.java
index e33c7f136..bc4b9dcb5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutPoolPartitionSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutPoolPartitionSpectraS3Request.java
@@ -36,8 +36,10 @@ public PutPoolPartitionSpectraS3Request(final String name, final PoolType type)
         this.name = name;
         this.type = type;
         
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("name", name);
+
+        this.updateQueryParam("type", type);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutPoolStorageDomainMemberSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutPoolStorageDomainMemberSpectraS3Request.java
index f417afe6f..c22d6115f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutPoolStorageDomainMemberSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutPoolStorageDomainMemberSpectraS3Request.java
@@ -39,8 +39,10 @@ public PutPoolStorageDomainMemberSpectraS3Request(final UUID poolPartitionId, fi
         this.poolPartitionId = poolPartitionId.toString();
         this.storageDomainId = storageDomainId.toString();
         
-        this.getQueryParams().put("pool_partition_id", poolPartitionId.toString());
-        this.getQueryParams().put("storage_domain_id", storageDomainId.toString());
+        this.updateQueryParam("pool_partition_id", poolPartitionId);
+
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     
@@ -48,8 +50,10 @@ public PutPoolStorageDomainMemberSpectraS3Request(final String poolPartitionId,
         this.poolPartitionId = poolPartitionId;
         this.storageDomainId = storageDomainId;
         
-        this.getQueryParams().put("pool_partition_id", UrlEscapers.urlFragmentEscaper().escape(poolPartitionId).replace("+", "%2B"));
-        this.getQueryParams().put("storage_domain_id", UrlEscapers.urlFragmentEscaper().escape(storageDomainId).replace("+", "%2B"));
+        this.updateQueryParam("pool_partition_id", poolPartitionId);
+
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
     }
 
     public PutPoolStorageDomainMemberSpectraS3Request withWritePreference(final WritePreferenceLevel writePreference) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3DataReplicationRuleSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3DataReplicationRuleSpectraS3Request.java
index 6378999f3..f6812f6aa 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3DataReplicationRuleSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3DataReplicationRuleSpectraS3Request.java
@@ -47,9 +47,12 @@ public PutS3DataReplicationRuleSpectraS3Request(final UUID dataPolicyId, final U
         this.targetId = targetId.toString();
         this.type = type;
         
-        this.getQueryParams().put("data_policy_id", dataPolicyId.toString());
-        this.getQueryParams().put("target_id", targetId.toString());
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("target_id", targetId);
+
+        this.updateQueryParam("type", type);
+
     }
 
     
@@ -58,9 +61,12 @@ public PutS3DataReplicationRuleSpectraS3Request(final String dataPolicyId, final
         this.targetId = targetId;
         this.type = type;
         
-        this.getQueryParams().put("data_policy_id", UrlEscapers.urlFragmentEscaper().escape(dataPolicyId).replace("+", "%2B"));
-        this.getQueryParams().put("target_id", UrlEscapers.urlFragmentEscaper().escape(targetId).replace("+", "%2B"));
-        this.getQueryParams().put("type", type.toString());
+        this.updateQueryParam("data_policy_id", dataPolicyId);
+
+        this.updateQueryParam("target_id", targetId);
+
+        this.updateQueryParam("type", type);
+
     }
 
     public PutS3DataReplicationRuleSpectraS3Request withInitialDataPlacement(final S3InitialDataPlacementPolicy initialDataPlacement) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetBucketNameSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetBucketNameSpectraS3Request.java
index 7640f140c..b7709c570 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetBucketNameSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetBucketNameSpectraS3Request.java
@@ -39,9 +39,12 @@ public PutS3TargetBucketNameSpectraS3Request(final String bucketId, final String
         this.name = name;
         this.targetId = targetId.toString();
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
-        this.getQueryParams().put("target_id", targetId.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("name", name);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
     
@@ -50,9 +53,12 @@ public PutS3TargetBucketNameSpectraS3Request(final String bucketId, final String
         this.name = name;
         this.targetId = targetId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
-        this.getQueryParams().put("target_id", UrlEscapers.urlFragmentEscaper().escape(targetId).replace("+", "%2B"));
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("name", name);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetFailureNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetFailureNotificationRegistrationSpectraS3Request.java
index cf5133e30..cb9dba85c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetFailureNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetFailureNotificationRegistrationSpectraS3Request.java
@@ -41,7 +41,8 @@ public class PutS3TargetFailureNotificationRegistrationSpectraS3Request extends
     public PutS3TargetFailureNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         this.notificationEndPoint = notificationEndPoint;
         
-        this.getQueryParams().put("notification_end_point", UrlEscapers.urlFragmentEscaper().escape(notificationEndPoint).replace("+", "%2B"));
+        this.updateQueryParam("notification_end_point", notificationEndPoint);
+
     }
 
     public PutS3TargetFailureNotificationRegistrationSpectraS3Request withFormat(final HttpResponseFormatType format) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetReadPreferenceSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetReadPreferenceSpectraS3Request.java
index 417f8a8b6..000c698d7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetReadPreferenceSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutS3TargetReadPreferenceSpectraS3Request.java
@@ -40,9 +40,12 @@ public PutS3TargetReadPreferenceSpectraS3Request(final String bucketId, final Ta
         this.readPreference = readPreference;
         this.targetId = targetId.toString();
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("read_preference", readPreference.toString());
-        this.getQueryParams().put("target_id", targetId.toString());
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("read_preference", readPreference);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
     
@@ -51,9 +54,12 @@ public PutS3TargetReadPreferenceSpectraS3Request(final String bucketId, final Ta
         this.readPreference = readPreference;
         this.targetId = targetId;
         
-        this.getQueryParams().put("bucket_id", bucketId);
-        this.getQueryParams().put("read_preference", readPreference.toString());
-        this.getQueryParams().put("target_id", UrlEscapers.urlFragmentEscaper().escape(targetId).replace("+", "%2B"));
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("read_preference", readPreference);
+
+        this.updateQueryParam("target_id", targetId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutStorageDomainSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutStorageDomainSpectraS3Request.java
index 214d05278..faf7b1e2b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutStorageDomainSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutStorageDomainSpectraS3Request.java
@@ -61,7 +61,8 @@ public class PutStorageDomainSpectraS3Request extends AbstractRequest {
     public PutStorageDomainSpectraS3Request(final String name) {
         this.name = name;
         
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
+        this.updateQueryParam("name", name);
+
     }
 
     public PutStorageDomainSpectraS3Request withAutoEjectMediaFullThreshold(final Long autoEjectMediaFullThreshold) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeDensityDirectiveSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeDensityDirectiveSpectraS3Request.java
index 3d3c8e018..37cf90a3c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeDensityDirectiveSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeDensityDirectiveSpectraS3Request.java
@@ -41,9 +41,12 @@ public PutTapeDensityDirectiveSpectraS3Request(final TapeDriveType density, fina
         this.partitionId = partitionId.toString();
         this.tapeType = tapeType;
         
-        this.getQueryParams().put("density", density.toString());
-        this.getQueryParams().put("partition_id", partitionId.toString());
-        this.getQueryParams().put("tape_type", tapeType.toString());
+        this.updateQueryParam("density", density);
+
+        this.updateQueryParam("partition_id", partitionId);
+
+        this.updateQueryParam("tape_type", tapeType);
+
     }
 
     
@@ -52,9 +55,12 @@ public PutTapeDensityDirectiveSpectraS3Request(final TapeDriveType density, fina
         this.partitionId = partitionId;
         this.tapeType = tapeType;
         
-        this.getQueryParams().put("density", density.toString());
-        this.getQueryParams().put("partition_id", UrlEscapers.urlFragmentEscaper().escape(partitionId).replace("+", "%2B"));
-        this.getQueryParams().put("tape_type", tapeType.toString());
+        this.updateQueryParam("density", density);
+
+        this.updateQueryParam("partition_id", partitionId);
+
+        this.updateQueryParam("tape_type", tapeType);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
index 5bc2450b5..e3806bbb0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
@@ -43,9 +43,12 @@ public PutTapeStorageDomainMemberSpectraS3Request(final UUID storageDomainId, fi
         this.tapePartitionId = tapePartitionId.toString();
         this.tapeType = tapeType;
         
-        this.getQueryParams().put("storage_domain_id", storageDomainId.toString());
-        this.getQueryParams().put("tape_partition_id", tapePartitionId.toString());
-        this.getQueryParams().put("tape_type", tapeType.toString());
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
+        this.updateQueryParam("tape_partition_id", tapePartitionId);
+
+        this.updateQueryParam("tape_type", tapeType);
+
     }
 
     
@@ -54,9 +57,12 @@ public PutTapeStorageDomainMemberSpectraS3Request(final String storageDomainId,
         this.tapePartitionId = tapePartitionId;
         this.tapeType = tapeType;
         
-        this.getQueryParams().put("storage_domain_id", UrlEscapers.urlFragmentEscaper().escape(storageDomainId).replace("+", "%2B"));
-        this.getQueryParams().put("tape_partition_id", UrlEscapers.urlFragmentEscaper().escape(tapePartitionId).replace("+", "%2B"));
-        this.getQueryParams().put("tape_type", tapeType.toString());
+        this.updateQueryParam("storage_domain_id", storageDomainId);
+
+        this.updateQueryParam("tape_partition_id", tapePartitionId);
+
+        this.updateQueryParam("tape_type", tapeType);
+
     }
 
     public PutTapeStorageDomainMemberSpectraS3Request withWritePreference(final WritePreferenceLevel writePreference) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutUserGroupMemberSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutUserGroupMemberSpectraS3Request.java
index 855207b92..7c839df3d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutUserGroupMemberSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutUserGroupMemberSpectraS3Request.java
@@ -36,8 +36,10 @@ public PutUserGroupMemberSpectraS3Request(final UUID groupId, final UUID memberU
         this.groupId = groupId.toString();
         this.memberUserId = memberUserId.toString();
         
-        this.getQueryParams().put("group_id", groupId.toString());
-        this.getQueryParams().put("member_user_id", memberUserId.toString());
+        this.updateQueryParam("group_id", groupId);
+
+        this.updateQueryParam("member_user_id", memberUserId);
+
     }
 
     
@@ -45,8 +47,10 @@ public PutUserGroupMemberSpectraS3Request(final String groupId, final String mem
         this.groupId = groupId;
         this.memberUserId = memberUserId;
         
-        this.getQueryParams().put("group_id", UrlEscapers.urlFragmentEscaper().escape(groupId).replace("+", "%2B"));
-        this.getQueryParams().put("member_user_id", UrlEscapers.urlFragmentEscaper().escape(memberUserId).replace("+", "%2B"));
+        this.updateQueryParam("group_id", groupId);
+
+        this.updateQueryParam("member_user_id", memberUserId);
+
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportAllTapesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportAllTapesSpectraS3Request.java
index d009505cb..f085e131b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportAllTapesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportAllTapesSpectraS3Request.java
@@ -40,7 +40,8 @@ public RawImportAllTapesSpectraS3Request(final String bucketId) {
         
         this.getQueryParams().put("operation", "import");
 
-        this.getQueryParams().put("bucket_id", bucketId);
+        this.updateQueryParam("bucket_id", bucketId);
+
     }
 
     public RawImportAllTapesSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportTapeSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportTapeSpectraS3Request.java
index 80603dc05..6100bf34b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportTapeSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RawImportTapeSpectraS3Request.java
@@ -43,7 +43,8 @@ public RawImportTapeSpectraS3Request(final String bucketId, final UUID tapeId) {
         
         this.getQueryParams().put("operation", "import");
 
-        this.getQueryParams().put("bucket_id", bucketId);
+        this.updateQueryParam("bucket_id", bucketId);
+
     }
 
     
@@ -53,7 +54,8 @@ public RawImportTapeSpectraS3Request(final String bucketId, final String tapeId)
         
         this.getQueryParams().put("operation", "import");
 
-        this.getQueryParams().put("bucket_id", bucketId);
+        this.updateQueryParam("bucket_id", bucketId);
+
     }
 
     public RawImportTapeSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterAzureTargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterAzureTargetSpectraS3Request.java
index bcd022d0f..a1ee0594e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterAzureTargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterAzureTargetSpectraS3Request.java
@@ -52,9 +52,12 @@ public RegisterAzureTargetSpectraS3Request(final String accountKey, final String
         this.accountName = accountName;
         this.name = name;
         
-        this.getQueryParams().put("account_key", UrlEscapers.urlFragmentEscaper().escape(accountKey).replace("+", "%2B"));
-        this.getQueryParams().put("account_name", UrlEscapers.urlFragmentEscaper().escape(accountName).replace("+", "%2B"));
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
+        this.updateQueryParam("account_key", accountKey);
+
+        this.updateQueryParam("account_name", accountName);
+
+        this.updateQueryParam("name", name);
+
     }
 
     public RegisterAzureTargetSpectraS3Request withAutoVerifyFrequencyInDays(final Integer autoVerifyFrequencyInDays) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterDs3TargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterDs3TargetSpectraS3Request.java
index 3caadae86..b1cf9646b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterDs3TargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterDs3TargetSpectraS3Request.java
@@ -60,10 +60,14 @@ public RegisterDs3TargetSpectraS3Request(final String adminAuthId, final String
         this.dataPathEndPoint = dataPathEndPoint;
         this.name = name;
         
-        this.getQueryParams().put("admin_auth_id", UrlEscapers.urlFragmentEscaper().escape(adminAuthId).replace("+", "%2B"));
-        this.getQueryParams().put("admin_secret_key", UrlEscapers.urlFragmentEscaper().escape(adminSecretKey).replace("+", "%2B"));
-        this.getQueryParams().put("data_path_end_point", UrlEscapers.urlFragmentEscaper().escape(dataPathEndPoint).replace("+", "%2B"));
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
+        this.updateQueryParam("admin_auth_id", adminAuthId);
+
+        this.updateQueryParam("admin_secret_key", adminSecretKey);
+
+        this.updateQueryParam("data_path_end_point", dataPathEndPoint);
+
+        this.updateQueryParam("name", name);
+
     }
 
     public RegisterDs3TargetSpectraS3Request withAccessControlReplication(final Ds3TargetAccessControlReplication accessControlReplication) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterS3TargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterS3TargetSpectraS3Request.java
index d19844f81..b4d0fefd6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterS3TargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/RegisterS3TargetSpectraS3Request.java
@@ -71,9 +71,12 @@ public RegisterS3TargetSpectraS3Request(final String accessKey, final String nam
         this.name = name;
         this.secretKey = secretKey;
         
-        this.getQueryParams().put("access_key", UrlEscapers.urlFragmentEscaper().escape(accessKey).replace("+", "%2B"));
-        this.getQueryParams().put("name", UrlEscapers.urlFragmentEscaper().escape(name).replace("+", "%2B"));
-        this.getQueryParams().put("secret_key", UrlEscapers.urlFragmentEscaper().escape(secretKey).replace("+", "%2B"));
+        this.updateQueryParam("access_key", accessKey);
+
+        this.updateQueryParam("name", name);
+
+        this.updateQueryParam("secret_key", secretKey);
+
     }
 
     public RegisterS3TargetSpectraS3Request withAutoVerifyFrequencyInDays(final Integer autoVerifyFrequencyInDays) {

From 6cb530b60eea4d28d709a6093e967e88bb284b7b Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Thu, 22 Jun 2017 10:03:56 -0600
Subject: [PATCH 059/229] Updating to 3.5.0

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 89efe81b0..ead13b699 100644
--- a/build.gradle
+++ b/build.gradle
@@ -15,7 +15,7 @@
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '3.4.1'
+    version = '3.5.0'
 }
 
 subprojects {

From b5ed227c226f825a3c6353355479f45b6b6fe0bb Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Thu, 22 Jun 2017 13:04:55 -0600
Subject: [PATCH 060/229] Updated default user to Adiministrator in integration
 tests

---
 .../integration/UsersAndGroups_Test.java      | 27 ++++++++++---------
 .../test/helpers/TempStorageUtil.java         |  4 ++-
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
index 6cc097024..3d4f4668a 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
@@ -34,6 +34,7 @@
 import java.util.UUID;
 
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
+import static com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil.DEFAULT_USER;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.junit.Assert.assertThat;
@@ -75,7 +76,7 @@ private static void setupBucket(final UUID dataPolicy) {
         try {
             HELPERS.ensureBucketExists(BUCKET_NAME, dataPolicy);
             spectraUUID = client.getUserSpectraS3(
-                    new GetUserSpectraS3Request("spectra")).getSpectraUserResult().getId();
+                    new GetUserSpectraS3Request(DEFAULT_USER)).getSpectraUserResult().getId();
             administratorsUUID = client.getGroupSpectraS3(
                     new GetGroupSpectraS3Request("Administrators")).getGroupResult().getId();
         } catch (final Exception e) {
@@ -89,16 +90,16 @@ public void getUserWithUUID() throws IOException, SignatureException {
         final GetUserSpectraS3Response getSpectraResponse = client
                 .getUserSpectraS3(new GetUserSpectraS3Request(spectraUUID));
 
-        assertThat(getSpectraResponse.getSpectraUserResult().getName(), is("spectra"));
+        assertThat(getSpectraResponse.getSpectraUserResult().getName(), is(DEFAULT_USER));
     }
 
     @Test
     public void getUserWithString() throws IOException, SignatureException {
 
         final GetUserSpectraS3Response getSpectraResponse = client
-                .getUserSpectraS3(new GetUserSpectraS3Request("spectra"));
+                .getUserSpectraS3(new GetUserSpectraS3Request(DEFAULT_USER));
 
-        assertThat(getSpectraResponse.getSpectraUserResult().getName(), is("spectra"));
+        assertThat(getSpectraResponse.getSpectraUserResult().getName(), is(DEFAULT_USER));
     }
 
     @Test
@@ -114,7 +115,7 @@ public void getUsers() throws IOException, SignatureException {
     public void getUsersWithName() throws IOException, SignatureException {
 
         final GetUsersSpectraS3Response getUsersResponse = client.getUsersSpectraS3(
-                new GetUsersSpectraS3Request().withName("spectra"));
+                new GetUsersSpectraS3Request().withName(DEFAULT_USER));
 
         assertThat(getUsersResponse.getSpectraUserListResult(), is(notNullValue()));
     }
@@ -467,7 +468,7 @@ public void getUserGroupMembersWithMemberUserIDString() throws IOException, Sign
 
         final GetGroupMembersSpectraS3Response getGroupMembersSpectraS3Response = client
                 .getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
-                        .withMemberUserId("spectra"));
+                        .withMemberUserId(DEFAULT_USER));
 
         assertThat(getGroupMembersSpectraS3Response.getGroupMemberListResult(), is(notNullValue()));
     }
@@ -612,7 +613,7 @@ public void putUserGroupMember() throws IOException, SignatureException {
                     new PutGroupSpectraS3Request("test_group"));
 
             putUserGroupMemberSpectraS3Response = client
-                    .putUserGroupMemberSpectraS3(new PutUserGroupMemberSpectraS3Request("test_group", "spectra"));
+                    .putUserGroupMemberSpectraS3(new PutUserGroupMemberSpectraS3Request("test_group", DEFAULT_USER));
 
             assertThat(putUserGroupMemberSpectraS3Response.getGroupMemberResult(), is(notNullValue()));
 
@@ -688,7 +689,7 @@ public void putGlobalBucketAclForUserString() throws IOException, SignatureExcep
         try {
             putGlobalBucketAclForUserSpectraS3Response = client.putGlobalBucketAclForUserSpectraS3(
                     new PutGlobalBucketAclForUserSpectraS3Request(BucketAclPermission.DELETE,
-                            "spectra"));
+                            DEFAULT_USER));
 
             assertThat(putGlobalBucketAclForUserSpectraS3Response.getBucketAclResult(), is(notNullValue()));
 
@@ -789,7 +790,7 @@ public void getBucketAclsWithUserUUID() throws IOException, SignatureException {
     @Test
     public void getBucketAclsWithUserIDString() throws IOException, SignatureException {
         final GetBucketAclsSpectraS3Response getBucketAclsSpectraS3Response = client
-                .getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request().withUserId("spectra"));
+                .getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request().withUserId(DEFAULT_USER));
 
         assertThat(getBucketAclsSpectraS3Response.getBucketAclListResult(), is(notNullValue()));
     }
@@ -912,7 +913,7 @@ public void putGlobalDataPolicyAclForUserString() throws IOException, SignatureE
         PutGlobalDataPolicyAclForUserSpectraS3Response putGlobalDataPolicyAclForUserSpectraS3Response = null;
         try {
             putGlobalDataPolicyAclForUserSpectraS3Response = client.putGlobalDataPolicyAclForUserSpectraS3(
-                    new PutGlobalDataPolicyAclForUserSpectraS3Request("spectra"));
+                    new PutGlobalDataPolicyAclForUserSpectraS3Request(DEFAULT_USER));
 
             assertThat(putGlobalDataPolicyAclForUserSpectraS3Response.getDataPolicyAclResult(), is(notNullValue()));
 
@@ -971,7 +972,7 @@ public void deleteDataPolicyAcl() throws IOException, SignatureException {
                 = null;
         try {
             putGlobalDataPolicyAclForUserSpectraS3Response = client.putGlobalDataPolicyAclForUserSpectraS3(
-                    new PutGlobalDataPolicyAclForUserSpectraS3Request("spectra"));
+                    new PutGlobalDataPolicyAclForUserSpectraS3Request(DEFAULT_USER));
 
             final DeleteDataPolicyAclSpectraS3Response deleteDataPolicyAclSpectraS3Response = client
                     .deleteDataPolicyAclSpectraS3(new DeleteDataPolicyAclSpectraS3Request(
@@ -998,7 +999,7 @@ public void getDataPolicyAcl() throws IOException, SignatureException {
         try {
             putGlobalDataPolicyAclForUserSpectraS3Response = client
                     .putGlobalDataPolicyAclForUserSpectraS3(
-                    new PutGlobalDataPolicyAclForUserSpectraS3Request("spectra"));
+                    new PutGlobalDataPolicyAclForUserSpectraS3Request(DEFAULT_USER));
 
             final GetDataPolicyAclSpectraS3Response getDataPolicyAclSpectraS3Response = client
                     .getDataPolicyAclSpectraS3(new GetDataPolicyAclSpectraS3Request(
@@ -1114,7 +1115,7 @@ public void getDataPolicyAclsWithUserString() throws IOException, SignatureExcep
 
         final GetDataPolicyAclsSpectraS3Response getDataPolicyAclsSpectraS3Response = client
                 .getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()
-                        .withUserId("spectra"));
+                        .withUserId(DEFAULT_USER));
 
         assertThat(getDataPolicyAclsSpectraS3Response.getDataPolicyAclListResult(), is(notNullValue()));
     }
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
index 34d4af2e7..040a80a65 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
@@ -36,6 +36,8 @@ public class TempStorageUtil {
     private static final String STORAGE_DOMAIN_NAME = "_sd";
     private static final String POOL_PARTITION_NAME = "_pp";
 
+    public static final String DEFAULT_USER = "Administrator";
+
     /**
      * Sets up a temporary data policy with a temporary storage domain and partition
      * for use in integration tests where the user may not currently have access to
@@ -102,7 +104,7 @@ public static UUID setupDataPolicy(
                 new PutDataPolicySpectraS3Request(testSetName + DATA_POLICY_NAME)
                         .withEndToEndCrcRequired(withEndToEndCrcRequired)
                         .withChecksumType(checksumType).withAlwaysForcePutJobCreation(true));
-        client.modifyUserSpectraS3(new ModifyUserSpectraS3Request("spectra")
+        client.modifyUserSpectraS3(new ModifyUserSpectraS3Request(DEFAULT_USER)
                 .withDefaultDataPolicyId(dataPolicyResponse.getDataPolicyResult().getId()));
         return dataPolicyResponse.getDataPolicyResult().getId();
     }

From 6f8adb7a96a1e36852e39e5f8336636bc1b65d00 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Thu, 22 Jun 2017 15:30:03 -0600
Subject: [PATCH 061/229] Updating broken tests due to 4.0 API changes

---
 .../com/spectralogic/ds3client/integration/Util.java     | 7 +++++++
 .../spectralogic/ds3client/integration/Smoke_Test.java   | 4 ++--
 .../ds3client/integration/UsersAndGroups_Test.java       | 9 +++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index 166f73532..af8b2c6c6 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -37,6 +37,7 @@
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assume.assumeThat;
+import static org.junit.Assume.assumeTrue;
 
 
 public class Util {
@@ -64,6 +65,12 @@ public static void assumeVersion1_2(final Ds3Client client) throws IOException {
         assumeThat(majorVersion, is(1));
     }
 
+    public static void assumeVersion3orLess(final Ds3Client client) throws IOException {
+        final int majorVersion = Integer.parseInt(client.getSystemInformationSpectraS3(
+                new GetSystemInformationSpectraS3Request()).getSystemInformationResult().getBuildInformation().getVersion().split("\\.")[0]);
+        assumeTrue(majorVersion <= 3);
+    }
+
     public static void loadBookTestData(final Ds3Client client, final String bucketName) throws IOException, URISyntaxException {
         LOG.info("Loading test data...");
         getLoadJob(client, bucketName, RESOURCE_BASE_NAME)
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
index 5df2ef496..0cabe88b3 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
@@ -143,8 +143,8 @@ public void getObjects() throws IOException, URISyntaxException {
             assertThat(response.getS3ObjectListResult().getS3Objects().size(), is(4));
             assertTrue(s3ObjectExists(response.getS3ObjectListResult().getS3Objects(), "beowulf.txt"));
 
-            assertThat(response.getPagingTruncated(), is(nullValue()));
-            assertThat(response.getPagingTotalResultCount(), is(nullValue()));
+            assertThat(response.getPagingTruncated(), is(0));
+            assertThat(response.getPagingTotalResultCount(), is(4));
         } finally {
             deleteAllContents(client,bucketName);
         }
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
index 3d4f4668a..1cf1da7c3 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
@@ -33,6 +33,7 @@
 import java.security.SignatureException;
 import java.util.UUID;
 
+import static com.spectralogic.ds3client.integration.Util.assumeVersion3orLess;
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
 import static com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil.DEFAULT_USER;
 import static org.hamcrest.Matchers.is;
@@ -546,6 +547,7 @@ public void getGroupGroupMembersWithLastPage() throws IOException, SignatureExce
 
     @Test
     public void getGroupGroupMembersWithPageStartMarkerUUID() throws IOException, SignatureException {
+        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
 
         final GetGroupMembersSpectraS3Response getGroupMembersSpectraS3Response = client
                 .getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
@@ -556,6 +558,7 @@ public void getGroupGroupMembersWithPageStartMarkerUUID() throws IOException, Si
 
     @Test
     public void getGroupGroupMembersWithPageStartMarkerString() throws IOException, SignatureException {
+        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
 
         final GetGroupMembersSpectraS3Response getGroupMembersSpectraS3Response = client
                 .getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
@@ -851,6 +854,8 @@ public void getBucketAclsWithLastPage() throws IOException, SignatureException {
 
     @Test
     public void getBucketAclsWithPageStartMarkerUUID() throws IOException, SignatureException {
+        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
+
         final GetBucketAclsSpectraS3Response getBucketAclsSpectraS3Response = client
                 .getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request()
                         .withPageStartMarker(UUID.randomUUID()));
@@ -860,6 +865,8 @@ public void getBucketAclsWithPageStartMarkerUUID() throws IOException, Signature
 
     @Test
     public void getBucketAclsWithPageStartMarkerString() throws IOException, SignatureException {
+        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
+
         final GetBucketAclsSpectraS3Response getBucketAclsSpectraS3Response = client
                 .getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request()
                         .withPageStartMarker(UUID.randomUUID().toString()));
@@ -1182,6 +1189,7 @@ public void getDataPolicyAclsWithPageLength() throws IOException, SignatureExcep
 
     @Test
     public void getDataPolicyAclsWithPageStartMarkerString() throws IOException, SignatureException {
+        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
 
         final GetDataPolicyAclsSpectraS3Response getDataPolicyAclsSpectraS3Response = client
                 .getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()
@@ -1192,6 +1200,7 @@ public void getDataPolicyAclsWithPageStartMarkerString() throws IOException, Sig
 
     @Test
     public void getDataPolicyAclsWithPageStartMarkerUUID() throws IOException, SignatureException {
+        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
 
         final GetDataPolicyAclsSpectraS3Response getDataPolicyAclsSpectraS3Response = client
                 .getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()

From 555189f95d254d90f56d5cefe4f54d4f3bd7e218 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 23 Jun 2017 13:08:10 -0600
Subject: [PATCH 062/229] Fixing minor bug in setting max upload size in bulk
 put

---
 .../commands/spectrads3/PutBulkJobSpectraS3Request.java         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBulkJobSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBulkJobSpectraS3Request.java
index 0539f2f6c..2407395a2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBulkJobSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBulkJobSpectraS3Request.java
@@ -88,7 +88,7 @@ public PutBulkJobSpectraS3Request withImplicitJobIdResolution(final boolean impl
 
 
     public PutBulkJobSpectraS3Request withMaxUploadSize(final long maxUploadSize) {
-        if (maxUploadSize > MIN_UPLOAD_SIZE_IN_BYTES) {
+        if (maxUploadSize >= MIN_UPLOAD_SIZE_IN_BYTES) {
             this.getQueryParams().put("max_upload_size", Long.toString(maxUploadSize));
         } else {
             this.getQueryParams().put("max_upload_size", MAX_UPLOAD_SIZE_IN_BYTES);

From 810b8ff885450fe83b86eb35eda2a9027c0852de Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 23 Jun 2017 13:30:27 -0600
Subject: [PATCH 063/229] Fixed bug in setting max upload size in bulk put

---
 .../commands/spectrads3/PutBulkJobSpectraS3Request.java         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBulkJobSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBulkJobSpectraS3Request.java
index 0539f2f6c..2407395a2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBulkJobSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutBulkJobSpectraS3Request.java
@@ -88,7 +88,7 @@ public PutBulkJobSpectraS3Request withImplicitJobIdResolution(final boolean impl
 
 
     public PutBulkJobSpectraS3Request withMaxUploadSize(final long maxUploadSize) {
-        if (maxUploadSize > MIN_UPLOAD_SIZE_IN_BYTES) {
+        if (maxUploadSize >= MIN_UPLOAD_SIZE_IN_BYTES) {
             this.getQueryParams().put("max_upload_size", Long.toString(maxUploadSize));
         } else {
             this.getQueryParams().put("max_upload_size", MAX_UPLOAD_SIZE_IN_BYTES);

From b03d0ac5d29671c015f8807136e9a77d3928f30d Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 23 Jun 2017 14:20:32 -0600
Subject: [PATCH 064/229] Update to 4.0.0 API

---
 .../ds3client/integration/Iterators_Test.java |  8 +-
 .../SpectraS3PaginationLoader_Test.java       |  3 +-
 .../com/spectralogic/ds3client/Ds3Client.java | 14 ++++
 .../spectralogic/ds3client/Ds3ClientImpl.java |  8 ++
 ...AggregatingJobSpectraS3ResponseParser.java | 49 ++++++++++++
 ...odifyTapeDriveSpectraS3ResponseParser.java | 49 ++++++++++++
 .../CloseAggregatingJobSpectraS3Request.java  | 61 +++++++++++++++
 .../CloseAggregatingJobSpectraS3Response.java | 36 +++++++++
 .../GetObjectsDetailsSpectraS3Request.java    | 14 ----
 ...bjectsWithFullDetailsSpectraS3Request.java | 14 ----
 .../spectrads3/GetTapesSpectraS3Request.java  | 14 ++++
 .../ModifyTapeDriveSpectraS3Request.java      | 74 +++++++++++++++++++
 .../ModifyTapeDriveSpectraS3Response.java     | 36 +++++++++
 .../GetObjectsFullDetailsLoader.java          |  4 +-
 .../GetObjectsFullDetailsLoaderFactory.java   |  6 +-
 ...etObjectsFullDetailsPaginatingCommand.java | 12 +--
 .../models/DetailedTapePartition.java         | 12 +++
 .../spectralogic/ds3client/models/Job.java    |  7 +-
 .../ds3client/models/MasterObjectList.java    |  7 +-
 .../models/NamedDetailedTapePartition.java    | 12 +++
 .../ds3client/models/PoolFailureType.java     |  1 +
 .../ds3client/models/TapeDrive.java           | 12 +++
 .../ds3client/models/TapeDriveType.java       |  1 +
 .../ds3client/models/TapeFailureType.java     |  1 +
 .../ds3client/models/TapePartition.java       | 12 +++
 .../models/TapePartitionFailureType.java      |  1 +
 .../ds3client/models/TapeType.java            |  1 +
 .../ds3client/models/TargetFailureType.java   |  1 +
 28 files changed, 414 insertions(+), 56 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/CloseAggregatingJobSpectraS3ResponseParser.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/ModifyTapeDriveSpectraS3ResponseParser.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/CloseAggregatingJobSpectraS3Request.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/CloseAggregatingJobSpectraS3Response.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Request.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Response.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
index f607395f5..8e6608172 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Iterators_Test.java
@@ -123,22 +123,22 @@ public void failedRequestGetBucket() {
 
    @Test
     public void emptyGetObjects() throws IOException {
-        emptyTest(new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, "", 10, RETRIES, true));
+        emptyTest(new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, 10, RETRIES, true));
     }
 
     @Test
     public void singlePageGetObjectsIterator() throws IOException, URISyntaxException {
-        paginate(new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, "", 10, RETRIES, true));
+        paginate(new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, 10, RETRIES, true));
     }
 
     @Test
     public void failedGetObjectsWithFullDetails() throws IOException, URISyntaxException {
-        testFailedRequest(new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, "", 10, RETRIES, true));
+        testFailedRequest(new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, 10, RETRIES, true));
     }
 
     @Test
     public void multiPageGetObjectsWithFullDetails() throws IOException, URISyntaxException {
-        paginate(new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, "", 2, RETRIES, true));
+        paginate(new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, 2, RETRIES, true));
     }
 
     private void emptyTest(final LazyIterable.LazyLoaderFactory lazyLoaderFactory) throws IOException {
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java
index d7772930d..c3e595c51 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java
@@ -39,7 +39,6 @@
 import java.util.UUID;
 
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
-import static com.spectralogic.ds3client.integration.Util.insecureFromEnv;
 import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -79,7 +78,7 @@ public void basicPagination() throws IOException {
             final int numObjects = 55;
             CLIENT.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(TEST_ENV_NAME, createTestList(numObjects)));
 
-            final GetObjectsFullDetailsLoaderFactory loaderFactory = new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, "", 10, RETRIES, false);
+            final GetObjectsFullDetailsLoaderFactory loaderFactory = new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, 10, RETRIES, false);
 
             final Set foundItems = new HashSet<>();
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index 0268450de..559dfc55e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -762,6 +762,13 @@ ClearAllCanceledJobsSpectraS3Response clearAllCanceledJobsSpectraS3(final ClearA
     ClearAllCompletedJobsSpectraS3Response clearAllCompletedJobsSpectraS3(final ClearAllCompletedJobsSpectraS3Request request)
             throws IOException;
 
+    @ResponsePayloadModel("MasterObjectList")
+    @Action("MODIFY")
+    @Resource("JOB")
+    
+    CloseAggregatingJobSpectraS3Response closeAggregatingJobSpectraS3(final CloseAggregatingJobSpectraS3Request request)
+            throws IOException;
+
     @ResponsePayloadModel("MasterObjectList")
     @Action("MODIFY")
     @Resource("BUCKET")
@@ -1996,6 +2003,13 @@ InspectTapeSpectraS3Response inspectTapeSpectraS3(final InspectTapeSpectraS3Requ
     ModifyAllTapePartitionsSpectraS3Response modifyAllTapePartitionsSpectraS3(final ModifyAllTapePartitionsSpectraS3Request request)
             throws IOException;
 
+    @ResponsePayloadModel("TapeDrive")
+    @Action("MODIFY")
+    @Resource("TAPE_DRIVE")
+    
+    ModifyTapeDriveSpectraS3Response modifyTapeDriveSpectraS3(final ModifyTapeDriveSpectraS3Request request)
+            throws IOException;
+
     @ResponsePayloadModel("TapePartition")
     @Action("MODIFY")
     @Resource("TAPE_PARTITION")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
index 8a92e5bb3..a04ac9ca7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
@@ -494,6 +494,10 @@ public ClearAllCompletedJobsSpectraS3Response clearAllCompletedJobsSpectraS3(fin
         return new ClearAllCompletedJobsSpectraS3ResponseParser().response(this.netClient.getResponse(request));
     }
     @Override
+    public CloseAggregatingJobSpectraS3Response closeAggregatingJobSpectraS3(final CloseAggregatingJobSpectraS3Request request) throws IOException {
+        return new CloseAggregatingJobSpectraS3ResponseParser().response(this.netClient.getResponse(request));
+    }
+    @Override
     public GetBulkJobSpectraS3Response getBulkJobSpectraS3(final GetBulkJobSpectraS3Request request) throws IOException {
         return new GetBulkJobSpectraS3ResponseParser().response(this.netClient.getResponse(request));
     }
@@ -1226,6 +1230,10 @@ public ModifyAllTapePartitionsSpectraS3Response modifyAllTapePartitionsSpectraS3
         return new ModifyAllTapePartitionsSpectraS3ResponseParser().response(this.netClient.getResponse(request));
     }
     @Override
+    public ModifyTapeDriveSpectraS3Response modifyTapeDriveSpectraS3(final ModifyTapeDriveSpectraS3Request request) throws IOException {
+        return new ModifyTapeDriveSpectraS3ResponseParser().response(this.netClient.getResponse(request));
+    }
+    @Override
     public ModifyTapePartitionSpectraS3Response modifyTapePartitionSpectraS3(final ModifyTapePartitionSpectraS3Request request) throws IOException {
         return new ModifyTapePartitionSpectraS3ResponseParser().response(this.netClient.getResponse(request));
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/CloseAggregatingJobSpectraS3ResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/CloseAggregatingJobSpectraS3ResponseParser.java
new file mode 100644
index 000000000..c3f2b24b3
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/CloseAggregatingJobSpectraS3ResponseParser.java
@@ -0,0 +1,49 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.parsers;
+
+import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
+import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
+import com.spectralogic.ds3client.commands.spectrads3.CloseAggregatingJobSpectraS3Response;
+import com.spectralogic.ds3client.models.MasterObjectList;
+import com.spectralogic.ds3client.networking.WebResponse;
+import com.spectralogic.ds3client.serializer.XmlOutput;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class CloseAggregatingJobSpectraS3ResponseParser extends AbstractResponseParser {
+    private final int[] expectedStatusCodes = new int[]{200};
+
+    @Override
+    public CloseAggregatingJobSpectraS3Response parseXmlResponse(final WebResponse response) throws IOException {
+        final int statusCode = response.getStatusCode();
+        if (ResponseParserUtils.validateStatusCode(statusCode, expectedStatusCodes)) {
+            switch (statusCode) {
+            case 200:
+                try (final InputStream inputStream = response.getResponseStream()) {
+                    final MasterObjectList result = XmlOutput.fromXml(inputStream, MasterObjectList.class);
+                    return new CloseAggregatingJobSpectraS3Response(result, this.getChecksum(), this.getChecksumType());
+                }
+
+            default:
+                assert false: "validateStatusCode should have made it impossible to reach this line";
+            }
+        }
+
+        throw ResponseParserUtils.createFailedRequest(response, expectedStatusCodes);
+    }
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/ModifyTapeDriveSpectraS3ResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/ModifyTapeDriveSpectraS3ResponseParser.java
new file mode 100644
index 000000000..91e753102
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/ModifyTapeDriveSpectraS3ResponseParser.java
@@ -0,0 +1,49 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.parsers;
+
+import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
+import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
+import com.spectralogic.ds3client.commands.spectrads3.ModifyTapeDriveSpectraS3Response;
+import com.spectralogic.ds3client.models.TapeDrive;
+import com.spectralogic.ds3client.networking.WebResponse;
+import com.spectralogic.ds3client.serializer.XmlOutput;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class ModifyTapeDriveSpectraS3ResponseParser extends AbstractResponseParser {
+    private final int[] expectedStatusCodes = new int[]{200};
+
+    @Override
+    public ModifyTapeDriveSpectraS3Response parseXmlResponse(final WebResponse response) throws IOException {
+        final int statusCode = response.getStatusCode();
+        if (ResponseParserUtils.validateStatusCode(statusCode, expectedStatusCodes)) {
+            switch (statusCode) {
+            case 200:
+                try (final InputStream inputStream = response.getResponseStream()) {
+                    final TapeDrive result = XmlOutput.fromXml(inputStream, TapeDrive.class);
+                    return new ModifyTapeDriveSpectraS3Response(result, this.getChecksum(), this.getChecksumType());
+                }
+
+            default:
+                assert false: "validateStatusCode should have made it impossible to reach this line";
+            }
+        }
+
+        throw ResponseParserUtils.createFailedRequest(response, expectedStatusCodes);
+    }
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/CloseAggregatingJobSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/CloseAggregatingJobSpectraS3Request.java
new file mode 100644
index 000000000..b3bdd12a0
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/CloseAggregatingJobSpectraS3Request.java
@@ -0,0 +1,61 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.spectrads3;
+
+import com.spectralogic.ds3client.networking.HttpVerb;
+import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import java.util.UUID;
+import com.google.common.net.UrlEscapers;
+
+public class CloseAggregatingJobSpectraS3Request extends AbstractRequest {
+
+    // Variables
+    
+    private final String jobId;
+
+    // Constructor
+    
+    
+    public CloseAggregatingJobSpectraS3Request(final UUID jobId) {
+        this.jobId = jobId.toString();
+        
+        this.getQueryParams().put("close_aggregating_job", null);
+    }
+
+    
+    public CloseAggregatingJobSpectraS3Request(final String jobId) {
+        this.jobId = jobId;
+        
+        this.getQueryParams().put("close_aggregating_job", null);
+    }
+
+
+    @Override
+    public HttpVerb getVerb() {
+        return HttpVerb.PUT;
+    }
+
+    @Override
+    public String getPath() {
+        return "/_rest_/job/" + jobId;
+    }
+    
+    public String getJobId() {
+        return this.jobId;
+    }
+
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/CloseAggregatingJobSpectraS3Response.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/CloseAggregatingJobSpectraS3Response.java
new file mode 100644
index 000000000..b742a0a30
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/CloseAggregatingJobSpectraS3Response.java
@@ -0,0 +1,36 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.spectrads3;
+
+import com.spectralogic.ds3client.models.MasterObjectList;
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
+
+public class CloseAggregatingJobSpectraS3Response extends AbstractResponse {
+    
+    private final MasterObjectList masterObjectListResult;
+
+    public CloseAggregatingJobSpectraS3Response(final MasterObjectList masterObjectListResult, final String checksum, final ChecksumType.Type checksumType) {
+        super(checksum, checksumType);
+        this.masterObjectListResult = masterObjectListResult;
+    }
+
+    public MasterObjectList getMasterObjectListResult() {
+        return this.masterObjectListResult;
+    }
+
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsDetailsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsDetailsSpectraS3Request.java
index 69d00b6f7..d8fd50532 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsDetailsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsDetailsSpectraS3Request.java
@@ -28,8 +28,6 @@ public class GetObjectsDetailsSpectraS3Request extends AbstractPaginationRequest
     
     private String bucketId;
 
-    private String folder;
-
     private boolean lastPage;
 
     private boolean latest;
@@ -60,13 +58,6 @@ public GetObjectsDetailsSpectraS3Request withBucketId(final String bucketId) {
     }
 
 
-    public GetObjectsDetailsSpectraS3Request withFolder(final String folder) {
-        this.folder = folder;
-        this.updateQueryParam("folder", folder);
-        return this;
-    }
-
-
     public GetObjectsDetailsSpectraS3Request withLastPage(final boolean lastPage) {
         this.lastPage = lastPage;
         if (this.lastPage) {
@@ -150,11 +141,6 @@ public String getBucketId() {
     }
 
 
-    public String getFolder() {
-        return this.folder;
-    }
-
-
     public boolean getLastPage() {
         return this.lastPage;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsWithFullDetailsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsWithFullDetailsSpectraS3Request.java
index 45ea25e44..5f021cdad 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsWithFullDetailsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsWithFullDetailsSpectraS3Request.java
@@ -28,8 +28,6 @@ public class GetObjectsWithFullDetailsSpectraS3Request extends AbstractPaginatio
     
     private String bucketId;
 
-    private String folder;
-
     private boolean includePhysicalPlacement;
 
     private boolean lastPage;
@@ -63,13 +61,6 @@ public GetObjectsWithFullDetailsSpectraS3Request withBucketId(final String bucke
     }
 
 
-    public GetObjectsWithFullDetailsSpectraS3Request withFolder(final String folder) {
-        this.folder = folder;
-        this.updateQueryParam("folder", folder);
-        return this;
-    }
-
-
     public GetObjectsWithFullDetailsSpectraS3Request withIncludePhysicalPlacement(final boolean includePhysicalPlacement) {
         this.includePhysicalPlacement = includePhysicalPlacement;
         if (this.includePhysicalPlacement) {
@@ -164,11 +155,6 @@ public String getBucketId() {
     }
 
 
-    public String getFolder() {
-        return this.folder;
-    }
-
-
     public boolean getIncludePhysicalPlacement() {
         return this.includePhysicalPlacement;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
index 11e205f88..6ced563a9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
@@ -62,6 +62,8 @@ public class GetTapesSpectraS3Request extends AbstractPaginationRequest {
 
     private String serialNumber;
 
+    private String sortBy;
+
     private TapeState state;
 
     private String storageDomainId;
@@ -209,6 +211,13 @@ public GetTapesSpectraS3Request withSerialNumber(final String serialNumber) {
     }
 
 
+    public GetTapesSpectraS3Request withSortBy(final String sortBy) {
+        this.sortBy = sortBy;
+        this.updateQueryParam("sort_by", sortBy);
+        return this;
+    }
+
+
     public GetTapesSpectraS3Request withState(final TapeState state) {
         this.state = state;
         this.updateQueryParam("state", state);
@@ -342,6 +351,11 @@ public String getSerialNumber() {
     }
 
 
+    public String getSortBy() {
+        return this.sortBy;
+    }
+
+
     public TapeState getState() {
         return this.state;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Request.java
new file mode 100644
index 000000000..dbb9bc0f7
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Request.java
@@ -0,0 +1,74 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.spectrads3;
+
+import com.spectralogic.ds3client.networking.HttpVerb;
+import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.models.Quiesced;
+import java.util.UUID;
+import com.google.common.net.UrlEscapers;
+
+public class ModifyTapeDriveSpectraS3Request extends AbstractRequest {
+
+    // Variables
+    
+    private final String tapeDriveId;
+
+    private Quiesced quiesced;
+
+    // Constructor
+    
+    
+    public ModifyTapeDriveSpectraS3Request(final UUID tapeDriveId) {
+        this.tapeDriveId = tapeDriveId.toString();
+        
+    }
+
+    
+    public ModifyTapeDriveSpectraS3Request(final String tapeDriveId) {
+        this.tapeDriveId = tapeDriveId;
+        
+    }
+
+    public ModifyTapeDriveSpectraS3Request withQuiesced(final Quiesced quiesced) {
+        this.quiesced = quiesced;
+        this.updateQueryParam("quiesced", quiesced);
+        return this;
+    }
+
+
+
+    @Override
+    public HttpVerb getVerb() {
+        return HttpVerb.PUT;
+    }
+
+    @Override
+    public String getPath() {
+        return "/_rest_/tape_drive/" + tapeDriveId;
+    }
+    
+    public String getTapeDriveId() {
+        return this.tapeDriveId;
+    }
+
+
+    public Quiesced getQuiesced() {
+        return this.quiesced;
+    }
+
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Response.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Response.java
new file mode 100644
index 000000000..154a9ea8a
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Response.java
@@ -0,0 +1,36 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.spectrads3;
+
+import com.spectralogic.ds3client.models.TapeDrive;
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
+
+public class ModifyTapeDriveSpectraS3Response extends AbstractResponse {
+    
+    private final TapeDrive tapeDriveResult;
+
+    public ModifyTapeDriveSpectraS3Response(final TapeDrive tapeDriveResult, final String checksum, final ChecksumType.Type checksumType) {
+        super(checksum, checksumType);
+        this.tapeDriveResult = tapeDriveResult;
+    }
+
+    public TapeDrive getTapeDriveResult() {
+        return this.tapeDriveResult;
+    }
+
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoader.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoader.java
index 0ea2b7701..b427ab27f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoader.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoader.java
@@ -28,9 +28,9 @@ public class GetObjectsFullDetailsLoader implements LazyIterable.LazyLoader paginator;
 
-    public GetObjectsFullDetailsLoader(final Ds3Client client, final String bucket, final String folder, final boolean includePhysicalDetails, final int pageLength, final int retryCount) {
+    public GetObjectsFullDetailsLoader(final Ds3Client client, final String bucket, final boolean includePhysicalDetails, final int pageLength, final int retryCount) {
         paginator = new SpectraS3PaginationLoader<>(
-                new GetObjectsFullDetailsPaginatingCommand(client, bucket, folder, includePhysicalDetails),
+                new GetObjectsFullDetailsPaginatingCommand(client, bucket, includePhysicalDetails),
                 pageLength,
                 retryCount);
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoaderFactory.java
index 3371e72e1..ac15c0afa 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetObjectsFullDetailsLoaderFactory.java
@@ -25,15 +25,13 @@
 public class GetObjectsFullDetailsLoaderFactory implements LazyIterable.LazyLoaderFactory {
     private final Ds3Client client;
     private final String bucket;
-    private final String folder;
     private final int pageLength;
     private final int retryCount;
     private final boolean includePhysicalDetails;
 
-    public GetObjectsFullDetailsLoaderFactory(final Ds3Client client, final String bucket, final String folder, final int pageLength, final int retryCount, final boolean includePhysicalDetails) {
+    public GetObjectsFullDetailsLoaderFactory(final Ds3Client client, final String bucket, final int pageLength, final int retryCount, final boolean includePhysicalDetails) {
         this.client = client;
         this.bucket = bucket;
-        this.folder = folder;
         this.pageLength = pageLength;
         this.includePhysicalDetails = includePhysicalDetails;
         this.retryCount = retryCount;
@@ -41,6 +39,6 @@ public GetObjectsFullDetailsLoaderFactory(final Ds3Client client, final String b
 
     @Override
     public LazyIterable.LazyLoader create() {
-        return new GetObjectsFullDetailsLoader(client, bucket, folder, includePhysicalDetails, pageLength, retryCount);
+        return new GetObjectsFullDetailsLoader(client, bucket, includePhysicalDetails, pageLength, retryCount);
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/commands/GetObjectsFullDetailsPaginatingCommand.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/commands/GetObjectsFullDetailsPaginatingCommand.java
index bb594e06e..32b156d21 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/commands/GetObjectsFullDetailsPaginatingCommand.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/commands/GetObjectsFullDetailsPaginatingCommand.java
@@ -20,7 +20,6 @@
 import com.spectralogic.ds3client.commands.spectrads3.GetObjectsWithFullDetailsSpectraS3Response;
 import com.spectralogic.ds3client.helpers.pagination.PaginatingCommand;
 import com.spectralogic.ds3client.models.DetailedS3Object;
-import com.spectralogic.ds3client.utils.Guard;
 
 import java.io.IOException;
 import java.util.List;
@@ -28,24 +27,17 @@
 public class GetObjectsFullDetailsPaginatingCommand implements PaginatingCommand {
     private final Ds3Client client;
     private final String bucket;
-    private final String folder;
     private final boolean includePhysicalDetails;
 
-    public GetObjectsFullDetailsPaginatingCommand(final Ds3Client client, final String bucket, final String folder, final boolean includePhysicalDetails) {
+    public GetObjectsFullDetailsPaginatingCommand(final Ds3Client client, final String bucket, final boolean includePhysicalDetails) {
         this.client = client;
         this.bucket = bucket;
-        this.folder = folder;
         this.includePhysicalDetails = includePhysicalDetails;
     }
 
     @Override
     public GetObjectsWithFullDetailsSpectraS3Request createRequest() {
-        final GetObjectsWithFullDetailsSpectraS3Request getObjectsWithFullDetailsSpectraS3Request = new GetObjectsWithFullDetailsSpectraS3Request().withBucketId(bucket).withIncludePhysicalPlacement(includePhysicalDetails);
-        if (Guard.isStringNullOrEmpty(folder)) {
-            return getObjectsWithFullDetailsSpectraS3Request;
-        } else {
-            return getObjectsWithFullDetailsSpectraS3Request.withFolder(folder);
-        }
+        return new GetObjectsWithFullDetailsSpectraS3Request().withBucketId(bucket).withIncludePhysicalPlacement(includePhysicalDetails);
     }
 
     @Override
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
index cdecdac6d..8ed233ab8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
@@ -53,6 +53,9 @@ public class DetailedTapePartition {
     @JsonProperty("Quiesced")
     private Quiesced quiesced;
 
+    @JsonProperty("SerialId")
+    private String serialId;
+
     @JsonProperty("SerialNumber")
     private String serialNumber;
 
@@ -142,6 +145,15 @@ public void setQuiesced(final Quiesced quiesced) {
     }
 
 
+    public String getSerialId() {
+        return this.serialId;
+    }
+
+    public void setSerialId(final String serialId) {
+        this.serialId = serialId;
+    }
+
+
     public String getSerialNumber() {
         return this.serialNumber;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Job.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Job.java
index 16ba8ffe3..49458a687 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Job.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Job.java
@@ -20,6 +20,7 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
 import java.lang.String;
+import java.lang.Boolean;
 import java.util.UUID;
 import java.util.List;
 import java.util.ArrayList;
@@ -46,7 +47,7 @@ public class Job {
     private long completedSizeInBytes;
 
     @JacksonXmlProperty(isAttribute = true, localName = "EntirelyInCache")
-    private boolean entirelyInCache;
+    private Boolean entirelyInCache;
 
     @JacksonXmlProperty(isAttribute = true, localName = "JobId")
     private UUID jobId;
@@ -134,11 +135,11 @@ public void setCompletedSizeInBytes(final long completedSizeInBytes) {
     }
 
 
-    public boolean getEntirelyInCache() {
+    public Boolean getEntirelyInCache() {
         return this.entirelyInCache;
     }
 
-    public void setEntirelyInCache(final boolean entirelyInCache) {
+    public void setEntirelyInCache(final Boolean entirelyInCache) {
         this.entirelyInCache = entirelyInCache;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MasterObjectList.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MasterObjectList.java
index 42c2bde32..f0e97e4c3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MasterObjectList.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MasterObjectList.java
@@ -20,6 +20,7 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
 import java.lang.String;
+import java.lang.Boolean;
 import java.util.UUID;
 import java.util.List;
 import java.util.ArrayList;
@@ -46,7 +47,7 @@ public class MasterObjectList {
     private long completedSizeInBytes;
 
     @JacksonXmlProperty(isAttribute = true, localName = "EntirelyInCache")
-    private boolean entirelyInCache;
+    private Boolean entirelyInCache;
 
     @JacksonXmlProperty(isAttribute = true, localName = "JobId")
     private UUID jobId;
@@ -138,11 +139,11 @@ public void setCompletedSizeInBytes(final long completedSizeInBytes) {
     }
 
 
-    public boolean getEntirelyInCache() {
+    public Boolean getEntirelyInCache() {
         return this.entirelyInCache;
     }
 
-    public void setEntirelyInCache(final boolean entirelyInCache) {
+    public void setEntirelyInCache(final Boolean entirelyInCache) {
         this.entirelyInCache = entirelyInCache;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
index ad8f7a470..1f0420295 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
@@ -53,6 +53,9 @@ public class NamedDetailedTapePartition {
     @JsonProperty("Quiesced")
     private Quiesced quiesced;
 
+    @JsonProperty("SerialId")
+    private String serialId;
+
     @JsonProperty("SerialNumber")
     private String serialNumber;
 
@@ -142,6 +145,15 @@ public void setQuiesced(final Quiesced quiesced) {
     }
 
 
+    public String getSerialId() {
+        return this.serialId;
+    }
+
+    public void setSerialId(final String serialId) {
+        this.serialId = serialId;
+    }
+
+
     public String getSerialNumber() {
         return this.serialNumber;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailureType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailureType.java
index 5f7a87d81..466e6b2ce 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailureType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailureType.java
@@ -22,6 +22,7 @@ public enum PoolFailureType {
     DATA_CHECKPOINT_MISSING,
     FORMAT_FAILED,
     IMPORT_FAILED,
+    IMPORT_INCOMPLETE,
     IMPORT_FAILED_DUE_TO_TAKE_OWNERSHIP_FAILURE,
     IMPORT_FAILED_DUE_TO_DATA_INTEGRITY,
     INSPECT_FAILED,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
index 74911ed31..063918953 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
@@ -47,6 +47,9 @@ public class TapeDrive {
     @JsonProperty("PartitionId")
     private UUID partitionId;
 
+    @JsonProperty("Quiesced")
+    private Quiesced quiesced;
+
     @JsonProperty("SerialNumber")
     private String serialNumber;
 
@@ -129,6 +132,15 @@ public void setPartitionId(final UUID partitionId) {
     }
 
 
+    public Quiesced getQuiesced() {
+        return this.quiesced;
+    }
+
+    public void setQuiesced(final Quiesced quiesced) {
+        this.quiesced = quiesced;
+    }
+
+
     public String getSerialNumber() {
         return this.serialNumber;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDriveType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDriveType.java
index 47d182b85..c084195a9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDriveType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDriveType.java
@@ -21,6 +21,7 @@ public enum TapeDriveType {
     LTO5,
     LTO6,
     LTO7,
+    LTO8,
     TS1140,
     TS1150
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailureType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailureType.java
index 69459c360..fc61c3e2a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailureType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailureType.java
@@ -29,6 +29,7 @@ public enum TapeFailureType {
     FORMAT_FAILED,
     GET_TAPE_INFORMATION_FAILED,
     IMPORT_FAILED,
+    IMPORT_INCOMPLETE,
     IMPORT_FAILED_DUE_TO_TAKE_OWNERSHIP_FAILURE,
     IMPORT_FAILED_DUE_TO_DATA_INTEGRITY,
     INSPECT_FAILED,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
index c86173810..d7dd9e19c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
@@ -46,6 +46,9 @@ public class TapePartition {
     @JsonProperty("Quiesced")
     private Quiesced quiesced;
 
+    @JsonProperty("SerialId")
+    private String serialId;
+
     @JsonProperty("SerialNumber")
     private String serialNumber;
 
@@ -122,6 +125,15 @@ public void setQuiesced(final Quiesced quiesced) {
     }
 
 
+    public String getSerialId() {
+        return this.serialId;
+    }
+
+    public void setSerialId(final String serialId) {
+        this.serialId = serialId;
+    }
+
+
     public String getSerialNumber() {
         return this.serialNumber;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureType.java
index c13e09db0..be8cdb2dd 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureType.java
@@ -27,6 +27,7 @@ public enum TapePartitionFailureType {
     ONLINE_STALLED_DUE_TO_NO_STORAGE_SLOTS,
     TAPE_DRIVE_IN_ERROR,
     TAPE_DRIVE_MISSING,
+    TAPE_DRIVE_QUIESCED,
     TAPE_DRIVE_TYPE_MISMATCH,
     TAPE_EJECTION_BY_OPERATOR_REQUIRED,
     TAPE_MEDIA_TYPE_INCOMPATIBLE,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeType.java
index 3df1166ef..e20bf41a5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeType.java
@@ -20,6 +20,7 @@ public enum TapeType {
     LTO5,
     LTO6,
     LTO7,
+    LTO8,
     LTO_CLEANING_TAPE,
     TS_JC,
     TS_JY,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TargetFailureType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TargetFailureType.java
index d75f3df99..92957bea4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TargetFailureType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TargetFailureType.java
@@ -18,6 +18,7 @@
 
 public enum TargetFailureType {
     IMPORT_FAILED,
+    IMPORT_INCOMPLETE,
     NOT_ONLINE,
     WRITE_FAILED,
     WRITE_INITIATE_FAILED,

From 470ca9085397df5667563be392c3d8b9af017937 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 26 Jun 2017 17:34:30 -0600
Subject: [PATCH 065/229] Updated contract used to generate 4.0.0 api

---
 ...{3_4_1_contract.xml => 4_0_0_contract.xml} | 185 ++++++++++++++++--
 1 file changed, 170 insertions(+), 15 deletions(-)
 rename contract/{3_4_1_contract.xml => 4_0_0_contract.xml} (98%)

diff --git a/contract/3_4_1_contract.xml b/contract/4_0_0_contract.xml
similarity index 98%
rename from contract/3_4_1_contract.xml
rename to contract/4_0_0_contract.xml
index 10fd02c10..939620807 100644
--- a/contract/3_4_1_contract.xml
+++ b/contract/4_0_0_contract.xml
@@ -2640,6 +2640,23 @@
                 
                 1.C079F9483A04E55B04DBB93A82435427
             
+            
+                
+                    
+                    
+                        
+                    
+                
+                
+                    
+                        200
+                        
+                            
+                        
+                    
+                
+                1.1212DC6708EBC50394B744582FAA65B4
+            
             
                 
                     
@@ -4425,7 +4442,6 @@
                 
                     
                         
-                        
                         
                         
                         
@@ -4457,13 +4473,12 @@
                         
                     
                 
-                1.F5177563B1D7F1A4A055D706B82C6734
+                1.6B1C8916BB856B9EEF24DE4CB561989B
             
             
                 
                     
                         
-                        
                         
                         
                         
@@ -4486,7 +4501,7 @@
                         
                     
                 
-                1.B8DE45F9011916BD826DFA339DD3646B
+                1.1F6F3DD14D68AD328398FA0C8C1BA523
             
             
                 
@@ -6337,6 +6352,7 @@
                         
                         
                         
+                        
                         
                         
                         
@@ -6352,8 +6368,14 @@
                             
                         
                     
+                    
+                        500
+                        
+                            
+                        
+                    
                 
-                1.4614C67BE4C4B0B22F40F86B47B40D2D
+                1.0BFC9FE27D3A590769329D70A0E2E872
             
             
                 
@@ -6478,6 +6500,29 @@
                 
                 1.AD707ECC69A1E835FADE9B7B896C7C53
             
+            
+                
+                    
+                        
+                    
+                    
+                
+                
+                    
+                        200
+                        
+                            
+                        
+                    
+                    
+                        400
+                        
+                            
+                        
+                    
+                
+                1.2EC7C7222330CC8A26AD23348C22DFBD
+            
             
                 
                     
@@ -6569,6 +6614,7 @@
             
                 
                     
+                        
                         
                     
                     
@@ -6584,11 +6630,12 @@
                         
                     
                 
-                1.6E171A954319F5E7A916A6A47BB1BB39
+                1.7D48EAAB19465D9008EFE5C6E3C020EA
             
             
                 
                     
+                        
                         
                     
                     
@@ -6604,7 +6651,7 @@
                         
                     
                 
-                1.55C29670AA54CE2E07EA9C9D31E82EAC
+                1.3669C6C2F1550E1554DB9CF129DCD965
             
             
                 
@@ -7754,6 +7801,12 @@
                             
                         
                     
+                    
+                        409
+                        
+                            
+                        
+                    
                 
                 1.0FC6240C9AA47CA11A06C45AF5EED527
             
@@ -8601,7 +8654,7 @@
                             
                             
                                 
-                                    
+                                    
                                     
                                 
                             
@@ -8614,7 +8667,7 @@
                             
                             
                                 
-                                    
+                                    
                                     
                                 
                             
@@ -8743,7 +8796,7 @@
                             
                             
                                 
-                                    
+                                    
                                     
                                 
                             
@@ -8756,7 +8809,7 @@
                             
                             
                                 
-                                    
+                                    
                                     
                                 
                             
@@ -9547,12 +9600,21 @@
                             
                             
                                 
-                                    
+                                    
                                     
                                 
                             
                         
                     
+                    
+                        
+                            
+                                
+                                    
+                                
+                            
+                        
+                    
                     
                         
                             
@@ -12239,6 +12301,11 @@
                             
                         
                     
+                    
+                        
+                            
+                        
+                    
                     
                         
                             
@@ -12417,6 +12484,8 @@
                     
                     
                     
+                    
+                    
                 
             
             
@@ -12733,6 +12802,15 @@
             
             
                 
+                    
+                        
+                            
+                                
+                                    
+                                
+                            
+                        
+                    
                     
                         
                             
@@ -12759,6 +12837,13 @@
                             
                         
                     
+                    
+                        
+                            
+                                
+                            
+                        
+                    
                     
                         
                             
@@ -12773,6 +12858,15 @@
                             
                         
                     
+                    
+                        
+                            
+                                
+                                    
+                                
+                            
+                        
+                    
                     
                         
                             
@@ -12865,6 +12959,12 @@
                             
                         
                     
+                    
+                        
+                            
+                            
+                        
+                    
                     
                         
                             
@@ -12986,7 +13086,7 @@
                     
                     
                         
-                            
+                            
                         
                     
                     
@@ -13004,6 +13104,11 @@
                             
                         
                     
+                    
+                        
+                            
+                        
+                    
                     
                         
                             
@@ -13124,6 +13229,9 @@
                             
                         
                     
+                    
+                        
+                    
                     
                         
                             
@@ -13252,6 +13360,11 @@
                             
                         
                     
+                    
+                        
+                            
+                        
+                    
                     
                         
                             
@@ -13572,6 +13685,7 @@
                     
                     
                     
+                    
                     
                     
                     
@@ -13610,6 +13724,11 @@
                                     
                                 
                             
+                            
+                                
+                                    
+                                
+                            
                         
                     
                     
@@ -13619,6 +13738,11 @@
                                     
                                 
                             
+                            
+                                
+                                    
+                                
+                            
                         
                     
                     
@@ -14052,6 +14176,11 @@
                                     
                                 
                             
+                            
+                                
+                                    
+                                
+                            
                         
                     
                     
@@ -14061,6 +14190,11 @@
                                     
                                 
                             
+                            
+                                
+                                    
+                                
+                            
                         
                     
                     
@@ -14478,6 +14612,11 @@
                             
                         
                     
+                    
+                        
+                            
+                        
+                    
                     
                         
                             
@@ -15397,6 +15536,9 @@
                             
                         
                     
+                    
+                        
+                    
                     
                         
                             
@@ -15496,8 +15638,13 @@
                             
                         
                     
-                    
+                    
                         
+                            
+                                
+                                    
+                                
+                            
                             
                                 
                             
@@ -15685,8 +15832,13 @@
                             
                         
                     
-                    
+                    
                         
+                            
+                                
+                                    
+                                
+                            
                             
                                 
                             
@@ -16246,6 +16398,9 @@
                             
                         
                     
+                    
+                        
+                    
                     
                         
                             

From 6a005136381958812a761f1982074d61f16b6b75 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Tue, 27 Jun 2017 14:22:06 -0600
Subject: [PATCH 066/229] Updated tests in UsersAndGroups to reflect change in
 4.0.0 api

---
 .../ds3client/integration/Util.java           |  4 +-
 .../integration/UsersAndGroups_Test.java      | 82 +++++++++++--------
 2 files changed, 49 insertions(+), 37 deletions(-)

diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index af8b2c6c6..144814b52 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -65,10 +65,10 @@ public static void assumeVersion1_2(final Ds3Client client) throws IOException {
         assumeThat(majorVersion, is(1));
     }
 
-    public static void assumeVersion3orLess(final Ds3Client client) throws IOException {
+    public static void assumeVersion4orHigher(final Ds3Client client) throws IOException {
         final int majorVersion = Integer.parseInt(client.getSystemInformationSpectraS3(
                 new GetSystemInformationSpectraS3Request()).getSystemInformationResult().getBuildInformation().getVersion().split("\\.")[0]);
-        assumeTrue(majorVersion <= 3);
+        assumeTrue(majorVersion >= 4);
     }
 
     public static void loadBookTestData(final Ds3Client client, final String bucketName) throws IOException, URISyntaxException {
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
index 1cf1da7c3..ebce894cb 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
@@ -33,7 +33,7 @@
 import java.security.SignatureException;
 import java.util.UUID;
 
-import static com.spectralogic.ds3client.integration.Util.assumeVersion3orLess;
+import static com.spectralogic.ds3client.integration.Util.assumeVersion4orHigher;
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
 import static com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil.DEFAULT_USER;
 import static org.hamcrest.Matchers.is;
@@ -181,6 +181,9 @@ public void ModifyUserDefaultDataPolicy() throws IOException, SignatureException
 
         final GetUserSpectraS3Response getSpectraResponse = client
                 .getUserSpectraS3(new GetUserSpectraS3Request(spectraUUID));
+
+        assertThat(getSpectraResponse.getSpectraUserResult(), is(notNullValue()));
+
         final ModifyUserSpectraS3Response modifyUserSpectraS3Response = client
                 .modifyUserSpectraS3(new ModifyUserSpectraS3Request(spectraUUID)
                         .withDefaultDataPolicyId(dataPolicyId));
@@ -341,7 +344,7 @@ public void getGroupsWithPageStartMarker() throws IOException, SignatureExceptio
                     .withPageStartMarker(UUID.randomUUID()));
             fail("The above should not be found and throw exception.");
         } catch (final FailedRequestException e) {
-            assertThat(e.getStatusCode(), is(410));
+            assertThat(e.getStatusCode(), is(404));
         }
     }
 
@@ -547,24 +550,28 @@ public void getGroupGroupMembersWithLastPage() throws IOException, SignatureExce
 
     @Test
     public void getGroupGroupMembersWithPageStartMarkerUUID() throws IOException, SignatureException {
-        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
+        assumeVersion4orHigher(client);
 
-        final GetGroupMembersSpectraS3Response getGroupMembersSpectraS3Response = client
-                .getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
-                        .withPageStartMarker(UUID.randomUUID()));
+        try {
+            client.getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
+                    .withPageStartMarker(UUID.randomUUID()));
 
-        assertThat(getGroupMembersSpectraS3Response.getGroupMemberListResult(), is(notNullValue()));
+        } catch (final FailedRequestException e) {
+            assertThat(e.getStatusCode(), is(404));
+        }
     }
 
     @Test
     public void getGroupGroupMembersWithPageStartMarkerString() throws IOException, SignatureException {
-        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
+        assumeVersion4orHigher(client);
 
-        final GetGroupMembersSpectraS3Response getGroupMembersSpectraS3Response = client
-                .getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
-                        .withPageStartMarker(UUID.randomUUID().toString()));
+        try {
+            client.getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
+                    .withPageStartMarker(UUID.randomUUID().toString()));
 
-        assertThat(getGroupMembersSpectraS3Response.getGroupMemberListResult(), is(notNullValue()));
+        } catch (final FailedRequestException e) {
+            assertThat(e.getStatusCode(), is(404));
+        }
     }
 
     @Test
@@ -854,24 +861,27 @@ public void getBucketAclsWithLastPage() throws IOException, SignatureException {
 
     @Test
     public void getBucketAclsWithPageStartMarkerUUID() throws IOException, SignatureException {
-        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
+        assumeVersion4orHigher(client);
 
-        final GetBucketAclsSpectraS3Response getBucketAclsSpectraS3Response = client
-                .getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request()
-                        .withPageStartMarker(UUID.randomUUID()));
+        try {
+            client.getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request()
+                    .withPageStartMarker(UUID.randomUUID()));
 
-        assertThat(getBucketAclsSpectraS3Response.getBucketAclListResult(), is(notNullValue()));
+        } catch (final FailedRequestException e) {
+            assertThat(e.getStatusCode(), is(404));
+        }
     }
 
     @Test
     public void getBucketAclsWithPageStartMarkerString() throws IOException, SignatureException {
-        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
+        assumeVersion4orHigher(client);
 
-        final GetBucketAclsSpectraS3Response getBucketAclsSpectraS3Response = client
-                .getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request()
-                        .withPageStartMarker(UUID.randomUUID().toString()));
-
-        assertThat(getBucketAclsSpectraS3Response.getBucketAclListResult(), is(notNullValue()));
+        try {
+            client.getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request()
+                    .withPageStartMarker(UUID.randomUUID().toString()));
+        } catch (final FailedRequestException e) {
+            assertThat(e.getStatusCode(), is(404));
+        }
     }
 
     @Test
@@ -1189,24 +1199,26 @@ public void getDataPolicyAclsWithPageLength() throws IOException, SignatureExcep
 
     @Test
     public void getDataPolicyAclsWithPageStartMarkerString() throws IOException, SignatureException {
-        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
-
-        final GetDataPolicyAclsSpectraS3Response getDataPolicyAclsSpectraS3Response = client
-                .getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()
-                        .withPageStartMarker(UUID.randomUUID().toString()));
+        assumeVersion4orHigher(client);
 
-        assertThat(getDataPolicyAclsSpectraS3Response.getDataPolicyAclListResult(), is(notNullValue()));
+        try {
+            client.getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()
+                    .withPageStartMarker(UUID.randomUUID().toString()));
+        } catch (final FailedRequestException e) {
+            assertThat(e.getStatusCode(), is(404));
+        }
     }
 
     @Test
     public void getDataPolicyAclsWithPageStartMarkerUUID() throws IOException, SignatureException {
-        assumeVersion3orLess(client); //TODO update once 4.0 error code is no longer 410
-
-        final GetDataPolicyAclsSpectraS3Response getDataPolicyAclsSpectraS3Response = client
-                .getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()
-                        .withPageStartMarker(UUID.randomUUID()));
+        assumeVersion4orHigher(client);
 
-        assertThat(getDataPolicyAclsSpectraS3Response.getDataPolicyAclListResult(), is(notNullValue()));
+        try {
+            client.getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()
+                    .withPageStartMarker(UUID.randomUUID()));
+        } catch (final FailedRequestException e) {
+            assertThat(e.getStatusCode(), is(404));
+        }
     }
 
     @Test

From 7697334c88829e947dbecc4a277856c798fae940 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Tue, 27 Jun 2017 14:41:19 -0600
Subject: [PATCH 067/229] Fixed oversight of updated tests

---
 .../ds3client/integration/UsersAndGroups_Test.java          | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
index ebce894cb..4ef77f7e7 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
@@ -555,6 +555,7 @@ public void getGroupGroupMembersWithPageStartMarkerUUID() throws IOException, Si
         try {
             client.getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
                     .withPageStartMarker(UUID.randomUUID()));
+            fail("The above should not be found and throw exception.");
 
         } catch (final FailedRequestException e) {
             assertThat(e.getStatusCode(), is(404));
@@ -568,6 +569,7 @@ public void getGroupGroupMembersWithPageStartMarkerString() throws IOException,
         try {
             client.getGroupMembersSpectraS3(new GetGroupMembersSpectraS3Request()
                     .withPageStartMarker(UUID.randomUUID().toString()));
+            fail("The above should not be found and throw exception.");
 
         } catch (final FailedRequestException e) {
             assertThat(e.getStatusCode(), is(404));
@@ -866,6 +868,7 @@ public void getBucketAclsWithPageStartMarkerUUID() throws IOException, Signature
         try {
             client.getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request()
                     .withPageStartMarker(UUID.randomUUID()));
+            fail("The above should not be found and throw exception.");
 
         } catch (final FailedRequestException e) {
             assertThat(e.getStatusCode(), is(404));
@@ -879,6 +882,7 @@ public void getBucketAclsWithPageStartMarkerString() throws IOException, Signatu
         try {
             client.getBucketAclsSpectraS3(new GetBucketAclsSpectraS3Request()
                     .withPageStartMarker(UUID.randomUUID().toString()));
+            fail("The above should not be found and throw exception.");
         } catch (final FailedRequestException e) {
             assertThat(e.getStatusCode(), is(404));
         }
@@ -1204,6 +1208,7 @@ public void getDataPolicyAclsWithPageStartMarkerString() throws IOException, Sig
         try {
             client.getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()
                     .withPageStartMarker(UUID.randomUUID().toString()));
+            fail("The above should not be found and throw exception.");
         } catch (final FailedRequestException e) {
             assertThat(e.getStatusCode(), is(404));
         }
@@ -1216,6 +1221,7 @@ public void getDataPolicyAclsWithPageStartMarkerUUID() throws IOException, Signa
         try {
             client.getDataPolicyAclsSpectraS3(new GetDataPolicyAclsSpectraS3Request()
                     .withPageStartMarker(UUID.randomUUID()));
+            fail("The above should not be found and throw exception.");
         } catch (final FailedRequestException e) {
             assertThat(e.getStatusCode(), is(404));
         }

From 8951e6f7b4c4620f6de0172a3faa9f5732cb6031 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 7 Jul 2017 16:03:30 -0600
Subject: [PATCH 068/229] Created PutFolder command which wraps PutObject

---
 .../helpers/FileSystemHelper_Test.java        | 49 +++++++++++++++----
 .../com/spectralogic/ds3client/Ds3Client.java |  4 ++
 .../spectralogic/ds3client/Ds3ClientImpl.java |  8 +++
 .../commands/decorators/PutFolderRequest.java | 48 ++++++++++++++++++
 .../decorators/PutFolderResponse.java         | 29 +++++++++++
 5 files changed, 128 insertions(+), 10 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
index c38e226ef..15e6d7159 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
@@ -15,10 +15,11 @@
 
 package com.spectralogic.ds3client.helpers;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Response;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.integration.Util;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
@@ -31,6 +32,8 @@
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -39,17 +42,15 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.*;
 
 public class FileSystemHelper_Test {
     private static final Logger LOG = LoggerFactory.getLogger(FileSystemHelper_Test.class);
@@ -313,4 +314,32 @@ public long getAvailableFileSpace(final Path path) throws IOException {
             throw new IOException("IOExceptionAtor");
         }
     }
+
+    @Test
+    public void createFolderWithSlash() throws IOException {
+        final String folderName = "FolderNameWithSlash/";
+
+        try {
+            final Ds3Object ds3Object = new Ds3Object(folderName, 0);
+            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(BUCKET_NAME, ImmutableList.of(ds3Object)));
+
+            client.putFolder(new PutFolderRequest(BUCKET_NAME, folderName, jobResponse.getMasterObjectList().getJobId()));
+        } finally {
+            deleteAllContents(client, BUCKET_NAME);
+        }
+    }
+
+    @Test
+    public void createFolderWithNoSlash() throws IOException {
+        final String folderName = "FolderNameNoSlash";
+
+        try {
+            final Ds3Object ds3Object = new Ds3Object(folderName + "/", 0);
+            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(BUCKET_NAME, ImmutableList.of(ds3Object)));
+
+            client.putFolder(new PutFolderRequest(BUCKET_NAME, folderName, jobResponse.getMasterObjectList().getJobId()));
+        } finally {
+            deleteAllContents(client, BUCKET_NAME);
+        }
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index 0268450de..514948d2f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -20,6 +20,8 @@
 import com.spectralogic.ds3client.annotations.Resource;
 import com.spectralogic.ds3client.annotations.ResponsePayloadModel;
 import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.commands.spectrads3.notifications.*;
 import com.spectralogic.ds3client.models.JobNode;
@@ -35,6 +37,8 @@ public interface Ds3Client extends Closeable {
 
     ConnectionDetails getConnectionDetails();
 
+    PutFolderResponse putFolder(final PutFolderRequest request)
+            throws IOException;
     
     
     AbortMultiPartUploadResponse abortMultiPartUpload(final AbortMultiPartUploadRequest request)
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
index 8a92e5bb3..9e29d492e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
@@ -18,6 +18,8 @@
 
 import java.io.IOException;
 import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.parsers.*;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.commands.spectrads3.notifications.*;
@@ -45,6 +47,12 @@ public ConnectionDetails getConnectionDetails() {
         return this.netClient.getConnectionDetails();
     }
 
+    @Override
+    public PutFolderResponse putFolder(final PutFolderRequest request) throws IOException {
+        final PutObjectResponse response = putObject(request.getPutObjectRequest());
+        return new PutFolderResponse(response);
+    }
+
     @Override
     public AbortMultiPartUploadResponse abortMultiPartUpload(final AbortMultiPartUploadRequest request) throws IOException {
         return new AbortMultiPartUploadResponseParser().response(this.netClient.getResponse(request));
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
new file mode 100644
index 000000000..114ff396c
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
@@ -0,0 +1,48 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.commands.decorators;
+
+import com.spectralogic.ds3client.commands.PutObjectRequest;
+
+import java.util.UUID;
+
+/**
+ * Decorates the {@link com.spectralogic.ds3client.commands.PutObjectRequest} and is used to
+ * ensure the correct creation of a folder using the Put Object command.
+ *
+ * A folder is created by putting an object with no content, of zero size, and with the
+ * content-length=0.  Also, the name of the folder must end with a slash '/'
+ */
+public class PutFolderRequest {
+
+    private final PutObjectRequest putObjectRequest;
+
+    public PutFolderRequest(final String bucketName, final String folderName, final UUID job) {
+        putObjectRequest = new PutObjectRequest(bucketName, ensureEndsWithSlash(folderName), job, 0, 0, null);
+        putObjectRequest.getHeaders().put("Content-Length", "0");
+    }
+
+    private static String ensureEndsWithSlash(final String bucketName) {
+        if (bucketName.endsWith("/")) {
+            return bucketName;
+        }
+        return bucketName + "/";
+    }
+
+    public PutObjectRequest getPutObjectRequest() {
+        return putObjectRequest;
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java
new file mode 100644
index 000000000..1707e7996
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java
@@ -0,0 +1,29 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.commands.decorators;
+
+import com.spectralogic.ds3client.commands.PutObjectResponse;
+
+/**
+ * Decorates the {@link com.spectralogic.ds3client.commands.PutObjectResponse} and is used
+ * by command {@link com.spectralogic.ds3client.Ds3Client#putFolder(PutFolderRequest)}
+ */
+public class PutFolderResponse extends PutObjectResponse {
+
+    public PutFolderResponse(final PutObjectResponse response) {
+        super(response.getChecksum(), response.getChecksumType());
+    }
+}

From 167ce51a0724b7c9020dd95c71d14406f93c7e16 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 10 Jul 2017 12:47:27 -0600
Subject: [PATCH 069/229] Added helper for creating folder

---
 .../helpers/FileSystemHelper_Test.java        | 17 ++++--------
 .../ds3client/integration/Smoke_Test.java     | 20 ++++++++++++++
 .../commands/decorators/PutFolderRequest.java | 13 ++++-----
 ...FolderNameMissingTrailingForwardSlash.java | 26 ++++++++++++++++++
 .../ds3client/helpers/Ds3ClientHelpers.java   |  6 +++++
 .../helpers/Ds3ClientHelpersImpl.java         | 27 +++++++++++++++++++
 .../ds3client/Ds3Client_Test.java             |  7 +++++
 7 files changed, 98 insertions(+), 18 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/FolderNameMissingTrailingForwardSlash.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
index 15e6d7159..badd3dddc 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
@@ -15,11 +15,8 @@
 
 package com.spectralogic.ds3client.helpers;
 
-import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
-import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
-import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Response;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.integration.Util;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
@@ -320,10 +317,8 @@ public void createFolderWithSlash() throws IOException {
         final String folderName = "FolderNameWithSlash/";
 
         try {
-            final Ds3Object ds3Object = new Ds3Object(folderName, 0);
-            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(BUCKET_NAME, ImmutableList.of(ds3Object)));
-
-            client.putFolder(new PutFolderRequest(BUCKET_NAME, folderName, jobResponse.getMasterObjectList().getJobId()));
+            final PutFolderResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
+            assertNotNull(response);
         } finally {
             deleteAllContents(client, BUCKET_NAME);
         }
@@ -334,10 +329,8 @@ public void createFolderWithNoSlash() throws IOException {
         final String folderName = "FolderNameNoSlash";
 
         try {
-            final Ds3Object ds3Object = new Ds3Object(folderName + "/", 0);
-            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(BUCKET_NAME, ImmutableList.of(ds3Object)));
-
-            client.putFolder(new PutFolderRequest(BUCKET_NAME, folderName, jobResponse.getMasterObjectList().getJobId()));
+            final PutFolderResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
+            assertNotNull(response);
         } finally {
             deleteAllContents(client, BUCKET_NAME);
         }
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
index 5df2ef496..60e27d24f 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
@@ -15,11 +15,14 @@
 
 package com.spectralogic.ds3client.integration;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.interfaces.BulkResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.*;
@@ -1656,4 +1659,21 @@ public void testGetObjectsWithFullDetails() throws IOException, URISyntaxExcepti
             deleteAllContents(client, bucketName);
         }
     }
+
+    @Test
+    public void createFolderWithSlash() throws IOException {
+        final String folderName = "FolderNameWithSlash/";
+        final String bucketName = "FolderNameWithSlashTestBucket";
+
+        try {
+            HELPERS.ensureBucketExists(bucketName, envDataPolicyId);
+            final Ds3Object ds3Object = new Ds3Object(folderName, 0);
+            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucketName, ImmutableList.of(ds3Object)));
+
+            final PutFolderResponse response = client.putFolder(new PutFolderRequest(bucketName, folderName, jobResponse.getMasterObjectList().getJobId()));
+            assertNotNull(response);
+        } finally {
+            deleteAllContents(client, bucketName);
+        }
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
index 114ff396c..8f0b0d55e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
@@ -16,6 +16,7 @@
 package com.spectralogic.ds3client.commands.decorators;
 
 import com.spectralogic.ds3client.commands.PutObjectRequest;
+import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 
 import java.util.UUID;
 
@@ -24,22 +25,22 @@
  * ensure the correct creation of a folder using the Put Object command.
  *
  * A folder is created by putting an object with no content, of zero size, and with the
- * content-length=0.  Also, the name of the folder must end with a slash '/'
+ * content-length=0.  Also, the name of the folder must end with a forward slash '/'.
  */
 public class PutFolderRequest {
 
     private final PutObjectRequest putObjectRequest;
 
     public PutFolderRequest(final String bucketName, final String folderName, final UUID job) {
-        putObjectRequest = new PutObjectRequest(bucketName, ensureEndsWithSlash(folderName), job, 0, 0, null);
+        validateNameEndsWithSlash(folderName);
+        putObjectRequest = new PutObjectRequest(bucketName, folderName, job, 0, 0, null);
         putObjectRequest.getHeaders().put("Content-Length", "0");
     }
 
-    private static String ensureEndsWithSlash(final String bucketName) {
-        if (bucketName.endsWith("/")) {
-            return bucketName;
+    private static void validateNameEndsWithSlash(final String folderName) {
+        if (!folderName.endsWith("/")) {
+            throw new FolderNameMissingTrailingForwardSlash(folderName);
         }
-        return bucketName + "/";
     }
 
     public PutObjectRequest getPutObjectRequest() {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/FolderNameMissingTrailingForwardSlash.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/FolderNameMissingTrailingForwardSlash.java
new file mode 100644
index 000000000..25614be28
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/FolderNameMissingTrailingForwardSlash.java
@@ -0,0 +1,26 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.exceptions;
+
+/**
+ * Denotes an invalid folder name. Folder names must end with a forward slash '/'.
+ */
+public class FolderNameMissingTrailingForwardSlash extends IllegalArgumentException {
+
+    public FolderNameMissingTrailingForwardSlash(final String folderName) {
+        super("Invalid folder name '" + folderName + "': folder names must end with a forward slash '/'.");
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 0e6072830..5845551c6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -18,6 +18,7 @@
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
@@ -580,4 +581,9 @@ public static String stripLeadingPath(final String objectName, final String pref
     public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(final String bucketName,
                                                                                              final Collection objectNames,
                                                                                              final Path destinationDirectory);
+
+    /**
+     * Creates a folder in the specified bucket
+     */
+    public abstract PutFolderResponse createFolder(final String bucketName, final String folderName) throws IOException;
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 3d08bf8e7..94e2b609d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -25,7 +25,10 @@
 import com.spectralogic.ds3client.commands.HeadBucketRequest;
 import com.spectralogic.ds3client.commands.HeadBucketResponse;
 import com.spectralogic.ds3client.commands.PutBucketRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
+import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
@@ -593,4 +596,28 @@ public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(
         return fileSystemHelper.objectsFromBucketWillFitInDirectory(this,
                 bucketName, objectNames, destinationDirectory);
     }
+
+    /**
+     * Creates a folder in the specified bucket.
+     */
+    @Override
+    public PutFolderResponse createFolder(final String bucketName, final String folderName) throws IOException {
+        final String normalizedFolderName = ensureNameEndsWithSlash(folderName);
+        final Ds3Object ds3Object = new Ds3Object(normalizedFolderName, 0);
+        final PutBulkJobSpectraS3Response jobResponse = client
+                .putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucketName, ImmutableList.of(ds3Object)));
+
+        return client.putFolder(new PutFolderRequest(bucketName, normalizedFolderName, jobResponse.getMasterObjectList().getJobId()));
+    }
+
+    /**
+     * Ensures that a folder names ends with a trailing forward slash. This prevents the
+     * accidental creation of zero-length files when attempting to create a folder.
+     */
+    private static String ensureNameEndsWithSlash(final String folderName) {
+        if (folderName.endsWith("/")) {
+            return folderName;
+        }
+        return folderName + "/";
+    }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 506e5b441..637799069 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -20,8 +20,10 @@
 import com.google.common.collect.Multimap;
 import com.google.common.collect.TreeMultimap;
 import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException;
+import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 import com.spectralogic.ds3client.models.*;
 import com.spectralogic.ds3client.models.Objects;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
@@ -1136,4 +1138,9 @@ public void getObjectVerifyFullPayload() throws IOException {
                         jobIdString,
                         0));
     }
+
+    @Test (expected = FolderNameMissingTrailingForwardSlash.class)
+    public void createFolderWithNoSlash() throws IOException {
+        new PutFolderRequest("BucketName", "FolderNameNoSlash", UUID.randomUUID());
+    }
 }

From 414f5ac378409710ee8ba51d797aad8d31298600 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 10 Jul 2017 12:53:23 -0600
Subject: [PATCH 070/229] Removed unused import

---
 .../com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 94e2b609d..8b19d4d26 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -28,7 +28,6 @@
 import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
 import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
-import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;

From 97a51ed4d6a5d580d468fcc71e69ede4613adbe8 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 7 Jul 2017 16:03:30 -0600
Subject: [PATCH 071/229] Created PutFolder command which wraps PutObject

---
 .../helpers/FileSystemHelper_Test.java        | 49 +++++++++++++++----
 .../com/spectralogic/ds3client/Ds3Client.java |  4 ++
 .../spectralogic/ds3client/Ds3ClientImpl.java |  8 +++
 .../commands/decorators/PutFolderRequest.java | 48 ++++++++++++++++++
 .../decorators/PutFolderResponse.java         | 29 +++++++++++
 5 files changed, 128 insertions(+), 10 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
index c38e226ef..15e6d7159 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
@@ -15,10 +15,11 @@
 
 package com.spectralogic.ds3client.helpers;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Response;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.integration.Util;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
@@ -31,6 +32,8 @@
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -39,17 +42,15 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.*;
 
 public class FileSystemHelper_Test {
     private static final Logger LOG = LoggerFactory.getLogger(FileSystemHelper_Test.class);
@@ -313,4 +314,32 @@ public long getAvailableFileSpace(final Path path) throws IOException {
             throw new IOException("IOExceptionAtor");
         }
     }
+
+    @Test
+    public void createFolderWithSlash() throws IOException {
+        final String folderName = "FolderNameWithSlash/";
+
+        try {
+            final Ds3Object ds3Object = new Ds3Object(folderName, 0);
+            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(BUCKET_NAME, ImmutableList.of(ds3Object)));
+
+            client.putFolder(new PutFolderRequest(BUCKET_NAME, folderName, jobResponse.getMasterObjectList().getJobId()));
+        } finally {
+            deleteAllContents(client, BUCKET_NAME);
+        }
+    }
+
+    @Test
+    public void createFolderWithNoSlash() throws IOException {
+        final String folderName = "FolderNameNoSlash";
+
+        try {
+            final Ds3Object ds3Object = new Ds3Object(folderName + "/", 0);
+            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(BUCKET_NAME, ImmutableList.of(ds3Object)));
+
+            client.putFolder(new PutFolderRequest(BUCKET_NAME, folderName, jobResponse.getMasterObjectList().getJobId()));
+        } finally {
+            deleteAllContents(client, BUCKET_NAME);
+        }
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index 559dfc55e..8e356c187 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -20,6 +20,8 @@
 import com.spectralogic.ds3client.annotations.Resource;
 import com.spectralogic.ds3client.annotations.ResponsePayloadModel;
 import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.commands.spectrads3.notifications.*;
 import com.spectralogic.ds3client.models.JobNode;
@@ -35,6 +37,8 @@ public interface Ds3Client extends Closeable {
 
     ConnectionDetails getConnectionDetails();
 
+    PutFolderResponse putFolder(final PutFolderRequest request)
+            throws IOException;
     
     
     AbortMultiPartUploadResponse abortMultiPartUpload(final AbortMultiPartUploadRequest request)
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
index a04ac9ca7..16c5bd388 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
@@ -18,6 +18,8 @@
 
 import java.io.IOException;
 import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.parsers.*;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.commands.spectrads3.notifications.*;
@@ -45,6 +47,12 @@ public ConnectionDetails getConnectionDetails() {
         return this.netClient.getConnectionDetails();
     }
 
+    @Override
+    public PutFolderResponse putFolder(final PutFolderRequest request) throws IOException {
+        final PutObjectResponse response = putObject(request.getPutObjectRequest());
+        return new PutFolderResponse(response);
+    }
+
     @Override
     public AbortMultiPartUploadResponse abortMultiPartUpload(final AbortMultiPartUploadRequest request) throws IOException {
         return new AbortMultiPartUploadResponseParser().response(this.netClient.getResponse(request));
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
new file mode 100644
index 000000000..114ff396c
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
@@ -0,0 +1,48 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.commands.decorators;
+
+import com.spectralogic.ds3client.commands.PutObjectRequest;
+
+import java.util.UUID;
+
+/**
+ * Decorates the {@link com.spectralogic.ds3client.commands.PutObjectRequest} and is used to
+ * ensure the correct creation of a folder using the Put Object command.
+ *
+ * A folder is created by putting an object with no content, of zero size, and with the
+ * content-length=0.  Also, the name of the folder must end with a slash '/'
+ */
+public class PutFolderRequest {
+
+    private final PutObjectRequest putObjectRequest;
+
+    public PutFolderRequest(final String bucketName, final String folderName, final UUID job) {
+        putObjectRequest = new PutObjectRequest(bucketName, ensureEndsWithSlash(folderName), job, 0, 0, null);
+        putObjectRequest.getHeaders().put("Content-Length", "0");
+    }
+
+    private static String ensureEndsWithSlash(final String bucketName) {
+        if (bucketName.endsWith("/")) {
+            return bucketName;
+        }
+        return bucketName + "/";
+    }
+
+    public PutObjectRequest getPutObjectRequest() {
+        return putObjectRequest;
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java
new file mode 100644
index 000000000..1707e7996
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java
@@ -0,0 +1,29 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.commands.decorators;
+
+import com.spectralogic.ds3client.commands.PutObjectResponse;
+
+/**
+ * Decorates the {@link com.spectralogic.ds3client.commands.PutObjectResponse} and is used
+ * by command {@link com.spectralogic.ds3client.Ds3Client#putFolder(PutFolderRequest)}
+ */
+public class PutFolderResponse extends PutObjectResponse {
+
+    public PutFolderResponse(final PutObjectResponse response) {
+        super(response.getChecksum(), response.getChecksumType());
+    }
+}

From c4014cbaf754d2f8dd5bb7af6ef1f0d5847439f2 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 10 Jul 2017 12:47:27 -0600
Subject: [PATCH 072/229] Added helper for creating folder

---
 .../helpers/FileSystemHelper_Test.java        | 17 ++++--------
 .../ds3client/integration/Smoke_Test.java     | 20 ++++++++++++++
 .../commands/decorators/PutFolderRequest.java | 13 ++++-----
 ...FolderNameMissingTrailingForwardSlash.java | 26 ++++++++++++++++++
 .../ds3client/helpers/Ds3ClientHelpers.java   |  6 +++++
 .../helpers/Ds3ClientHelpersImpl.java         | 27 +++++++++++++++++++
 .../ds3client/Ds3Client_Test.java             |  7 +++++
 7 files changed, 98 insertions(+), 18 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/FolderNameMissingTrailingForwardSlash.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
index 15e6d7159..badd3dddc 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
@@ -15,11 +15,8 @@
 
 package com.spectralogic.ds3client.helpers;
 
-import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
-import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
-import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Response;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.integration.Util;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
@@ -320,10 +317,8 @@ public void createFolderWithSlash() throws IOException {
         final String folderName = "FolderNameWithSlash/";
 
         try {
-            final Ds3Object ds3Object = new Ds3Object(folderName, 0);
-            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(BUCKET_NAME, ImmutableList.of(ds3Object)));
-
-            client.putFolder(new PutFolderRequest(BUCKET_NAME, folderName, jobResponse.getMasterObjectList().getJobId()));
+            final PutFolderResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
+            assertNotNull(response);
         } finally {
             deleteAllContents(client, BUCKET_NAME);
         }
@@ -334,10 +329,8 @@ public void createFolderWithNoSlash() throws IOException {
         final String folderName = "FolderNameNoSlash";
 
         try {
-            final Ds3Object ds3Object = new Ds3Object(folderName + "/", 0);
-            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(BUCKET_NAME, ImmutableList.of(ds3Object)));
-
-            client.putFolder(new PutFolderRequest(BUCKET_NAME, folderName, jobResponse.getMasterObjectList().getJobId()));
+            final PutFolderResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
+            assertNotNull(response);
         } finally {
             deleteAllContents(client, BUCKET_NAME);
         }
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
index 0cabe88b3..616f31ada 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
@@ -15,11 +15,14 @@
 
 package com.spectralogic.ds3client.integration;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.interfaces.BulkResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.*;
@@ -1656,4 +1659,21 @@ public void testGetObjectsWithFullDetails() throws IOException, URISyntaxExcepti
             deleteAllContents(client, bucketName);
         }
     }
+
+    @Test
+    public void createFolderWithSlash() throws IOException {
+        final String folderName = "FolderNameWithSlash/";
+        final String bucketName = "FolderNameWithSlashTestBucket";
+
+        try {
+            HELPERS.ensureBucketExists(bucketName, envDataPolicyId);
+            final Ds3Object ds3Object = new Ds3Object(folderName, 0);
+            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucketName, ImmutableList.of(ds3Object)));
+
+            final PutFolderResponse response = client.putFolder(new PutFolderRequest(bucketName, folderName, jobResponse.getMasterObjectList().getJobId()));
+            assertNotNull(response);
+        } finally {
+            deleteAllContents(client, bucketName);
+        }
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
index 114ff396c..8f0b0d55e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
@@ -16,6 +16,7 @@
 package com.spectralogic.ds3client.commands.decorators;
 
 import com.spectralogic.ds3client.commands.PutObjectRequest;
+import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 
 import java.util.UUID;
 
@@ -24,22 +25,22 @@
  * ensure the correct creation of a folder using the Put Object command.
  *
  * A folder is created by putting an object with no content, of zero size, and with the
- * content-length=0.  Also, the name of the folder must end with a slash '/'
+ * content-length=0.  Also, the name of the folder must end with a forward slash '/'.
  */
 public class PutFolderRequest {
 
     private final PutObjectRequest putObjectRequest;
 
     public PutFolderRequest(final String bucketName, final String folderName, final UUID job) {
-        putObjectRequest = new PutObjectRequest(bucketName, ensureEndsWithSlash(folderName), job, 0, 0, null);
+        validateNameEndsWithSlash(folderName);
+        putObjectRequest = new PutObjectRequest(bucketName, folderName, job, 0, 0, null);
         putObjectRequest.getHeaders().put("Content-Length", "0");
     }
 
-    private static String ensureEndsWithSlash(final String bucketName) {
-        if (bucketName.endsWith("/")) {
-            return bucketName;
+    private static void validateNameEndsWithSlash(final String folderName) {
+        if (!folderName.endsWith("/")) {
+            throw new FolderNameMissingTrailingForwardSlash(folderName);
         }
-        return bucketName + "/";
     }
 
     public PutObjectRequest getPutObjectRequest() {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/FolderNameMissingTrailingForwardSlash.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/FolderNameMissingTrailingForwardSlash.java
new file mode 100644
index 000000000..25614be28
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/FolderNameMissingTrailingForwardSlash.java
@@ -0,0 +1,26 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.exceptions;
+
+/**
+ * Denotes an invalid folder name. Folder names must end with a forward slash '/'.
+ */
+public class FolderNameMissingTrailingForwardSlash extends IllegalArgumentException {
+
+    public FolderNameMissingTrailingForwardSlash(final String folderName) {
+        super("Invalid folder name '" + folderName + "': folder names must end with a forward slash '/'.");
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 0e6072830..5845551c6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -18,6 +18,7 @@
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
@@ -580,4 +581,9 @@ public static String stripLeadingPath(final String objectName, final String pref
     public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(final String bucketName,
                                                                                              final Collection objectNames,
                                                                                              final Path destinationDirectory);
+
+    /**
+     * Creates a folder in the specified bucket
+     */
+    public abstract PutFolderResponse createFolder(final String bucketName, final String folderName) throws IOException;
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 3d08bf8e7..94e2b609d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -25,7 +25,10 @@
 import com.spectralogic.ds3client.commands.HeadBucketRequest;
 import com.spectralogic.ds3client.commands.HeadBucketResponse;
 import com.spectralogic.ds3client.commands.PutBucketRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
+import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
+import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
@@ -593,4 +596,28 @@ public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(
         return fileSystemHelper.objectsFromBucketWillFitInDirectory(this,
                 bucketName, objectNames, destinationDirectory);
     }
+
+    /**
+     * Creates a folder in the specified bucket.
+     */
+    @Override
+    public PutFolderResponse createFolder(final String bucketName, final String folderName) throws IOException {
+        final String normalizedFolderName = ensureNameEndsWithSlash(folderName);
+        final Ds3Object ds3Object = new Ds3Object(normalizedFolderName, 0);
+        final PutBulkJobSpectraS3Response jobResponse = client
+                .putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucketName, ImmutableList.of(ds3Object)));
+
+        return client.putFolder(new PutFolderRequest(bucketName, normalizedFolderName, jobResponse.getMasterObjectList().getJobId()));
+    }
+
+    /**
+     * Ensures that a folder names ends with a trailing forward slash. This prevents the
+     * accidental creation of zero-length files when attempting to create a folder.
+     */
+    private static String ensureNameEndsWithSlash(final String folderName) {
+        if (folderName.endsWith("/")) {
+            return folderName;
+        }
+        return folderName + "/";
+    }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 506e5b441..637799069 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -20,8 +20,10 @@
 import com.google.common.collect.Multimap;
 import com.google.common.collect.TreeMultimap;
 import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException;
+import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 import com.spectralogic.ds3client.models.*;
 import com.spectralogic.ds3client.models.Objects;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
@@ -1136,4 +1138,9 @@ public void getObjectVerifyFullPayload() throws IOException {
                         jobIdString,
                         0));
     }
+
+    @Test (expected = FolderNameMissingTrailingForwardSlash.class)
+    public void createFolderWithNoSlash() throws IOException {
+        new PutFolderRequest("BucketName", "FolderNameNoSlash", UUID.randomUUID());
+    }
 }

From d3fad5301bb328d217119f3811d9d990a66d2a27 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 10 Jul 2017 12:53:23 -0600
Subject: [PATCH 073/229] Removed unused import

---
 .../com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 94e2b609d..8b19d4d26 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -28,7 +28,6 @@
 import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
 import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
-import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;

From 945e18bbb1b34fd8fc0d70178c4fa53887d50760 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Mon, 17 Jul 2017 16:40:25 -0600
Subject: [PATCH 074/229] Update build.gradle

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index ead13b699..a2ee62b60 100644
--- a/build.gradle
+++ b/build.gradle
@@ -15,7 +15,7 @@
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '3.5.0'
+    version = '4.0.0'
 }
 
 subprojects {

From fe2b82fbcc27cec6f064315624c77f908634a6c8 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Mon, 17 Jul 2017 17:12:14 -0600
Subject: [PATCH 075/229] Update README.md

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 0bdba6178..b87ca2001 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ The SDK can also be included directly into a Maven or Gradle build. There is als
       
         com.spectralogic.ds3
         ds3-sdk
-        3.4.0
+        4.0.0
         
       
     ...
@@ -64,8 +64,8 @@ repositories {
 
 dependencies {
     ...
-    compile 'com.spectralogic.ds3:ds3-sdk:3.4.0'
-    // compile 'com.spectralogic.ds3:ds3-sdk:3.4.0:all'
+    compile 'com.spectralogic.ds3:ds3-sdk:4.0.0'
+    // compile 'com.spectralogic.ds3:ds3-sdk:4.0.0:all'
     ...
 }
 

From 26780477c802087ddbbc6ba9b81ba35ccfc66d98 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Mon, 31 Jul 2017 13:33:31 -0600
Subject: [PATCH 076/229] We now send metadata with every blob, where before we
 only sent metadata with a blob if the blob offset was 0. Cleaning up some
 compiler warnings.

---
 .../helpers/Ds3ClientHelpersImpl.java         |   8 +-
 .../PutJobTransferMethod.java                 |   2 +-
 .../ds3client/helpers/Metadata_Test.java      | 128 ++++++++++++++++++
 3 files changed, 131 insertions(+), 7 deletions(-)
 create mode 100644 ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 8b19d4d26..0d562c292 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -131,17 +131,13 @@ public Ds3ClientHelpers.Job startWriteJob(final String bucket,
     }
 
     private TransferStrategyBuilder makeTransferStrategyBuilder() {
-        final EventDispatcher eventDispatcher = new EventDispatcherImpl(eventRunner);
-
-        final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
+        return new TransferStrategyBuilder()
                 .withDs3Client(client)
                 .withNumChunkAttemptRetries(maxChunkAttempts)
                 .withNumTransferRetries(maxObjectTransferAttempts)
                 .withChunkRetryDelayInSeconds(secondsBetweenChunkAttempts)
                 .withEventRunner(eventRunner)
-                .withEventDispatcher(eventDispatcher);
-
-        return transferStrategyBuilder;
+                .withEventDispatcher(new EventDispatcherImpl(eventRunner));
     }
 
     private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
index f7eb5eb63..29520216c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
@@ -120,7 +120,7 @@ private void addCheckSum(final BulkObject blob, final PutObjectRequest putObject
     }
 
     private void addMetadata(final BulkObject blob, final PutObjectRequest putObjectRequest) {
-        if (blob.getOffset() == 0 && metadataAccess != null) {
+        if (metadataAccess != null) {
             final Map metadata = metadataAccess.getMetadataValue(blob.getName());
             if ( ! Guard.isMapNullOrEmpty(metadata)) {
                 final ImmutableMap immutableMetadata = ImmutableMap.copyOf(metadata);
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
new file mode 100644
index 000000000..11848b0d6
--- /dev/null
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
@@ -0,0 +1,128 @@
+package com.spectralogic.ds3client.helpers;
+
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.spectrads3.AllocateJobChunkSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.AllocateJobChunkSpectraS3Response;
+import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
+import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;
+import com.spectralogic.ds3client.models.BulkObject;
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.models.JobChunkClientProcessingOrderGuarantee;
+import com.spectralogic.ds3client.models.JobNode;
+import com.spectralogic.ds3client.models.JobRequestType;
+import com.spectralogic.ds3client.models.JobStatus;
+import com.spectralogic.ds3client.models.MasterObjectList;
+import com.spectralogic.ds3client.models.Objects;
+import com.spectralogic.ds3client.models.Priority;
+import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.nio.channels.SeekableByteChannel;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class Metadata_Test {
+    @Test
+    public void testThatWeSendMetadataWithEachBlob() throws IOException {
+        final JobNode jobNode = new JobNode();
+        jobNode.setEndPoint("endpoint");
+        jobNode.setHttpPort(80);
+        jobNode.setHttpsPort(443);
+        jobNode.setId(UUID.randomUUID());
+
+        final String bucketName = "bucket";
+        final String fileName = "aFile.txt";
+
+        final BulkObject blob1 = new BulkObject();
+        blob1.setBucket(bucketName);
+        blob1.setId(UUID.randomUUID());
+        blob1.setInCache(false);
+        blob1.setLatest(true);
+        blob1.setLength(2);
+        blob1.setOffset(0);
+        blob1.setVersion(1);
+        blob1.setName(fileName);
+
+        final BulkObject blob2 = new BulkObject();
+        blob2.setBucket(bucketName);
+        blob2.setId(UUID.randomUUID());
+        blob2.setInCache(false);
+        blob2.setLatest(true);
+        blob2.setLength(2);
+        blob2.setOffset(2);
+        blob2.setVersion(1);
+        blob2.setName(fileName);
+
+        final Objects chunk = new Objects();
+        chunk.setChunkId(UUID.randomUUID());
+        chunk.setChunkNumber(1);
+        chunk.setNodeId(UUID.randomUUID());
+        chunk.setObjects(Arrays.asList(blob1, blob2));
+
+        final MasterObjectList masterObjectList = new MasterObjectList();
+        masterObjectList.setAggregating(false);
+        masterObjectList.setBucketName(bucketName);
+        masterObjectList.setCachedSizeInBytes(4);
+        masterObjectList.setChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.IN_ORDER);
+        masterObjectList.setCompletedSizeInBytes(4);
+        masterObjectList.setJobId(UUID.randomUUID());
+        masterObjectList.setEntirelyInCache(false);
+        masterObjectList.setName("name");
+        masterObjectList.setNaked(false);
+        masterObjectList.setOriginalSizeInBytes(4);
+        masterObjectList.setPriority(Priority.NORMAL);
+        masterObjectList.setRequestType(JobRequestType.PUT);
+        masterObjectList.setStartDate(new Date());
+        masterObjectList.setStatus(JobStatus.IN_PROGRESS);
+        masterObjectList.setUserId(UUID.randomUUID());
+        masterObjectList.setUserName("Gracie");
+        masterObjectList.setNodes(Collections.singletonList(jobNode));
+        masterObjectList.setObjects(Collections.singletonList(chunk));
+
+        final Map metadata = new HashMap() {
+            { put("fileType", "text"); }
+        };
+
+        final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder = new Ds3ClientHelpers.ObjectChannelBuilder() {
+            @Override
+            public SeekableByteChannel buildChannel(final String key) throws IOException {
+                return new ByteArraySeekableByteChannel(4);
+            }
+        };
+
+        final AllocateJobChunkSpectraS3Response allocateJobChunkSpectraS3Response = new AllocateJobChunkSpectraS3Response(
+                chunk, 0, AllocateJobChunkSpectraS3Response.Status.ALLOCATED, "checksum", ChecksumType.Type.NONE
+        );
+
+        final Ds3Client ds3Client = Mockito.mock(Ds3Client.class);
+        Mockito.when(ds3Client.allocateJobChunkSpectraS3(Mockito.any(AllocateJobChunkSpectraS3Request.class)))
+                .thenReturn(allocateJobChunkSpectraS3Response);
+
+        final AtomicInteger numBlobsForWhichWeGotMetadata = new AtomicInteger(0);
+
+        final TransferStrategy transferStrategy = new TransferStrategyBuilder()
+                .withDs3Client(ds3Client)
+                .withMasterObjectList(masterObjectList)
+                .withChannelBuilder(objectChannelBuilder)
+                .withMetadataAccess(new MetadataAccess() {
+                    @Override
+                    public Map getMetadataValue(final String filename) {
+                        numBlobsForWhichWeGotMetadata.incrementAndGet();
+                        return metadata;
+                    }
+                })
+                .makePutTransferStrategy();
+
+        transferStrategy.transfer();
+
+        assertEquals(chunk.getObjects().size(), numBlobsForWhichWeGotMetadata.get());
+    }
+}

From d822869c8a839416f2d2ef2d499ea7d88c93bb19 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Mon, 31 Jul 2017 13:37:46 -0600
Subject: [PATCH 077/229] We now send metadata with every blob, where before we
 only sent metadata with a blob if the blob offset was 0. Cleaning up some
 compiler warnings.

---
 .../ds3client/helpers/Metadata_Test.java          | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
index 11848b0d6..217283da9 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
@@ -1,3 +1,18 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
 package com.spectralogic.ds3client.helpers;
 
 import com.spectralogic.ds3client.Ds3Client;

From 2f83f24ad8cca184d46d42bddb4dd425bd5e0aa8 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Mon, 31 Jul 2017 13:48:17 -0600
Subject: [PATCH 078/229] Ridiculous codacity whine.

---
 .../com/spectralogic/ds3client/helpers/Metadata_Test.java    | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
index 217283da9..948977f86 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
@@ -102,9 +102,8 @@ public void testThatWeSendMetadataWithEachBlob() throws IOException {
         masterObjectList.setNodes(Collections.singletonList(jobNode));
         masterObjectList.setObjects(Collections.singletonList(chunk));
 
-        final Map metadata = new HashMap() {
-            { put("fileType", "text"); }
-        };
+        final Map metadata = new HashMap<>();
+        metadata.put("fileType", "text");
 
         final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder = new Ds3ClientHelpers.ObjectChannelBuilder() {
             @Override

From e939415d61e21672953fc2ba23f2070d3533a2c9 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 2 Aug 2017 14:20:27 -0600
Subject: [PATCH 079/229] Prototype for handling requests with IDs payload

---
 .../interfaces/AbstractIdsPayloadRequest.java | 54 +++++++++++++++++++
 ...spectBlobAzureTargetsSpectraS3Request.java | 10 ++--
 .../ds3client/models/requestpayloads/Ids.java | 44 +++++++++++++++
 .../ds3client/Ds3Client_Test.java             | 18 +++++++
 4 files changed, 122 insertions(+), 4 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractIdsPayloadRequest.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/models/requestpayloads/Ids.java

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractIdsPayloadRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractIdsPayloadRequest.java
new file mode 100644
index 000000000..8a7f0c9f4
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractIdsPayloadRequest.java
@@ -0,0 +1,54 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.commands.interfaces;
+
+import com.spectralogic.ds3client.models.requestpayloads.Ids;
+import com.spectralogic.ds3client.serializer.XmlOutput;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+
+/**
+ * Abstract class that handles request payload of ids, in format:
+ * id1id2...
+ */
+public abstract class AbstractIdsPayloadRequest extends AbstractRequest {
+
+    private final List ids;
+    private long size;
+
+    public AbstractIdsPayloadRequest(final List ids) {
+        super();
+        this.ids = ids;
+    }
+
+    @Override
+    public InputStream getStream() {
+        final Ids requestPayload = new Ids(ids);
+        final String xmlOutput = XmlOutput.toXml(requestPayload);
+        final byte[] stringBytes = xmlOutput.getBytes(Charset.forName("UTF-8"));
+        this.size = stringBytes.length;
+
+        return new ByteArrayInputStream(stringBytes);
+    }
+
+    @Override
+    public long getSize() {
+        return this.size;
+    }
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobAzureTargetsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobAzureTargetsSpectraS3Request.java
index 58be2d5f7..6b04e24dd 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobAzureTargetsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobAzureTargetsSpectraS3Request.java
@@ -16,10 +16,12 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands.spectrads3;
 
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 
-public class ClearSuspectBlobAzureTargetsSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class ClearSuspectBlobAzureTargetsSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -28,8 +30,8 @@ public class ClearSuspectBlobAzureTargetsSpectraS3Request extends AbstractReques
     // Constructor
     
     
-    public ClearSuspectBlobAzureTargetsSpectraS3Request() {
-        
+    public ClearSuspectBlobAzureTargetsSpectraS3Request(final List ids) {
+        super(ids);
     }
 
     public ClearSuspectBlobAzureTargetsSpectraS3Request withForce(final boolean force) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/requestpayloads/Ids.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/requestpayloads/Ids.java
new file mode 100644
index 000000000..a740555bd
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/requestpayloads/Ids.java
@@ -0,0 +1,44 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.models.requestpayloads;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
+
+import java.util.List;
+
+/**
+ * Represents a request payload that is used by {@link com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest}
+ * and is used to marshal to xml a list of IDs in proper format
+ */
+public class Ids {
+
+    @JsonProperty("Id")
+    @JacksonXmlElementWrapper( useWrapping = false)
+    private List ids;
+
+    public Ids(final List ids) {
+        this.ids = ids;
+    }
+
+    public List getIds() {
+        return ids;
+    }
+
+    public void setIds(final List ids) {
+        this.ids = ids;
+    }
+}
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 637799069..337d362b8 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -1143,4 +1143,22 @@ public void getObjectVerifyFullPayload() throws IOException {
     public void createFolderWithNoSlash() throws IOException {
         new PutFolderRequest("BucketName", "FolderNameNoSlash", UUID.randomUUID());
     }
+
+    @Test
+    public void clearSuspectBlobAzureTargetsSpectraS3Test() throws IOException {
+        final String expectedRequestContent = "id1id2id3";
+
+        final List ids = new ArrayList<>();
+        ids.add("id1");
+        ids.add("id2");
+        ids.add("id3");
+
+        final Map queryParams = new HashMap<>();
+
+        MockNetwork
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_azure_target", queryParams, expectedRequestContent)
+                .returning(204, "")
+                .asClient()
+                .clearSuspectBlobAzureTargetsSpectraS3(new ClearSuspectBlobAzureTargetsSpectraS3Request(ids));
+    }
 }

From d8ef7e9e91df4a5bb38585bfbaadb17997ae1d01 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Thu, 3 Aug 2017 11:04:23 -0600
Subject: [PATCH 080/229] Regenerated 4.0.0 API with updated contract and fix
 for SA-193

---
 .../java/com/spectralogic/ds3client/Ds3Client.java |  1 +
 ...earSuspectBlobAzureTargetsSpectraS3Request.java |  4 ++--
 .../ClearSuspectBlobPoolsSpectraS3Request.java     | 10 ++++++----
 .../ClearSuspectBlobS3TargetsSpectraS3Request.java | 10 ++++++----
 .../ClearSuspectBlobTapesSpectraS3Request.java     | 10 ++++++----
 .../spectrads3/GetPoolsSpectraS3Request.java       | 14 ++++++++++++++
 ...BlobAzureTargetsAsDegradedSpectraS3Request.java | 10 ++++++----
 ...ctBlobDs3TargetsAsDegradedSpectraS3Request.java | 10 ++++++----
 ...SuspectBlobPoolsAsDegradedSpectraS3Request.java | 10 ++++++----
 ...ectBlobS3TargetsAsDegradedSpectraS3Request.java | 10 ++++++----
 ...SuspectBlobTapesAsDegradedSpectraS3Request.java | 10 ++++++----
 .../ds3client/models/TapeDriveType.java            |  3 ++-
 12 files changed, 67 insertions(+), 35 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index 8e356c187..fbee586c3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -39,6 +39,7 @@ public interface Ds3Client extends Closeable {
 
     PutFolderResponse putFolder(final PutFolderRequest request)
             throws IOException;
+
     
     
     AbortMultiPartUploadResponse abortMultiPartUpload(final AbortMultiPartUploadRequest request)
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobAzureTargetsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobAzureTargetsSpectraS3Request.java
index 6b04e24dd..eee52d8c9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobAzureTargetsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobAzureTargetsSpectraS3Request.java
@@ -16,8 +16,8 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands.spectrads3;
 
-import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 import com.spectralogic.ds3client.networking.HttpVerb;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
 import java.util.List;
 
@@ -29,9 +29,9 @@ public class ClearSuspectBlobAzureTargetsSpectraS3Request extends AbstractIdsPay
 
     // Constructor
     
-    
     public ClearSuspectBlobAzureTargetsSpectraS3Request(final List ids) {
         super(ids);
+        
     }
 
     public ClearSuspectBlobAzureTargetsSpectraS3Request withForce(final boolean force) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobPoolsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobPoolsSpectraS3Request.java
index 3bebff3c7..e8f3c6a79 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobPoolsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobPoolsSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class ClearSuspectBlobPoolsSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class ClearSuspectBlobPoolsSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class ClearSuspectBlobPoolsSpectraS3Request extends AbstractRequest {
 
     // Constructor
     
-    
-    public ClearSuspectBlobPoolsSpectraS3Request() {
+    public ClearSuspectBlobPoolsSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobS3TargetsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobS3TargetsSpectraS3Request.java
index 8be876acb..d00850f37 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobS3TargetsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobS3TargetsSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class ClearSuspectBlobS3TargetsSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class ClearSuspectBlobS3TargetsSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class ClearSuspectBlobS3TargetsSpectraS3Request extends AbstractRequest {
 
     // Constructor
     
-    
-    public ClearSuspectBlobS3TargetsSpectraS3Request() {
+    public ClearSuspectBlobS3TargetsSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobTapesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobTapesSpectraS3Request.java
index 85de3c5c1..e90d1ca61 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobTapesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobTapesSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class ClearSuspectBlobTapesSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class ClearSuspectBlobTapesSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class ClearSuspectBlobTapesSpectraS3Request extends AbstractRequest {
 
     // Constructor
     
-    
-    public ClearSuspectBlobTapesSpectraS3Request() {
+    public ClearSuspectBlobTapesSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPoolsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPoolsSpectraS3Request.java
index 730e861c3..b783870c7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPoolsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPoolsSpectraS3Request.java
@@ -33,6 +33,8 @@ public class GetPoolsSpectraS3Request extends AbstractPaginationRequest {
 
     private String bucketId;
 
+    private String guid;
+
     private PoolHealth health;
 
     private boolean lastPage;
@@ -78,6 +80,13 @@ public GetPoolsSpectraS3Request withBucketId(final String bucketId) {
     }
 
 
+    public GetPoolsSpectraS3Request withGuid(final String guid) {
+        this.guid = guid;
+        this.updateQueryParam("guid", guid);
+        return this;
+    }
+
+
     public GetPoolsSpectraS3Request withHealth(final PoolHealth health) {
         this.health = health;
         this.updateQueryParam("health", health);
@@ -208,6 +217,11 @@ public String getBucketId() {
     }
 
 
+    public String getGuid() {
+        return this.guid;
+    }
+
+
     public PoolHealth getHealth() {
         return this.health;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request.java
index 6bf269469..dcc159302 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request extends Abstr
 
     // Constructor
     
-    
-    public MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request() {
+    public MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request.java
index be8b21f0e..92c05c26f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request extends Abstrac
 
     // Constructor
     
-    
-    public MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request() {
+    public MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobPoolsAsDegradedSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobPoolsAsDegradedSpectraS3Request.java
index 24754359e..bc22f5769 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobPoolsAsDegradedSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobPoolsAsDegradedSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class MarkSuspectBlobPoolsAsDegradedSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class MarkSuspectBlobPoolsAsDegradedSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class MarkSuspectBlobPoolsAsDegradedSpectraS3Request extends AbstractRequ
 
     // Constructor
     
-    
-    public MarkSuspectBlobPoolsAsDegradedSpectraS3Request() {
+    public MarkSuspectBlobPoolsAsDegradedSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request.java
index 8410e7b2e..492c9133a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request extends Abstract
 
     // Constructor
     
-    
-    public MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request() {
+    public MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobTapesAsDegradedSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobTapesAsDegradedSpectraS3Request.java
index f6687cc3a..b9f8228b4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobTapesAsDegradedSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/MarkSuspectBlobTapesAsDegradedSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class MarkSuspectBlobTapesAsDegradedSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class MarkSuspectBlobTapesAsDegradedSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class MarkSuspectBlobTapesAsDegradedSpectraS3Request extends AbstractRequ
 
     // Constructor
     
-    
-    public MarkSuspectBlobTapesAsDegradedSpectraS3Request() {
+    public MarkSuspectBlobTapesAsDegradedSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDriveType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDriveType.java
index c084195a9..901f21fc0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDriveType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDriveType.java
@@ -23,5 +23,6 @@ public enum TapeDriveType {
     LTO7,
     LTO8,
     TS1140,
-    TS1150
+    TS1150,
+    TS1155
 }
\ No newline at end of file

From 726d19320115c3499178385bdcdf33008dd46693 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Fri, 4 Aug 2017 14:22:17 -0600
Subject: [PATCH 081/229] Fix for https://jira.spectralogic.com/browse/JSDK-245
 Null Pointer exception for get system info

Not exactly sure why we got the NPE, as extracting the files after pulling down the 'all' jar shows the .properties file where we expect it with the right contents, so...
---
 .../ds3client/utils/PropertyUtils.java            | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
index c12170f8d..33e0164d5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
@@ -47,13 +47,18 @@ public static String getSdkVersion() {
         try (final InputStream propertiesStream = PropertyUtils.class.getClassLoader().getResourceAsStream(PROPERTIES_FILE_NAME)) {
             if (propertiesStream != null) {
                 properties.load(propertiesStream);
-                final String versionFromPropFile = properties.get(SDK_VERSION_PROPERTY_NAME).toString();
-                if (!Guard.isStringNullOrEmpty(versionFromPropFile)) {
-                    versionProperty.set(versionFromPropFile);
+                final Object versionPropertyObject = properties.get(SDK_VERSION_PROPERTY_NAME);
+
+                if (versionPropertyObject != null) {
+                    final String versionFromPropFile = properties.get(SDK_VERSION_PROPERTY_NAME).toString();
+
+                    if ( ! Guard.isStringNullOrEmpty(versionFromPropFile)) {
+                        versionProperty.set(versionFromPropFile);
+                    }
                 }
             }
-        } catch (final IOException e) {
-            LOG.warn("Could not read properties file.", e);
+        } catch (final Throwable t) {
+            LOG.warn("Could not read properties file.", t);
         }
 
         return versionProperty.get();

From 98002efc8cf155ff2652324c0d4c910b8c04b2e5 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Fri, 4 Aug 2017 14:25:31 -0600
Subject: [PATCH 082/229] Removing unused import

---
 .../java/com/spectralogic/ds3client/utils/PropertyUtils.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
index 33e0164d5..243693ba5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
@@ -15,7 +15,6 @@
 
 package com.spectralogic.ds3client.utils;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.Properties;
 import java.util.concurrent.atomic.AtomicReference;

From 4d15253da4349d16d599446a0cc7833efa56837d Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Mon, 7 Aug 2017 10:54:15 -0600
Subject: [PATCH 083/229] Making sure methods remain the same

---
 .../pagination/GetBucketKeyLoaderFactory.java |  1 +
 .../pagination/GetBucketLoaderFactory.java    | 32 +++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
index 7fb40d110..4fde53fe7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketKeyLoaderFactory.java
@@ -64,6 +64,7 @@ public FileSystemKey apply(@Nullable final CommonPrefixes input) {
     private final int defaultListObjectsRetries;
     private final Function> function;
 
+
     public GetBucketKeyLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries, final Function> function) {
         this.client = client;
         this.bucket = bucket;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
new file mode 100644
index 000000000..65397bc48
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
@@ -0,0 +1,32 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers.pagination;
+
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.utils.collections.LazyIterable;
+
+public class GetBucketLoaderFactory extends GetBucketKeyLoaderFactory {
+
+    public GetBucketLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries) {
+        super(client, bucket, keyPrefix, "/" , nextMarker, maxKeys, defaultListObjectsRetries, contentsFunction);
+    }
+
+    @Override
+    public LazyIterable.LazyLoader create() {
+        return super.create();
+    }
+}
\ No newline at end of file

From e91adf044431eac298b9cb449f129d4a07bad2f9 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Mon, 7 Aug 2017 11:32:21 -0600
Subject: [PATCH 084/229] Adding some javadoc andd removing a stray newline

---
 .../pagination/GetBucketLoaderFactory.java       | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
index 65397bc48..3e2ba0520 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
@@ -19,12 +19,28 @@
 import com.spectralogic.ds3client.models.Contents;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 
+/**
+ *  This class is a child of {@link GetBucketKeyLoaderFactory}
+ *  that provides a sane default for mapping {@link com.spectralogic.ds3client.models.ListBucketResult}
+ */
 public class GetBucketLoaderFactory extends GetBucketKeyLoaderFactory {
 
+    /**
+     *
+     * @param client
+     * @param bucket
+     * @param keyPrefix
+     * @param nextMarker
+     * @param maxKeys
+     * @param defaultListObjectsRetries
+     */
     public GetBucketLoaderFactory(final Ds3Client client, final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys, final int defaultListObjectsRetries) {
         super(client, bucket, keyPrefix, "/" , nextMarker, maxKeys, defaultListObjectsRetries, contentsFunction);
     }
 
+    /**
+     * @return {@link LazyIterable.LazyLoader}
+     */
     @Override
     public LazyIterable.LazyLoader create() {
         return super.create();

From 5e60bee14c57248eb3ad1e8028168b03e819c184 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Mon, 7 Aug 2017 12:38:14 -0600
Subject: [PATCH 085/229] Adding some javadoc andd removing a stray newline

---
 .../ds3client/helpers/pagination/GetBucketLoaderFactory.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
index 3e2ba0520..48a1d6d4c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/pagination/GetBucketLoaderFactory.java
@@ -22,6 +22,8 @@
 /**
  *  This class is a child of {@link GetBucketKeyLoaderFactory}
  *  that provides a sane default for mapping {@link com.spectralogic.ds3client.models.ListBucketResult}
+ *  "/" will be used for the delimiter, and {@link com.spectralogic.ds3client.models.ListBucketResult} will be mapped to
+ *  {@link LazyIterable.LazyLoader} which was the old behavior.
  */
 public class GetBucketLoaderFactory extends GetBucketKeyLoaderFactory {
 

From 630993b78b7e9adccaa4224b7aa2ec67cdb5763f Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 11 Aug 2017 14:51:18 -0600
Subject: [PATCH 086/229] Unit tests validating request payloads

---
 .../ds3client/models/S3ObjectList.java        |   2 +-
 .../multipart/CompleteMultipartUpload.java    |   4 +
 .../ds3client/models/multipart/Part.java      |   5 +
 .../ds3client/Ds3Client_Test.java             | 365 +++++++++++++++++-
 .../ds3client/models/ModelParsing_Test.java   |  14 +-
 5 files changed, 366 insertions(+), 24 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java
index 7ad03981e..8905c00bd 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java
@@ -45,4 +45,4 @@ public void setS3Objects(final List s3Objects) {
         this.s3Objects = s3Objects;
     }
 
-}
\ No newline at end of file
+}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/CompleteMultipartUpload.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/CompleteMultipartUpload.java
index e7366eb67..c9c4fa96b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/CompleteMultipartUpload.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/CompleteMultipartUpload.java
@@ -31,6 +31,10 @@ public CompleteMultipartUpload() {
         //Pass
     }
 
+    public CompleteMultipartUpload(final List parts) {
+        this.parts = parts;
+    }
+
     public List getParts() {
         return parts;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/Part.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/Part.java
index 6bfe38252..6bf68fc0c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/Part.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/Part.java
@@ -29,6 +29,11 @@ public Part() {
         //Pass
     }
 
+    public Part(final int partNumber, final String eTag) {
+        this.partNumber = partNumber;
+        this.eTag = eTag;
+    }
+
     public int getPartNumber() {
         return partNumber;
     }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 337d362b8..1862f4dfb 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -15,10 +15,7 @@
 
 package com.spectralogic.ds3client;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.TreeMultimap;
+import com.google.common.collect.*;
 import com.spectralogic.ds3client.commands.*;
 import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
 import com.spectralogic.ds3client.commands.spectrads3.*;
@@ -27,7 +24,11 @@
 import com.spectralogic.ds3client.models.*;
 import com.spectralogic.ds3client.models.Objects;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.models.bulk.PartialDs3Object;
 import com.spectralogic.ds3client.models.common.Credentials;
+import com.spectralogic.ds3client.models.common.Range;
+import com.spectralogic.ds3client.models.multipart.CompleteMultipartUpload;
+import com.spectralogic.ds3client.models.multipart.Part;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.networking.FailedRequestUsingMgmtPortException;
 import com.spectralogic.ds3client.networking.HttpVerb;
@@ -35,7 +36,6 @@
 import com.spectralogic.ds3client.utils.ResourceUtils;
 import org.junit.Before;
 import org.junit.Test;
-import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
@@ -73,6 +73,16 @@ public class Ds3Client_Test {
 
     private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
 
+    private final static String IDS_REQUEST_PAYLOAD = "id1id2id3";
+    private final static ImmutableList IDS_REQUEST_PAYLOAD_LIST = ImmutableList.of("id1", "id2", "id3");
+
+    private static final String SIMPLE_OBJECT_REQUEST_PAYLOAD = "";
+    private static final List SIMPLE_OBJECT_LIST = Arrays.asList(
+            new Ds3Object("file1"),
+            new Ds3Object("file2"),
+            new Ds3Object("file3")
+    );
+
     @Before
     public void setTimeZone() {
         DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -531,7 +541,7 @@ MasterObjectList performRestCall(final Ds3Client client, final String bucket, fi
                 throws IOException;
     }
     
-    public void runBulkTest(final BulkCommand command, final BulkTestDriver driver) throws IOException {
+    private void runBulkTest(final BulkCommand command, final BulkTestDriver driver) throws IOException {
         final List objects = Arrays.asList(
             new Ds3Object("file1", 256),
             new Ds3Object("file2", 1202),
@@ -1146,19 +1156,348 @@ public void createFolderWithNoSlash() throws IOException {
 
     @Test
     public void clearSuspectBlobAzureTargetsSpectraS3Test() throws IOException {
-        final String expectedRequestContent = "id1id2id3";
+        MockNetwork
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_azure_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .clearSuspectBlobAzureTargetsSpectraS3(new ClearSuspectBlobAzureTargetsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void clearSuspectBlobPoolsSpectraS3Test() throws IOException {
+        MockNetwork
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_pool", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .clearSuspectBlobPoolsSpectraS3(new ClearSuspectBlobPoolsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void clearSuspectBlobS3TargetsSpectraS3Test() throws IOException {
+        MockNetwork
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_s3_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .clearSuspectBlobS3TargetsSpectraS3(new ClearSuspectBlobS3TargetsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void clearSuspectBlobTapesSpectraS3Test() throws IOException {
+        MockNetwork
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_tape", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .clearSuspectBlobTapesSpectraS3(new ClearSuspectBlobTapesSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void markSuspectBlobAzureTargetsAsDegradedSpectraS3Test() throws IOException {
+        MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_azure_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .markSuspectBlobAzureTargetsAsDegradedSpectraS3(new MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void markSuspectBlobDs3TargetsAsDegradedSpectraS3Test() throws IOException {
+        MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_ds3_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .markSuspectBlobDs3TargetsAsDegradedSpectraS3(new MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void markSuspectBlobPoolsAsDegradedSpectraS3Test() throws IOException {
+        MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_pool", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .markSuspectBlobPoolsAsDegradedSpectraS3(new MarkSuspectBlobPoolsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void markSuspectBlobS3TargetsAsDegradedSpectraS3Test() throws IOException {
+        MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_s3_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .markSuspectBlobS3TargetsAsDegradedSpectraS3(new MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void markSuspectBlobTapesAsDegradedSpectraS3Test() throws IOException {
+        MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_tape", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .returning(204, "")
+                .asClient()
+                .markSuspectBlobTapesAsDegradedSpectraS3(new MarkSuspectBlobTapesAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
+    }
+
+    @Test
+    public void completeMultiPartUploadTest() throws IOException {
+        final String expectedRequestContent = "17a112844c1a2327e617f530cb06dccf827162e29f4e40da7f521d0794b57770ba";
+        final String expectedResponse = "http://my-server/bucketName/objectbucketNameobjectb54357faf0632cce46e942fa68356b38";
+
+        final List parts = new ArrayList<>();
+        parts.add(new Part(1, "7a112844c1a2327e617f530cb06dccf8"));
+        parts.add(new Part(2, "7162e29f4e40da7f521d0794b57770ba"));
+
+        final String bucketName = "bucketName";
+        final String objectName = "object";
+        final String uploadId = "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA";
+        final String etag = "b54357faf0632cce46e942fa68356b38";
+        final String location = "http://my-server/bucketName/object";
+
+        final CompleteMultipartUpload completeMultipartUpload = new CompleteMultipartUpload(parts);
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("upload_id", uploadId);
+
+        final Map responseHeaders = new HashMap<>();
+        responseHeaders.put("etag", etag);
+
+        final CompleteMultiPartUploadResponse response = MockNetwork
+                .expecting(HttpVerb.POST, "/bucketName/object", queryParams, expectedRequestContent)
+                .returning(200, expectedResponse, responseHeaders)
+                .asClient()
+                .completeMultiPartUpload(new CompleteMultiPartUploadRequest(
+                        bucketName,
+                        objectName,
+                        completeMultipartUpload,
+                        uploadId));
+
+        assertThat(response.getCompleteMultipartUploadResult().getLocation(), is(location));
+        assertThat(response.getCompleteMultipartUploadResult().getBucket(), is(bucketName));
+        assertThat(response.getCompleteMultipartUploadResult().getKey(), is(objectName));
+        assertThat(response.getCompleteMultipartUploadResult().getETag(), is(etag));
+    }
+
+    @Test
+    public void putMultiPartUploadPartTest() throws IOException {
+        final String requestContent = "this is the part content";
+        final String bucketName = "bucketName";
+        final String objectName = "object";
+        final int partNumber = 2;
+        final String uploadId = "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA";
+        final String eTag = "b54357faf0632cce46e942fa68356b38";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("part_number", String.valueOf(partNumber));
+        queryParams.put("upload_id", uploadId);
+
+        final Map responseHeaders = new HashMap<>();
+        responseHeaders.put("etag", eTag);
+
+        MockNetwork
+                .expecting(HttpVerb.PUT, "/bucketName/object", queryParams, requestContent)
+                .returning(200, "", responseHeaders)
+                .asClient()
+                .putMultiPartUploadPart(new PutMultiPartUploadPartRequest(
+                        bucketName,
+                        objectName,
+                        new ByteArraySeekableByteChannel(requestContent.getBytes()),
+                        partNumber, requestContent.getBytes().length,
+                        uploadId));
+    }
+
+    @Test
+    public void getBulkJobSpectraS3WithMixedPayloadTest() throws IOException {
+        final List objects = Arrays.asList(
+                new Ds3Object("file1", 256),
+                new PartialDs3Object("file2", new Range(1202, 1402)),
+                new Ds3Object("file3", 2523)
+        );
+
+        final String bucketName = "bulkTest";
+
+        final String expectedRequestPayload = "";
+
+        final String xmlResponse = "";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "start_bulk_get");
+
+        final GetBulkJobSpectraS3Response response = MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, expectedRequestPayload)
+                .returning(200, xmlResponse)
+                .asClient()
+                .getBulkJobSpectraS3(new GetBulkJobSpectraS3Request(bucketName, objects));
 
-        final List ids = new ArrayList<>();
-        ids.add("id1");
-        ids.add("id2");
-        ids.add("id3");
+        final List objectListList = response.getMasterObjectList().getObjects();
+        assertThat(objectListList.size(), is(1));
+
+        final List objectList = objectListList.get(0).getObjects();
+        assertThat(objectList.size(), is(3));
+
+        assertObjectEquals(objectList.get(0), "file2", 1202);
+        assertObjectEquals(objectList.get(1), "file1", 256);
+        assertObjectEquals(objectList.get(2), "file3", 2523);
+    }
+
+
+    @Test
+    public void verifyBulkJobSpectraS3Test() throws IOException {
+        final List objects = Arrays.asList(
+                new Ds3Object("file1"),
+                new PartialDs3Object("file2", new Range(1202, 1402)),
+                new Ds3Object("file3")
+        );
+
+        final String bucketName = "bulkTest";
+
+        final String expectedRequestPayload = "";
+
+        final String xmlResponse = "";
 
         final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "start_bulk_verify");
+
+        final VerifyBulkJobSpectraS3Response response = MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, expectedRequestPayload)
+                .returning(200, xmlResponse)
+                .asClient()
+                .verifyBulkJobSpectraS3(new VerifyBulkJobSpectraS3Request(bucketName, objects));
+
+        final List objectListList = response.getMasterObjectListResult().getObjects();
+        assertThat(objectListList.size(), is(1));
+
+        final List objectList = objectListList.get(0).getObjects();
+        assertThat(objectList.size(), is(3));
+
+        assertObjectEquals(objectList.get(0), "file2", 1202);
+        assertObjectEquals(objectList.get(1), "file1", 256);
+        assertObjectEquals(objectList.get(2), "file3", 2523);
+    }
+
+    @Test
+    public void getPhysicalPlacementForObjectsTest() throws IOException {
+        final String responsePayload = "";
+        final String bucketName = "BucketName";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "get_physical_placement");
+
+        final GetPhysicalPlacementForObjectsSpectraS3Response response = MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD)
+                .returning(200, responsePayload)
+                .asClient()
+                .getPhysicalPlacementForObjectsSpectraS3(new GetPhysicalPlacementForObjectsSpectraS3Request(bucketName, SIMPLE_OBJECT_LIST));
+
+        assertThat(response.getPhysicalPlacementResult().getAzureTargets(), is(nullValue()));
+        assertThat(response.getPhysicalPlacementResult().getDs3Targets(), is(nullValue()));
+        assertThat(response.getPhysicalPlacementResult().getPools(), is(nullValue()));
+        assertThat(response.getPhysicalPlacementResult().getS3Targets(), is(nullValue()));
+        assertThat(response.getPhysicalPlacementResult().getTapes(), is(nullValue()));
+    }
+
+    @Test
+    public void getPhysicalPlacementForObjectsWithFullDetailsTest() throws IOException {
+        final String responsePayload = "";
+        final String bucketName = "BucketName";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "get_physical_placement");
+        queryParams.put("full_details", null);
+
+        final GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Response response = MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD)
+                .returning(200, responsePayload)
+                .asClient()
+                .getPhysicalPlacementForObjectsWithFullDetailsSpectraS3(new GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request(bucketName, SIMPLE_OBJECT_LIST));
+
+        assertThat(response.getBulkObjectListResult().getObjects().size(), is(1));
+    }
+
+    @Test
+    public void verifyPhysicalPlacementForObjectsTest() throws IOException {
+        final String responsePayload = "false10000t1false48d30ecb-84f1-4721-9832-7aa165a1dd7776343269-c32a-4cb0-aec4-57a9dccce6eaPENDING_INSPECTIONfalse20000LTO5false";
+        final String bucketName = "BucketName";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "verify_physical_placement");
+
+        final VerifyPhysicalPlacementForObjectsSpectraS3Response response = MockNetwork
+                .expecting(HttpVerb.GET, "/_rest_/bucket/" + bucketName, queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD)
+                .returning(200, responsePayload)
+                .asClient()
+                .verifyPhysicalPlacementForObjectsSpectraS3(new VerifyPhysicalPlacementForObjectsSpectraS3Request(bucketName, SIMPLE_OBJECT_LIST));
+
+        assertThat(response.getPhysicalPlacementResult().getAzureTargets(), is(nullValue()));
+        assertThat(response.getPhysicalPlacementResult().getDs3Targets(), is(nullValue()));
+        assertThat(response.getPhysicalPlacementResult().getPools(), is(nullValue()));
+        assertThat(response.getPhysicalPlacementResult().getTapes().size(), is(1));
+    }
+
+    @Test
+    public void verifyPhysicalPlacementForObjectsWithFullDetailsTest() throws IOException {
+        final String responsePayload = "false10000t1false5a7bb215-4aff-4806-b217-5fe01ade6a2c2e5b25fc-546e-45b0-951e-8f3d80bb7823PENDING_INSPECTIONfalse20000LTO5false";
+        final String bucketName = "BucketName";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "verify_physical_placement");
+        queryParams.put("full_details", null);
+
+        final VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Response response = MockNetwork
+                .expecting(HttpVerb.GET, "/_rest_/bucket/" + bucketName, queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD)
+                .returning(200, responsePayload)
+                .asClient()
+                .verifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3(new VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request(bucketName, SIMPLE_OBJECT_LIST));
+
+        assertThat(response.getBulkObjectListResult().getObjects().size(), is(1));
+    }
+
+    @Test
+    public void ejectStorageDomainBlobsTest() throws IOException {
+        final String bucketId = "BucketId";
+        final String storageDomainId = "StorageDomainId";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "eject");
+        queryParams.put("blobs", null);
+        queryParams.put("bucket_id", bucketId);
+        queryParams.put("storage_domain_id", storageDomainId);
 
         MockNetwork
-                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_azure_target", queryParams, expectedRequestContent)
+                .expecting(HttpVerb.PUT, "/_rest_/tape", queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
-                .clearSuspectBlobAzureTargetsSpectraS3(new ClearSuspectBlobAzureTargetsSpectraS3Request(ids));
+                .ejectStorageDomainBlobsSpectraS3(new EjectStorageDomainBlobsSpectraS3Request(bucketId, SIMPLE_OBJECT_LIST, storageDomainId));
+    }
+
+    @Test
+    public void replicatePutJobTest() throws IOException {
+        final String responsePayload = "";
+        final String requestPayload = "This is the request payload content";
+        final String bucketName = "BucketName";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "start_bulk_put");
+        queryParams.put("replicate", null);
+
+        final ReplicatePutJobSpectraS3Response response = MockNetwork
+                .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, requestPayload)
+                .returning(200, responsePayload)
+                .asClient()
+                .replicatePutJobSpectraS3(new ReplicatePutJobSpectraS3Request(bucketName, requestPayload));
+
+        assertThat(response.getMasterObjectListResult().getObjects().size(), is(1));
+    }
+
+    @Test
+    public void getBlobPersistenceTest() throws IOException {
+        final String responsePayload = "This is the response payload content";
+        final String requestPayload = "This is the request payload content";
+
+        final Map queryParams = new HashMap<>();
+
+        final GetBlobPersistenceSpectraS3Response response = MockNetwork
+                .expecting(HttpVerb.GET, "/_rest_/blob_persistence", queryParams, requestPayload)
+                .returning(200, responsePayload)
+                .asClient()
+                .getBlobPersistenceSpectraS3(new GetBlobPersistenceSpectraS3Request(requestPayload));
+
+        assertThat(response.getStringResult(), is(responsePayload));
     }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java
index 6448b9078..371395bba 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java
@@ -92,17 +92,11 @@ public void completeMultipartUpload_ToString_Test() {
                 "20c78aef83f66abc1fa1e8477f296d394" +
                 "";
 
-        final Part part1 = new Part();
-        part1.setPartNumber(1);
-        part1.seteTag("a54357aff0632cce46d942af68356b38");
+        final ImmutableList parts = ImmutableList.of(
+                new Part(1, "a54357aff0632cce46d942af68356b38"),
+                new Part(2, "0c78aef83f66abc1fa1e8477f296d394"));
 
-        final Part part2 = new Part();
-        part2.setPartNumber(2);
-        part2.seteTag("0c78aef83f66abc1fa1e8477f296d394");
-
-        final ImmutableList parts = ImmutableList.of(part1, part2);
-        final CompleteMultipartUpload input = new CompleteMultipartUpload();
-        input.setParts(parts.asList());
+        final CompleteMultipartUpload input = new CompleteMultipartUpload(parts.asList());
 
         final String result = XmlOutput.toXml(input);
 

From 60e6a5af28912e102af84f761fd6b63df8da7a9c Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 16 Aug 2017 15:52:30 -0600
Subject: [PATCH 087/229] Fixed get blobs on tape commands to not have request
 payload

---
 ...GetBlobsOnAzureTargetSpectraS3Request.java | 41 +----------
 .../GetBlobsOnDs3TargetSpectraS3Request.java  | 41 +----------
 .../GetBlobsOnPoolSpectraS3Request.java       | 41 +----------
 .../GetBlobsOnS3TargetSpectraS3Request.java   | 41 +----------
 .../GetBlobsOnTapeSpectraS3Request.java       | 44 +-----------
 .../ds3client/models/S3ObjectList.java        |  2 +-
 .../ds3client/Ds3Client_Test.java             | 72 +++++++++++++++++++
 7 files changed, 79 insertions(+), 203 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnAzureTargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnAzureTargetSpectraS3Request.java
index 88de54734..609488811 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnAzureTargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnAzureTargetSpectraS3Request.java
@@ -17,15 +17,6 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.models.bulk.Ds3Object;
-import com.spectralogic.ds3client.models.bulk.Ds3ObjectList;
-import com.spectralogic.ds3client.serializer.XmlOutput;
-import com.spectralogic.ds3client.utils.Guard;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.List;
-import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
 
@@ -35,41 +26,17 @@ public class GetBlobsOnAzureTargetSpectraS3Request extends AbstractRequest {
     
     private final String azureTarget;
 
-    private final List objects;
-    private long size = 0;
-
     // Constructor
     
     
-    public GetBlobsOnAzureTargetSpectraS3Request(final String azureTarget, final List objects) {
+    public GetBlobsOnAzureTargetSpectraS3Request(final String azureTarget) {
         this.azureTarget = azureTarget;
-        this.objects = objects;
         
         this.getQueryParams().put("operation", "get_physical_placement");
 
     }
 
 
-    @Override
-    public InputStream getStream() {
-        if (Guard.isNullOrEmpty(objects)) {
-            return null;
-        }
-        final Ds3ObjectList objects = new Ds3ObjectList();
-        objects.setObjects(this.objects);
-
-        final String xmlOutput = XmlOutput.toXml(objects, false);
-
-        final byte[] stringBytes = xmlOutput.getBytes(Charset.forName("UTF-8"));
-        this.size = stringBytes.length;
-        return new ByteArrayInputStream(stringBytes);
-    }
-
-    @Override
-    public long getSize() {
-        return this.size;
-    }
-
     @Override
     public HttpVerb getVerb() {
         return HttpVerb.GET;
@@ -84,10 +51,4 @@ public String getAzureTarget() {
         return this.azureTarget;
     }
 
-
-    public List getObjects() {
-        return this.objects;
-    }
-
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnDs3TargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnDs3TargetSpectraS3Request.java
index 23c37d74f..e47168715 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnDs3TargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnDs3TargetSpectraS3Request.java
@@ -17,15 +17,6 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.models.bulk.Ds3Object;
-import com.spectralogic.ds3client.models.bulk.Ds3ObjectList;
-import com.spectralogic.ds3client.serializer.XmlOutput;
-import com.spectralogic.ds3client.utils.Guard;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.List;
-import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
 
@@ -35,41 +26,17 @@ public class GetBlobsOnDs3TargetSpectraS3Request extends AbstractRequest {
     
     private final String ds3Target;
 
-    private final List objects;
-    private long size = 0;
-
     // Constructor
     
     
-    public GetBlobsOnDs3TargetSpectraS3Request(final String ds3Target, final List objects) {
+    public GetBlobsOnDs3TargetSpectraS3Request(final String ds3Target) {
         this.ds3Target = ds3Target;
-        this.objects = objects;
         
         this.getQueryParams().put("operation", "get_physical_placement");
 
     }
 
 
-    @Override
-    public InputStream getStream() {
-        if (Guard.isNullOrEmpty(objects)) {
-            return null;
-        }
-        final Ds3ObjectList objects = new Ds3ObjectList();
-        objects.setObjects(this.objects);
-
-        final String xmlOutput = XmlOutput.toXml(objects, false);
-
-        final byte[] stringBytes = xmlOutput.getBytes(Charset.forName("UTF-8"));
-        this.size = stringBytes.length;
-        return new ByteArrayInputStream(stringBytes);
-    }
-
-    @Override
-    public long getSize() {
-        return this.size;
-    }
-
     @Override
     public HttpVerb getVerb() {
         return HttpVerb.GET;
@@ -84,10 +51,4 @@ public String getDs3Target() {
         return this.ds3Target;
     }
 
-
-    public List getObjects() {
-        return this.objects;
-    }
-
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnPoolSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnPoolSpectraS3Request.java
index 8d21607b8..cafda5b05 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnPoolSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnPoolSpectraS3Request.java
@@ -17,15 +17,6 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.models.bulk.Ds3Object;
-import com.spectralogic.ds3client.models.bulk.Ds3ObjectList;
-import com.spectralogic.ds3client.serializer.XmlOutput;
-import com.spectralogic.ds3client.utils.Guard;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.List;
-import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
 
@@ -35,41 +26,17 @@ public class GetBlobsOnPoolSpectraS3Request extends AbstractRequest {
     
     private final String pool;
 
-    private final List objects;
-    private long size = 0;
-
     // Constructor
     
     
-    public GetBlobsOnPoolSpectraS3Request(final List objects, final String pool) {
+    public GetBlobsOnPoolSpectraS3Request(final String pool) {
         this.pool = pool;
-        this.objects = objects;
         
         this.getQueryParams().put("operation", "get_physical_placement");
 
     }
 
 
-    @Override
-    public InputStream getStream() {
-        if (Guard.isNullOrEmpty(objects)) {
-            return null;
-        }
-        final Ds3ObjectList objects = new Ds3ObjectList();
-        objects.setObjects(this.objects);
-
-        final String xmlOutput = XmlOutput.toXml(objects, false);
-
-        final byte[] stringBytes = xmlOutput.getBytes(Charset.forName("UTF-8"));
-        this.size = stringBytes.length;
-        return new ByteArrayInputStream(stringBytes);
-    }
-
-    @Override
-    public long getSize() {
-        return this.size;
-    }
-
     @Override
     public HttpVerb getVerb() {
         return HttpVerb.GET;
@@ -84,10 +51,4 @@ public String getPool() {
         return this.pool;
     }
 
-
-    public List getObjects() {
-        return this.objects;
-    }
-
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnS3TargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnS3TargetSpectraS3Request.java
index 0ba390808..831f0a39e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnS3TargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnS3TargetSpectraS3Request.java
@@ -17,15 +17,6 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.models.bulk.Ds3Object;
-import com.spectralogic.ds3client.models.bulk.Ds3ObjectList;
-import com.spectralogic.ds3client.serializer.XmlOutput;
-import com.spectralogic.ds3client.utils.Guard;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.List;
-import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
 
@@ -35,41 +26,17 @@ public class GetBlobsOnS3TargetSpectraS3Request extends AbstractRequest {
     
     private final String s3Target;
 
-    private final List objects;
-    private long size = 0;
-
     // Constructor
     
     
-    public GetBlobsOnS3TargetSpectraS3Request(final List objects, final String s3Target) {
+    public GetBlobsOnS3TargetSpectraS3Request(final String s3Target) {
         this.s3Target = s3Target;
-        this.objects = objects;
         
         this.getQueryParams().put("operation", "get_physical_placement");
 
     }
 
 
-    @Override
-    public InputStream getStream() {
-        if (Guard.isNullOrEmpty(objects)) {
-            return null;
-        }
-        final Ds3ObjectList objects = new Ds3ObjectList();
-        objects.setObjects(this.objects);
-
-        final String xmlOutput = XmlOutput.toXml(objects, false);
-
-        final byte[] stringBytes = xmlOutput.getBytes(Charset.forName("UTF-8"));
-        this.size = stringBytes.length;
-        return new ByteArrayInputStream(stringBytes);
-    }
-
-    @Override
-    public long getSize() {
-        return this.size;
-    }
-
     @Override
     public HttpVerb getVerb() {
         return HttpVerb.GET;
@@ -84,10 +51,4 @@ public String getS3Target() {
         return this.s3Target;
     }
 
-
-    public List getObjects() {
-        return this.objects;
-    }
-
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Request.java
index 09d3aa238..453689c31 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Request.java
@@ -17,15 +17,6 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.models.bulk.Ds3Object;
-import com.spectralogic.ds3client.models.bulk.Ds3ObjectList;
-import com.spectralogic.ds3client.serializer.XmlOutput;
-import com.spectralogic.ds3client.utils.Guard;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.List;
-import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
@@ -36,50 +27,25 @@ public class GetBlobsOnTapeSpectraS3Request extends AbstractRequest {
     
     private final String tapeId;
 
-    private final List objects;
-    private long size = 0;
-
     // Constructor
     
     
-    public GetBlobsOnTapeSpectraS3Request(final List objects, final UUID tapeId) {
+    public GetBlobsOnTapeSpectraS3Request(final UUID tapeId) {
         this.tapeId = tapeId.toString();
-        this.objects = objects;
         
         this.getQueryParams().put("operation", "get_physical_placement");
 
     }
 
     
-    public GetBlobsOnTapeSpectraS3Request(final List objects, final String tapeId) {
+    public GetBlobsOnTapeSpectraS3Request(final String tapeId) {
         this.tapeId = tapeId;
-        this.objects = objects;
         
         this.getQueryParams().put("operation", "get_physical_placement");
 
     }
 
 
-    @Override
-    public InputStream getStream() {
-        if (Guard.isNullOrEmpty(objects)) {
-            return null;
-        }
-        final Ds3ObjectList objects = new Ds3ObjectList();
-        objects.setObjects(this.objects);
-
-        final String xmlOutput = XmlOutput.toXml(objects, false);
-
-        final byte[] stringBytes = xmlOutput.getBytes(Charset.forName("UTF-8"));
-        this.size = stringBytes.length;
-        return new ByteArrayInputStream(stringBytes);
-    }
-
-    @Override
-    public long getSize() {
-        return this.size;
-    }
-
     @Override
     public HttpVerb getVerb() {
         return HttpVerb.GET;
@@ -94,10 +60,4 @@ public String getTapeId() {
         return this.tapeId;
     }
 
-
-    public List getObjects() {
-        return this.objects;
-    }
-
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java
index 8905c00bd..7ad03981e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java
@@ -45,4 +45,4 @@ public void setS3Objects(final List s3Objects) {
         this.s3Objects = s3Objects;
     }
 
-}
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 1862f4dfb..ab7138f29 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -83,6 +83,8 @@ public class Ds3Client_Test {
             new Ds3Object("file3")
     );
 
+    private static final String SIMPLE_BULK_OBJECT_LIST_RESPONSE = "";
+
     @Before
     public void setTimeZone() {
         DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -1500,4 +1502,74 @@ public void getBlobPersistenceTest() throws IOException {
 
         assertThat(response.getStringResult(), is(responsePayload));
     }
+
+    @Test
+    public void getBlobsOnAzureTargetTest() throws IOException {
+        final String target = "Target";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "get_physical_placement");
+
+        MockNetwork
+                .expecting(HttpVerb.GET, "/_rest_/azure_target/" + target, queryParams, null)
+                .returning(200, SIMPLE_BULK_OBJECT_LIST_RESPONSE)
+                .asClient()
+                .getBlobsOnAzureTargetSpectraS3(new GetBlobsOnAzureTargetSpectraS3Request(target));
+    }
+
+    @Test
+    public void getBlobsOnTapeTest() throws IOException {
+        final String target = "Target";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "get_physical_placement");
+
+        MockNetwork
+                .expecting(HttpVerb.GET, "/_rest_/tape/" + target, queryParams, null)
+                .returning(200, SIMPLE_BULK_OBJECT_LIST_RESPONSE)
+                .asClient()
+                .getBlobsOnTapeSpectraS3(new GetBlobsOnTapeSpectraS3Request(target));
+    }
+
+    @Test
+    public void getBlobsOnS3TargetTest() throws IOException {
+        final String target = "Target";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "get_physical_placement");
+
+        MockNetwork
+                .expecting(HttpVerb.GET, "/_rest_/s3_target/" + target, queryParams, null)
+                .returning(200, SIMPLE_BULK_OBJECT_LIST_RESPONSE)
+                .asClient()
+                .getBlobsOnS3TargetSpectraS3(new GetBlobsOnS3TargetSpectraS3Request(target));
+    }
+
+    @Test
+    public void getBlobsOnPoolTest() throws IOException {
+        final String target = "Target";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "get_physical_placement");
+
+        MockNetwork
+                .expecting(HttpVerb.GET, "/_rest_/pool/" + target, queryParams, null)
+                .returning(200, SIMPLE_BULK_OBJECT_LIST_RESPONSE)
+                .asClient()
+                .getBlobsOnPoolSpectraS3(new GetBlobsOnPoolSpectraS3Request(target));
+    }
+
+    @Test
+    public void getBlobsOnDs3TargetTest() throws IOException {
+        final String target = "Target";
+
+        final Map queryParams = new HashMap<>();
+        queryParams.put("operation", "get_physical_placement");
+
+        MockNetwork
+                .expecting(HttpVerb.GET, "/_rest_/ds3_target/" + target, queryParams, null)
+                .returning(200, SIMPLE_BULK_OBJECT_LIST_RESPONSE)
+                .asClient()
+                .getBlobsOnDs3TargetSpectraS3(new GetBlobsOnDs3TargetSpectraS3Request(target));
+    }
 }

From 895f48c4947eee44769e1842df01ed739f83fe25 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 16 Aug 2017 16:47:57 -0600
Subject: [PATCH 088/229] Added integration test for GetBlobsOnTape to verify
 no request payload

---
 .../ds3client/integration/Smoke_Test.java     | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
index 616f31ada..2a36c04de 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
@@ -54,12 +54,7 @@
 import java.nio.channels.SeekableByteChannel;
 import java.nio.charset.Charset;
 import java.nio.file.*;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -67,9 +62,11 @@
 
 import static com.spectralogic.ds3client.integration.Util.*;
 import static com.spectralogic.ds3client.integration.test.helpers.ABMTestHelper.*;
+import static com.spectralogic.ds3client.utils.Guard.isNotNullAndNotEmpty;
 import static org.hamcrest.CoreMatchers.*;
 import static org.junit.Assert.*;
 import static org.junit.Assume.assumeThat;
+import static org.junit.Assume.assumeTrue;
 
 public class Smoke_Test {
 
@@ -1676,4 +1673,19 @@ public void createFolderWithSlash() throws IOException {
             deleteAllContents(client, bucketName);
         }
     }
+
+    @Test
+    public void getBlobsOnTape() throws IOException {
+        final GetTapesSpectraS3Response getTapes = client.getTapesSpectraS3(new GetTapesSpectraS3Request());
+
+        // There must be at least one tape on the BP to run this test
+        assumeTrue(isNotNullAndNotEmpty(getTapes.getTapeListResult().getTapes()));
+
+        final UUID tapeId = getTapes.getTapeListResult().getTapes().get(0).getId();
+
+        final GetBlobsOnTapeSpectraS3Response getBlobs = client
+                .getBlobsOnTapeSpectraS3(new GetBlobsOnTapeSpectraS3Request(tapeId));
+
+        assertThat(getBlobs.getBulkObjectListResult(), is(notNullValue()));
+    }
 }

From dc32a739564ee2aa720af4f085b9123fa4904347 Mon Sep 17 00:00:00 2001
From: DenverM80 
Date: Mon, 21 Aug 2017 16:33:38 -0600
Subject: [PATCH 089/229] Force US locale for compatibility with English BP
 server

---
 .../ds3client/networking/NetworkClientImpl.java     |  2 +-
 .../spectralogic/ds3client/utils/DateFormatter.java | 11 ++++++-----
 .../ds3client/utils/DateFormatter_Test.java         | 13 ++++++++++++-
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java
index 91d821f6b..938be7c24 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java
@@ -271,7 +271,7 @@ private String buildPath() {
 
         private void addHeaders(final HttpRequest httpRequest) {
             // Add common headers.
-            final String date = DateFormatter.dateToRfc882();
+            final String date = DateFormatter.dateToRfc822();
             httpRequest.addHeader(HOST, NetUtils.buildHostField(NetworkClientImpl.this.connectionDetails));
             httpRequest.addHeader(DATE, date);
             httpRequest.addHeader(CONTENT_TYPE, this.ds3Request.getContentType());
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/DateFormatter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/DateFormatter.java
index 80ad6d688..f39c15132 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/DateFormatter.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/DateFormatter.java
@@ -17,19 +17,20 @@
 
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.Locale;
 import java.util.TimeZone;
 
 public final class DateFormatter {
     final static private String RFC822FORMAT = "EEE, dd MMM yyyy HH:mm:ss Z";
     /**
-     * Returns a RFC-882 formatted string with the current time.
+     * Returns a RFC-822 formatted string with the current time.
      */
-    public static String dateToRfc882() {
-        return dateToRfc882(new Date());
+    public static String dateToRfc822() {
+        return dateToRfc822(new Date());
     }
 
-    public static String dateToRfc882(final Date date) {
-        final SimpleDateFormat sdf = new SimpleDateFormat(RFC822FORMAT);
+    public static String dateToRfc822(final Date date) {
+        final SimpleDateFormat sdf = new SimpleDateFormat(RFC822FORMAT, Locale.US);
         sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
         return sdf.format(date);
     }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/DateFormatter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/DateFormatter_Test.java
index a43d926c2..93506a694 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/DateFormatter_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/DateFormatter_Test.java
@@ -19,6 +19,7 @@
 import org.junit.Test;
 
 import java.util.Date;
+import java.util.Locale;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -29,6 +30,16 @@ public class DateFormatter_Test {
     public void formatDate() {
         final Date date = new Date(1390414308132L);
 
-        assertThat(DateFormatter.dateToRfc882(date), is("Wed, 22 Jan 2014 18:11:48 +0000"));
+        assertThat(DateFormatter.dateToRfc822(date), is("Wed, 22 Jan 2014 18:11:48 +0000"));
+    }
+
+    @Test
+    public void formatDateMandarin() {
+        final Locale defaultLocale = Locale.getDefault();
+        Locale.setDefault(Locale.CHINESE);
+        final Date date = new Date(1390414308132L);
+
+        assertThat(DateFormatter.dateToRfc822(date), is("Wed, 22 Jan 2014 18:11:48 +0000"));
+        Locale.setDefault(defaultLocale);
     }
 }

From cef11c0d29892fbb215e9ac98370e50af3a1fb66 Mon Sep 17 00:00:00 2001
From: Eric Bergstrom 
Date: Tue, 22 Aug 2017 15:22:03 -0600
Subject: [PATCH 090/229] Correcting issue where FileObjectGetter does not
 respect symlinks

---
 .gitignore                                    |  1 +
 .../ds3client/helpers/FileObjectGetter.java   |  2 +-
 .../helpers/FileObjectGetter_Test.java        | 31 +++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 1440e929b..629bdb4d3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@ atlassian-ide-plugin.xml
 bin/
 *.log
 *.rc
+.DS_Store
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java
index 4015f7664..4f99d36ee 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java
@@ -44,7 +44,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException {
         final Path objectPath = this.root.resolve(key);
         final Path parentPath = objectPath.getParent();
         if (parentPath != null) {
-            Files.createDirectories(parentPath);
+            Files.createDirectories(FileUtils.resolveForSymbolic(parentPath));
         }
 
         if ( ! FileUtils.isTransferablePath(objectPath)) {
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java
index 82c7ac73b..6df56d140 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java
@@ -18,9 +18,13 @@
 import com.spectralogic.ds3client.utils.Platform;
 import org.apache.commons.io.FileUtils;
 import org.junit.Assume;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -28,8 +32,12 @@
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class FileObjectGetter_Test {
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
     @Test
     public void testThatNamedPipeThrows() throws IOException, InterruptedException {
         Assume.assumeFalse(Platform.isWindows());
@@ -77,4 +85,27 @@ public void testThatFileReportsAsRegularOnWindows() throws IOException, Interrup
 
         assertFalse(caughtException.get());
     }
+
+    @Test
+    public void testThatSymbolicLinksAreResolved() {
+        Assume.assumeTrue(Platform.isMac());
+        final String message = "Hello World";
+        final String file = "file.txt";
+        try {
+            final Path tempDirectory = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "ds3");
+            final Path realDirectory = Files.createDirectory(Paths.get(tempDirectory.toString(), "dir"));
+            final Path symbolicDirectory = Paths.get(tempDirectory.toString(), "symbolic");
+            Files.createSymbolicLink(symbolicDirectory, realDirectory);
+            Files.createFile(Paths.get(realDirectory.toString(), file));
+            final ByteBuffer bb = ByteBuffer.wrap(message.getBytes());
+
+            final SeekableByteChannel getterChannel = new FileObjectGetter(symbolicDirectory).buildChannel(file);
+            getterChannel.write(bb);
+            getterChannel.close();
+            final String content = new String(Files.readAllBytes(Paths.get(realDirectory.toString(), file)));
+            assertTrue(message.equals(content));
+        } catch (final IOException e) {
+            fail("Symbolic links are not handled correctly");
+        }
+    }
 }

From e808e7cd69d41789340a03d936d4aae825155a90 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 22 Aug 2017 15:46:40 -0600
Subject: [PATCH 091/229] Run tests intended to use symbolic links on un*x-like
 platforms.

---
 .../spectralogic/ds3client/helpers/FileObjectGetter_Test.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java
index 6df56d140..423d7d891 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/FileObjectGetter_Test.java
@@ -88,7 +88,7 @@ public void testThatFileReportsAsRegularOnWindows() throws IOException, Interrup
 
     @Test
     public void testThatSymbolicLinksAreResolved() {
-        Assume.assumeTrue(Platform.isMac());
+        Assume.assumeFalse(Platform.isWindows());
         final String message = "Hello World";
         final String file = "file.txt";
         try {

From a65aa286b9865f976fefb0491082db62c0d787cc Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 28 Aug 2017 16:20:34 -0600
Subject: [PATCH 092/229] Tests for zero length objects and moved create file
 to helpers

---
 .../helpers/FileSystemHelper_Test.java        |   6 +-
 .../integration/PutJobManagement_Test.java    | 128 ++++++++++++------
 .../ds3client/integration/Smoke_Test.java     |  20 ---
 .../com/spectralogic/ds3client/Ds3Client.java |   6 -
 .../spectralogic/ds3client/Ds3ClientImpl.java |   8 --
 .../commands/decorators/PutFolderRequest.java |  49 -------
 .../decorators/PutFolderResponse.java         |  29 ----
 .../ds3client/helpers/Ds3ClientHelpers.java   |   6 +-
 .../helpers/Ds3ClientHelpersImpl.java         |  20 ++-
 .../ds3client/Ds3Client_Test.java             |   6 -
 .../commands/GetObjectRequest_Test.java       |   5 +-
 11 files changed, 111 insertions(+), 172 deletions(-)
 delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
 delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
index badd3dddc..446b24601 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
@@ -16,7 +16,7 @@
 package com.spectralogic.ds3client.helpers;
 
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
+import com.spectralogic.ds3client.commands.PutObjectResponse;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.integration.Util;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
@@ -317,7 +317,7 @@ public void createFolderWithSlash() throws IOException {
         final String folderName = "FolderNameWithSlash/";
 
         try {
-            final PutFolderResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
+            final PutObjectResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
             assertNotNull(response);
         } finally {
             deleteAllContents(client, BUCKET_NAME);
@@ -329,7 +329,7 @@ public void createFolderWithNoSlash() throws IOException {
         final String folderName = "FolderNameNoSlash";
 
         try {
-            final PutFolderResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
+            final PutObjectResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
             assertNotNull(response);
         } finally {
             deleteAllContents(client, BUCKET_NAME);
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
index ec279b579..e8e953fdb 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
@@ -15,6 +15,7 @@
 
 package com.spectralogic.ds3client.integration;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
@@ -24,50 +25,31 @@
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.commands.spectrads3.notifications.*;
 import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException;
-import com.spectralogic.ds3client.helpers.ChecksumFunction;
-import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
-import com.spectralogic.ds3client.helpers.FailureEventListener;
-import com.spectralogic.ds3client.helpers.FileObjectGetter;
-import com.spectralogic.ds3client.helpers.FileObjectPutter;
-import com.spectralogic.ds3client.helpers.JobPart;
-import com.spectralogic.ds3client.helpers.ObjectCompletedListener;
+import com.spectralogic.ds3client.helpers.*;
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
-import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy;
-import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryBehavior;
-import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryDelayBehavior;
-import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ClientDefinedChunkAttemptRetryDelayBehavior;
-import com.spectralogic.ds3client.helpers.strategy.blobstrategy.MaxChunkAttemptsRetryBehavior;
-import com.spectralogic.ds3client.helpers.strategy.blobstrategy.PutSequentialBlobStrategy;
+import com.spectralogic.ds3client.helpers.strategy.blobstrategy.*;
 import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy;
 import com.spectralogic.ds3client.helpers.strategy.channelstrategy.SequentialFileReaderChannelStrategy;
-import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher;
-import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl;
-import com.spectralogic.ds3client.helpers.strategy.transferstrategy.MaxNumObjectTransferAttemptsDecorator;
-import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferMethod;
-import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferRetryDecorator;
-import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
-import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;
-import com.spectralogic.ds3client.integration.test.helpers.ABMTestHelper;
-import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory;
-import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
-import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
+import com.spectralogic.ds3client.helpers.strategy.transferstrategy.*;
+import com.spectralogic.ds3client.integration.test.helpers.*;
+import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory.ClientFailureType;
 import com.spectralogic.ds3client.metadata.MetadataAccessImpl;
 import com.spectralogic.ds3client.models.*;
+import com.spectralogic.ds3client.models.Objects;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
 import com.spectralogic.ds3client.utils.Platform;
 import com.spectralogic.ds3client.utils.ResourceUtils;
-
-import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory.ClientFailureType;
-
 import com.spectralogic.ds3client.utils.hashing.ChecksumUtils;
 import com.spectralogic.ds3client.utils.hashing.Hasher;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.BufferedInputStream;
 import java.io.File;
@@ -83,26 +65,15 @@
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShim;
-
 import static com.spectralogic.ds3client.integration.Util.RESOURCE_BASE_NAME;
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
 import static org.hamcrest.Matchers.*;
 import static org.junit.Assert.*;
 
-import com.spectralogic.ds3client.helpers.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 public class PutJobManagement_Test {
     private static final Logger LOG = LoggerFactory.getLogger(PutJobManagement_Test.class);
 
@@ -157,6 +128,52 @@ public void nakedS3Put() throws IOException, URISyntaxException {
         }
     }
 
+    @SuppressWarnings("deprecation")
+    @Test
+    public void nakedS3PutEmptyFile() throws IOException, URISyntaxException {
+        try {
+            final String fileName = "emptyFile.txt";
+
+            final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(0);
+            final PutObjectResponse putObjectResponse = client.putObject(
+                    new PutObjectRequest(BUCKET_NAME, fileName, channel, 0));
+            assertThat(putObjectResponse, is(notNullValue()));
+
+            final GetBucketResponse request = client.getBucket(new GetBucketRequest(BUCKET_NAME));
+            final ListBucketResult result = request.getListBucketResult();
+
+            assertThat(result.getObjects().size(), is(1));
+            final Contents contents = result.getObjects().get(0);
+            assertThat(contents.getKey(), is(fileName));
+            assertThat(contents.getSize(), is((long) 0));
+        } finally {
+            deleteAllContents(client, BUCKET_NAME);
+        }
+    }
+
+    @SuppressWarnings("deprecation")
+    @Test
+    public void nakedS3PutFolder() throws IOException, URISyntaxException {
+        try {
+            final String fileName = "NakedS3Folder/";
+
+            final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(0);
+            final PutObjectResponse putObjectResponse = client.putObject(
+                    new PutObjectRequest(BUCKET_NAME, fileName, channel, 0));
+
+            assertThat(putObjectResponse, is(notNullValue()));
+            final GetBucketResponse request = client.getBucket(new GetBucketRequest(BUCKET_NAME));
+            final ListBucketResult result = request.getListBucketResult();
+            assertThat(result.getObjects().size(), is(1));
+
+            final Contents contents = result.getObjects().get(0);
+            assertThat(contents.getKey(), is(fileName));
+            assertThat(contents.getSize(), is((long) 0));
+        } finally {
+            deleteAllContents(client, BUCKET_NAME);
+        }
+    }
+
     @Test
     public void getActiveJobs() throws IOException, URISyntaxException {
         try {
@@ -1102,6 +1119,39 @@ private interface ObjectTransferExceptionHandler {
         boolean handleException(final Throwable t);
     }
 
+    @Test
+    public void testWriteJobWithZeroLengthObject() throws Exception {
+
+        final Ds3Client client = Ds3ClientBuilder.fromEnv()
+                .withHttps(false)
+                .build();
+
+        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), null);
+        final Path tempFile = Files.createTempFile(tempDirectory, "EmptyFile", null);
+        final String fileName = tempFile.getFileName().toString();
+
+        try {
+            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client);
+
+            final ImmutableList objects = ImmutableList.of(new Ds3Object(fileName, 0));
+
+            final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects);
+            writeJob.transfer(new FileObjectPutter(tempDirectory));
+
+            final GetBucketResponse request = client.getBucket(new GetBucketRequest(BUCKET_NAME));
+            final ListBucketResult result = request.getListBucketResult();
+            assertThat(result.getObjects().size(), is(1));
+
+            final Contents contents = result.getObjects().get(0);
+            assertThat(contents.getKey(), is(fileName));
+            assertThat(contents.getSize(), is((long) 0));
+
+        } finally {
+            FileUtils.deleteDirectory(tempDirectory.toFile());
+            deleteAllContents(client, BUCKET_NAME);
+        }
+    }
+
     @Test
     public void testWriteJobWithRetriesAndUserDefinedChecksum() throws Exception {
         final int maxNumObjectTransferAttempts = 3;
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
index 2a36c04de..8849ed328 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
@@ -15,14 +15,11 @@
 
 package com.spectralogic.ds3client.integration;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.*;
-import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
-import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.interfaces.BulkResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.*;
@@ -1657,23 +1654,6 @@ public void testGetObjectsWithFullDetails() throws IOException, URISyntaxExcepti
         }
     }
 
-    @Test
-    public void createFolderWithSlash() throws IOException {
-        final String folderName = "FolderNameWithSlash/";
-        final String bucketName = "FolderNameWithSlashTestBucket";
-
-        try {
-            HELPERS.ensureBucketExists(bucketName, envDataPolicyId);
-            final Ds3Object ds3Object = new Ds3Object(folderName, 0);
-            final PutBulkJobSpectraS3Response jobResponse = client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucketName, ImmutableList.of(ds3Object)));
-
-            final PutFolderResponse response = client.putFolder(new PutFolderRequest(bucketName, folderName, jobResponse.getMasterObjectList().getJobId()));
-            assertNotNull(response);
-        } finally {
-            deleteAllContents(client, bucketName);
-        }
-    }
-
     @Test
     public void getBlobsOnTape() throws IOException {
         final GetTapesSpectraS3Response getTapes = client.getTapesSpectraS3(new GetTapesSpectraS3Request());
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index fbee586c3..b81e82ce0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -20,8 +20,6 @@
 import com.spectralogic.ds3client.annotations.Resource;
 import com.spectralogic.ds3client.annotations.ResponsePayloadModel;
 import com.spectralogic.ds3client.commands.*;
-import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
-import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.commands.spectrads3.notifications.*;
 import com.spectralogic.ds3client.models.JobNode;
@@ -36,10 +34,6 @@
 public interface Ds3Client extends Closeable {
 
     ConnectionDetails getConnectionDetails();
-
-    PutFolderResponse putFolder(final PutFolderRequest request)
-            throws IOException;
-
     
     
     AbortMultiPartUploadResponse abortMultiPartUpload(final AbortMultiPartUploadRequest request)
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
index 16c5bd388..a04ac9ca7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
@@ -18,8 +18,6 @@
 
 import java.io.IOException;
 import com.spectralogic.ds3client.commands.*;
-import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
-import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
 import com.spectralogic.ds3client.commands.parsers.*;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.commands.spectrads3.notifications.*;
@@ -47,12 +45,6 @@ public ConnectionDetails getConnectionDetails() {
         return this.netClient.getConnectionDetails();
     }
 
-    @Override
-    public PutFolderResponse putFolder(final PutFolderRequest request) throws IOException {
-        final PutObjectResponse response = putObject(request.getPutObjectRequest());
-        return new PutFolderResponse(response);
-    }
-
     @Override
     public AbortMultiPartUploadResponse abortMultiPartUpload(final AbortMultiPartUploadRequest request) throws IOException {
         return new AbortMultiPartUploadResponseParser().response(this.netClient.getResponse(request));
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
deleted file mode 100644
index 8f0b0d55e..000000000
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderRequest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * ******************************************************************************
- *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
- *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
- *   this file except in compliance with the License. A copy of the License is located at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *   or in the "license" file accompanying this file.
- *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
- *   specific language governing permissions and limitations under the License.
- * ****************************************************************************
- */
-
-package com.spectralogic.ds3client.commands.decorators;
-
-import com.spectralogic.ds3client.commands.PutObjectRequest;
-import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
-
-import java.util.UUID;
-
-/**
- * Decorates the {@link com.spectralogic.ds3client.commands.PutObjectRequest} and is used to
- * ensure the correct creation of a folder using the Put Object command.
- *
- * A folder is created by putting an object with no content, of zero size, and with the
- * content-length=0.  Also, the name of the folder must end with a forward slash '/'.
- */
-public class PutFolderRequest {
-
-    private final PutObjectRequest putObjectRequest;
-
-    public PutFolderRequest(final String bucketName, final String folderName, final UUID job) {
-        validateNameEndsWithSlash(folderName);
-        putObjectRequest = new PutObjectRequest(bucketName, folderName, job, 0, 0, null);
-        putObjectRequest.getHeaders().put("Content-Length", "0");
-    }
-
-    private static void validateNameEndsWithSlash(final String folderName) {
-        if (!folderName.endsWith("/")) {
-            throw new FolderNameMissingTrailingForwardSlash(folderName);
-        }
-    }
-
-    public PutObjectRequest getPutObjectRequest() {
-        return putObjectRequest;
-    }
-}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java
deleted file mode 100644
index 1707e7996..000000000
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/decorators/PutFolderResponse.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * ******************************************************************************
- *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
- *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
- *   this file except in compliance with the License. A copy of the License is located at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *   or in the "license" file accompanying this file.
- *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
- *   specific language governing permissions and limitations under the License.
- * ****************************************************************************
- */
-
-package com.spectralogic.ds3client.commands.decorators;
-
-import com.spectralogic.ds3client.commands.PutObjectResponse;
-
-/**
- * Decorates the {@link com.spectralogic.ds3client.commands.PutObjectResponse} and is used
- * by command {@link com.spectralogic.ds3client.Ds3Client#putFolder(PutFolderRequest)}
- */
-public class PutFolderResponse extends PutObjectResponse {
-
-    public PutFolderResponse(final PutObjectResponse response) {
-        super(response.getChecksum(), response.getChecksumType());
-    }
-}
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 5845551c6..99f25d7df 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -18,12 +18,12 @@
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
+import com.spectralogic.ds3client.commands.PutObjectResponse;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
+import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.models.Contents;
-import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.utils.Predicate;
 
@@ -585,5 +585,5 @@ public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInD
     /**
      * Creates a folder in the specified bucket
      */
-    public abstract PutFolderResponse createFolder(final String bucketName, final String folderName) throws IOException;
+    public abstract PutObjectResponse createFolder(final String bucketName, final String folderName) throws IOException;
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 0d562c292..9ab225d87 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -22,11 +22,7 @@
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.HeadBucketRequest;
-import com.spectralogic.ds3client.commands.HeadBucketResponse;
-import com.spectralogic.ds3client.commands.PutBucketRequest;
-import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
-import com.spectralogic.ds3client.commands.decorators.PutFolderResponse;
+import com.spectralogic.ds3client.commands.*;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
@@ -43,6 +39,7 @@
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.models.common.Range;
 import com.spectralogic.ds3client.networking.FailedRequestException;
+import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -596,13 +593,22 @@ public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(
      * Creates a folder in the specified bucket.
      */
     @Override
-    public PutFolderResponse createFolder(final String bucketName, final String folderName) throws IOException {
+    public PutObjectResponse createFolder(final String bucketName, final String folderName) throws IOException {
         final String normalizedFolderName = ensureNameEndsWithSlash(folderName);
+
         final Ds3Object ds3Object = new Ds3Object(normalizedFolderName, 0);
         final PutBulkJobSpectraS3Response jobResponse = client
                 .putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucketName, ImmutableList.of(ds3Object)));
 
-        return client.putFolder(new PutFolderRequest(bucketName, normalizedFolderName, jobResponse.getMasterObjectList().getJobId()));
+        final PutObjectRequest folderRequest = new PutObjectRequest(
+                bucketName,
+                normalizedFolderName,
+                new ByteArraySeekableByteChannel(0),
+                jobResponse.getMasterObjectList().getJobId(),
+                0,
+                0);
+
+        return client.putObject(folderRequest);
     }
 
     /**
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index ab7138f29..6f70fc9af 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -17,7 +17,6 @@
 
 import com.google.common.collect.*;
 import com.spectralogic.ds3client.commands.*;
-import com.spectralogic.ds3client.commands.decorators.PutFolderRequest;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException;
 import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
@@ -1151,11 +1150,6 @@ public void getObjectVerifyFullPayload() throws IOException {
                         0));
     }
 
-    @Test (expected = FolderNameMissingTrailingForwardSlash.class)
-    public void createFolderWithNoSlash() throws IOException {
-        new PutFolderRequest("BucketName", "FolderNameNoSlash", UUID.randomUUID());
-    }
-
     @Test
     public void clearSuspectBlobAzureTargetsSpectraS3Test() throws IOException {
         MockNetwork
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/GetObjectRequest_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/GetObjectRequest_Test.java
index 0aea872ba..3163096f2 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/GetObjectRequest_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/GetObjectRequest_Test.java
@@ -16,6 +16,7 @@
 package com.spectralogic.ds3client.commands;
 
 import com.spectralogic.ds3client.models.common.Range;
+import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -33,7 +34,7 @@ public void singleRangeFormat() {
        final GetObjectRequest request = new GetObjectRequest(
                "bucketName",
                "objectName",
-               null,
+               new ByteArraySeekableByteChannel(0),
                UUID.randomUUID().toString(),
                0);
 
@@ -52,7 +53,7 @@ public void multipleRangeFormat() {
         final GetObjectRequest request = new GetObjectRequest(
                 "bucketName",
                 "objectName",
-                null,
+                new ByteArraySeekableByteChannel(0),
                 UUID.randomUUID().toString(),
                 0);
 

From ba9ea4f06011ca83c78719f06beed6764b7783cb Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 28 Aug 2017 16:58:32 -0600
Subject: [PATCH 093/229] Removed unnecessary client creation

---
 .../ds3client/integration/PutJobManagement_Test.java         | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
index e8e953fdb..067bf9fb0 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
@@ -1121,11 +1121,6 @@ private interface ObjectTransferExceptionHandler {
 
     @Test
     public void testWriteJobWithZeroLengthObject() throws Exception {
-
-        final Ds3Client client = Ds3ClientBuilder.fromEnv()
-                .withHttps(false)
-                .build();
-
         final Path tempDirectory = Files.createTempDirectory(Paths.get("."), null);
         final Path tempFile = Files.createTempFile(tempDirectory, "EmptyFile", null);
         final String fileName = tempFile.getFileName().toString();

From 14d6bd90f8e33910442beee4ccef759281fbe75c Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Tue, 29 Aug 2017 13:56:01 -0600
Subject: [PATCH 094/229] Updated create folder helper to have void return type

---
 .../ds3client/helpers/FileSystemHelper_Test.java            | 6 ++----
 .../spectralogic/ds3client/helpers/Ds3ClientHelpers.java    | 2 +-
 .../ds3client/helpers/Ds3ClientHelpersImpl.java             | 4 ++--
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
index 446b24601..ebe263f06 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
@@ -317,8 +317,7 @@ public void createFolderWithSlash() throws IOException {
         final String folderName = "FolderNameWithSlash/";
 
         try {
-            final PutObjectResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
-            assertNotNull(response);
+            HELPERS.createFolder(BUCKET_NAME, folderName);
         } finally {
             deleteAllContents(client, BUCKET_NAME);
         }
@@ -329,8 +328,7 @@ public void createFolderWithNoSlash() throws IOException {
         final String folderName = "FolderNameNoSlash";
 
         try {
-            final PutObjectResponse response = HELPERS.createFolder(BUCKET_NAME, folderName);
-            assertNotNull(response);
+            HELPERS.createFolder(BUCKET_NAME, folderName);
         } finally {
             deleteAllContents(client, BUCKET_NAME);
         }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 99f25d7df..1818dfe05 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -585,5 +585,5 @@ public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInD
     /**
      * Creates a folder in the specified bucket
      */
-    public abstract PutObjectResponse createFolder(final String bucketName, final String folderName) throws IOException;
+    public abstract void createFolder(final String bucketName, final String folderName) throws IOException;
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index 9ab225d87..af21b361e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -593,7 +593,7 @@ public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(
      * Creates a folder in the specified bucket.
      */
     @Override
-    public PutObjectResponse createFolder(final String bucketName, final String folderName) throws IOException {
+    public void createFolder(final String bucketName, final String folderName) throws IOException {
         final String normalizedFolderName = ensureNameEndsWithSlash(folderName);
 
         final Ds3Object ds3Object = new Ds3Object(normalizedFolderName, 0);
@@ -608,7 +608,7 @@ public PutObjectResponse createFolder(final String bucketName, final String fold
                 0,
                 0);
 
-        return client.putObject(folderRequest);
+        client.putObject(folderRequest);
     }
 
     /**

From ab2f88ac50517f01e0e64cd1fa80ec0adf39ee26 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Tue, 29 Aug 2017 14:03:29 -0600
Subject: [PATCH 095/229] Removed unnecessary imports

---
 .../ds3client/helpers/FileSystemHelper_Test.java            | 1 -
 .../spectralogic/ds3client/helpers/Ds3ClientHelpers.java    | 1 -
 .../ds3client/helpers/Ds3ClientHelpersImpl.java             | 6 ++++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
index ebe263f06..2e904d99e 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java
@@ -16,7 +16,6 @@
 package com.spectralogic.ds3client.helpers;
 
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.PutObjectResponse;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.integration.Util;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 1818dfe05..91145eeba 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -18,7 +18,6 @@
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.PutObjectResponse;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index af21b361e..caedd1e05 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -22,7 +22,10 @@
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.commands.HeadBucketRequest;
+import com.spectralogic.ds3client.commands.HeadBucketResponse;
+import com.spectralogic.ds3client.commands.PutBucketRequest;
+import com.spectralogic.ds3client.commands.PutObjectRequest;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.events.EventRunner;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
@@ -30,7 +33,6 @@
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
 import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
-import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;

From 9a28ea91822dd01f4eb1ed50545bc1f5b7b2d30e Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Wed, 30 Aug 2017 10:36:27 -0600
Subject: [PATCH 096/229] Java 8 and kotlin (#504)

* Adding java 8 and kotlin to the build.  Updated to the latest version of gradle.  Updated the listObjectsForDirectory helper method to be lazy.  Updated the start_job commands to fully pass the Iterable along instead of copying it to a List.  Added a paginated bucket delete call that minimizes the number of API calls that are used to delete all the objects in a bucket.  Fixed an issue in the payload generation of the bulk request body to work around an issue when trying to serialize the payload from an Iterable.  Updated the version of Jackson.

* Updating dependencies to their latest version.  Pulling most of the version information into a single file.  Fixed a bug in the network code introduced by reving the http client version.

* Updating the shadowjar build to reflect the most recent changes, and to make the service loader work correctly as well.

* Updating the shadowjar build more so that all the versions reference the same variable

* Working on the last few finishing touches on the shadowJar setup.  We're excluding the ds3-interfaces and ds3-utils projects from the metadata build since those classes must be included as apart of the ds3-sdk project, and will already exist.

* Updating the shadow build for the metadata module

* Removing some unused import.  Removing some un-needed type declarations.  Performing some minor code cleanup

* Adding copyright information

* Adding another missing copyright

* Updating to use a kotlin built in function instead of our own Guard class

* Removing a debug print statement
---
 .travis.yml                                   |   2 -
 build.gradle                                  |  30 +++--
 ds3-metadata/build.gradle                     |  29 +++--
 ds3-sdk-integration/build.gradle              |   8 +-
 .../ds3client/integration/Util.java           |  18 +--
 .../test/helpers/TempStorageUtil.java         |   2 +-
 ds3-sdk-samples/build.gradle                  |   2 +-
 ds3-sdk/build.gradle                          |  34 ++++--
 .../commands/interfaces/AbstractRequest.java  |   2 +-
 .../ds3client/helpers/Ds3ClientHelpers.java   |  25 ++--
 .../helpers/Ds3ClientHelpersImpl.java         | 109 +++++++-----------
 .../helpers/strategy/StrategyUtils.java       |   1 -
 .../GetSequentialBlobStrategy.java            |   2 -
 .../helpers/util/PartialObjectHelpers.java    |   5 +-
 .../ds3client/models/bulk/Ds3ObjectList.java  |  12 +-
 .../ds3client/serializer/XmlOutput.java       |   3 +
 .../ds3client/utils/JobUtils.java             |   1 -
 .../utils/collections/StreamWrapper.java      |  43 +++----
 .../ds3client/helpers/DeleteBucket.kt         |  38 ++++++
 .../ds3client/helpers/FileTreeWalker.kt       |  35 ++++++
 .../utils/collections/IterableExtensions.kt   |  24 ++++
 .../utils/collections/WindowedIterator.kt     |  43 +++++++
 .../ds3client/ConnectionFixture.java          |   2 +-
 .../ds3client/Ds3Client_Test.java             |  40 +++----
 .../spectralogic/ds3client/MockNetwork.java   |   2 +-
 .../SeekableByteChannelInputStream_Test.java  |   2 +-
 .../utils/ResponseParserUtils_Test.java       |   2 +-
 .../helpers/Ds3ClientHelpers_Test.java        |   6 +-
 .../ds3client/helpers/RequestMatchers.java    |   4 +-
 .../channelbuilders/StreamObjectPutter.java   |   2 +-
 .../ds3client/utils/NetUtils_Test.java        |   6 +-
 .../ds3client/utils/Signature_Test.java       |   3 -
 .../ds3client/helpers/FileTreeWalker_Test.kt  |  50 ++++++++
 .../collections/WindowedIterator_Test.kt      |  76 ++++++++++++
 ds3-utils/build.gradle                        |  31 +----
 .../exceptions/AggregateException.java        |   0
 .../exceptions/AggregateException_Test.java   |   0
 gradle.properties                             |  11 ++
 gradle/wrapper/gradle-wrapper.jar             | Bin 54208 -> 54783 bytes
 gradle/wrapper/gradle-wrapper.properties      |   4 +-
 40 files changed, 475 insertions(+), 234 deletions(-)
 rename ds3-interfaces/build.gradle => ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/StreamWrapper.java (54%)
 create mode 100644 ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/DeleteBucket.kt
 create mode 100644 ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker.kt
 create mode 100644 ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/IterableExtensions.kt
 create mode 100644 ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator.kt
 create mode 100644 ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker_Test.kt
 create mode 100644 ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator_Test.kt
 rename {ds3-interfaces => ds3-utils}/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java (100%)
 rename {ds3-interfaces => ds3-utils}/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java (100%)
 create mode 100644 gradle.properties

diff --git a/.travis.yml b/.travis.yml
index 5b850af23..8d5b0b1e1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,4 @@
 language: java
 jdk:
   - oraclejdk8
-  - oraclejdk7
-  - openjdk7
 script: ./gradlew -S clean jar ds3-sdk:test ds3-utils:test ds3-metadata:test
diff --git a/build.gradle b/build.gradle
index a2ee62b60..7d6f0212a 100644
--- a/build.gradle
+++ b/build.gradle
@@ -13,45 +13,61 @@
  *  ****************************************************************************
  */
 
+buildscript {
+    ext.kotlin_version = '1.1.4-2'
+
+    repositories {
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
+}
+
+
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '4.0.0'
+    version = '4.0.1'
 }
 
 subprojects {
     apply plugin: 'maven'
     apply plugin: 'java'
+    apply plugin: 'kotlin'
     apply plugin: 'findbugs'
 
-    sourceCompatibility = JavaVersion.VERSION_1_7
+    sourceCompatibility = JavaVersion.VERSION_1_8
     repositories {
         mavenCentral()
         mavenLocal()
     }
 
     dependencies {
-        compile     'org.slf4j:slf4j-api:1.7.22'
-        testCompile ('org.mockito:mockito-core:1.10.19') {
+        compile "org.slf4j:slf4j-api:$slf4jVersion"
+        testCompile ("org.mockito:mockito-core:$mockitoVersion") {
             exclude group: 'org.hamcrest'
         }
 
-        testCompile 'junit:junit:4.12'
-        testCompile 'org.slf4j:slf4j-simple:1.7.22'
+        testCompile "junit:junit:$junitVersion"
+        testCompile "org.slf4j:slf4j-simple:$slf4jVersion"
     }
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '3.5'
+    gradleVersion = '4.1'
 }
 
 project(':ds3-sdk') {
     dependencies {
+        compile project(':ds3-interfaces')
         compile project(':ds3-utils')
     }
 }
 
 project(':ds3-metadata') {
     dependencies {
+        compile project(':ds3-interfaces')
         compile project(':ds3-utils')
     }
 }
diff --git a/ds3-metadata/build.gradle b/ds3-metadata/build.gradle
index 9d45f587d..6002069a0 100644
--- a/ds3-metadata/build.gradle
+++ b/ds3-metadata/build.gradle
@@ -16,7 +16,7 @@
 buildscript {
     repositories { jcenter() }
     dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
     }
 }
 
@@ -26,14 +26,19 @@ shadowJar {
     relocate 'org.apache', 'ds3metafatjar.org.apache'
     relocate 'com.google', 'ds3metafatjar.com.google'
     relocate 'com.sun.jna', 'ds3metafatjar.net.java.dev.jna'
+    relocate 'org.jetbrains', 'ds3metafatjar.org.jetbrains'
+    relocate 'org.intellij', 'ds3metafatjar.org.intellij'
+    relocate 'org.codehaus', 'ds3metafatjar.org.codehaus'
 
     dependencies {
-        exclude(dependency('org.hamcrest:hamcrest-library:1.3'))
-        exclude(dependency('org.mockito:mockito-core:1.10.19'))
-        exclude(dependency('junit:junit:4.12'))
-        exclude(dependency('org.slf4j:slf4j-api:1.7.22'))
-        exclude(dependency('org.slf4j:slf4j-simple:1.7.22'))
-        exclude(dependency('org.apache.commons:commons-lang3:3.0'))
+        exclude(project(":ds3-interfaces")) // this is being excluded since it must be used with the sdk, which already has this dependency included
+        exclude(project(":ds3-utils")) // this is being excluded since it must be used with the sdk, which already has this dependency included
+        exclude(dependency("org.hamcrest:hamcrest-library:$hamcrestVersion"))
+        exclude(dependency("org.mockito:mockito-core:$mockitoVersion"))
+        exclude(dependency("junit:junit:$junitVersion"))
+        exclude(dependency("org.slf4j:slf4j-api:$slf4jVersion"))
+        exclude(dependency("org.slf4j:slf4j-simple:$slf4jVersion"))
+        exclude(dependency("org.apache.commons:commons-lang3:$commonslangVersion"))
     }
 }
 
@@ -42,10 +47,10 @@ artifacts {
 }
 
 dependencies {
-    compile     group: 'net.java.dev.jna', name: 'jna-platform', version: '4.2.2'
-    compile     group: 'net.java.dev.jna', name: 'jna', version: '4.2.2'
-    compile     'commons-io:commons-io:2.4'
-    compile     'org.apache.httpcomponents:httpclient:4.5.1'
+    compile     group: 'net.java.dev.jna', name: 'jna-platform', version: "$jnaVersion"
+    compile     group: 'net.java.dev.jna', name: 'jna', version: "$jnaVersion"
+    compile     "commons-io:commons-io:$commonsioVersion"
+    compile     "org.apache.httpcomponents:httpclient:$httpclientVersion"
 
-    testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0'
+    testCompile group: 'org.apache.commons', name: 'commons-lang3', version: "$commonslangVersion"
 }
diff --git a/ds3-sdk-integration/build.gradle b/ds3-sdk-integration/build.gradle
index 56fcfed45..362cf2907 100644
--- a/ds3-sdk-integration/build.gradle
+++ b/ds3-sdk-integration/build.gradle
@@ -14,9 +14,9 @@
  */
 
 dependencies {
-    compile 'commons-codec:commons-codec:1.10'
-    compile 'junit:junit:4.12'
-    testCompile 'org.hamcrest:hamcrest-library:1.3'
-    testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0'
+    compile "commons-codec:commons-codec:$commonscodecVersion"
+    compile "junit:junit:$junitVersion"
+    testCompile "org.hamcrest:hamcrest-library:$hamcrestVersion"
+    testCompile group: 'org.apache.commons', name: 'commons-lang3', version: "$commonslangVersion"
 }
 
diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index 144814b52..245e5ad99 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -20,6 +20,7 @@
 import com.spectralogic.ds3client.commands.DeleteBucketRequest;
 import com.spectralogic.ds3client.commands.DeleteObjectRequest;
 import com.spectralogic.ds3client.commands.spectrads3.GetSystemInformationSpectraS3Request;
+import com.spectralogic.ds3client.helpers.DeleteBucket;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.channelbuilders.PrefixAdderObjectChannelBuilder;
 import com.spectralogic.ds3client.models.Contents;
@@ -40,7 +41,7 @@
 import static org.junit.Assume.assumeTrue;
 
 
-public class Util {
+public final class Util {
     private static final Logger LOG = LoggerFactory.getLogger(Util.class);
     public static final String RESOURCE_BASE_NAME = "books/";
     public static final String[] BOOKS = {"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"};
@@ -110,22 +111,11 @@ public static void loadBookTestDataWithPrefix(final Ds3Client client, final Stri
     }
 
     public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException {
-        final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
-
-        final Iterable objects = helpers.listObjects(bucketName);
-        for(final Contents contents : objects) {
-            client.deleteObject(new DeleteObjectRequest(bucketName, contents.getKey()));
-        }
-
-        client.deleteBucket(new DeleteBucketRequest(bucketName));
+        Ds3ClientHelpers.wrap(client).deleteBucket(bucketName);
     }
 
     public static void deleteBucketContents(final Ds3Client client, final String bucketName) throws IOException {
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
-
-        final Iterable objects = helpers.listObjects(bucketName);
-        for(final Contents contents : objects) {
-            client.deleteObject(new DeleteObjectRequest(bucketName, contents.getKey()));
-        }
+        DeleteBucket.INSTANCE.deleteBucketContents(helpers, bucketName);
     }
 }
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
index 040a80a65..0f7efe21d 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
@@ -30,7 +30,7 @@
  * and partition for use in the integration tests to avoid error if the BP does not currently
  * have a partition available for running the unit tests.
  */
-public class TempStorageUtil {
+public final class TempStorageUtil {
 
     private static final String DATA_POLICY_NAME = "_dp";
     private static final String STORAGE_DOMAIN_NAME = "_sd";
diff --git a/ds3-sdk-samples/build.gradle b/ds3-sdk-samples/build.gradle
index e0b2a6709..f71a1040a 100644
--- a/ds3-sdk-samples/build.gradle
+++ b/ds3-sdk-samples/build.gradle
@@ -18,5 +18,5 @@ apply plugin: 'application'
 mainClassName = 'com.spectralogic.ds3client.samples.PartialObjectGetExample'
 
 dependencies {
-    compile 'org.slf4j:slf4j-simple:1.7.22'
+    compile "org.slf4j:slf4j-simple:$slf4jVersion"
 }
diff --git a/ds3-sdk/build.gradle b/ds3-sdk/build.gradle
index 87f7c6925..5b74d8b55 100644
--- a/ds3-sdk/build.gradle
+++ b/ds3-sdk/build.gradle
@@ -19,7 +19,7 @@ import java.nio.file.Path
 buildscript {
     repositories { jcenter() }
     dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
     }
 }
 
@@ -27,17 +27,25 @@ apply plugin: 'com.github.johnrengelman.shadow'
 
 shadowJar {
     relocate 'com.google', 'ds3fatjar.com.google'
-    relocate 'org.apache', 'ds3fatjar.org.apache'
+    relocate 'org.jetbrains', 'ds3fatjar.org.jetbrains'
+    relocate 'org.intellij', 'ds3fatjar.org.intellij'
     relocate 'org.codehaus', 'ds3fatjar.org.codehaus'
+    relocate 'kotlin', 'ds3fatjar.kotlin'
+    relocate 'edu.umd', 'ds3fatjar.edu.emd'
+    relocate 'net.jcip', 'ds3fatjar.net.jcip'
+    relocate 'com.ctc', 'ds3fatjar.com.ctc'
+    relocate 'org.apache', 'ds3fatjar.org.apache'
     relocate 'com.fasterxml', 'ds3fatjar.com.fasterxml'
     dependencies {
-        exclude(dependency('org.hamcrest:hamcrest-library:1.3'))
-        exclude(dependency('org.mockito:mockito-core:1.10.19'))
-        exclude(dependency('junit:junit:4.12'))
-        exclude(dependency('org.slf4j:slf4j-api:1.7.22'))
-        exclude(dependency('org.slf4j:slf4j-simple:1.7.22'))
-        exclude(dependency('org.apache.commons:commons-lang3:3.0'))
+        exclude(dependency("org.hamcrest:hamcrest-library:$hamcrestVersion"))
+        exclude(dependency("org.mockito:mockito-core:$mockitoVersion"))
+        exclude(dependency("junit:junit:$junitVersion"))
+        exclude(dependency("org.slf4j:slf4j-api:$slf4jVersion"))
+        exclude(dependency("org.slf4j:slf4j-simple:$slf4jVersion"))
+        exclude(dependency("org.apache.commons:commons-lang3:$commonslangVersion"))
     }
+
+    mergeServiceFiles()
 }
 
 artifacts {
@@ -78,10 +86,12 @@ jar {
 jar.dependsOn genConfigProperties
 
 dependencies {
-    compile     'org.apache.httpcomponents:httpclient:4.5.1'
+    compile     "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    compile     "org.apache.httpcomponents:httpclient:$httpclientVersion"
+    compile     "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion"
+    compile     "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:$jacksonVersion"
+    compile     "com.google.guava:guava:$guavaVersion"
     compile     'org.codehaus.woodstox:woodstox-core-asl:4.4.1'
-    compile     'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.7.1'
-    compile     'com.google.guava:guava:20.0'
     compile     'com.google.code.findbugs:annotations:3.0.1'
-    testCompile 'org.hamcrest:hamcrest-library:1.3'
+    testCompile "org.hamcrest:hamcrest-library:$hamcrestVersion"
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java
index faae9155a..4097d8425 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java
@@ -36,7 +36,7 @@ private static RequestHeaders buildDefaultHeaders() {
 
     @Override
     public String getContentType() {
-        return ContentType.APPLICATION_XML.toString();
+        return ContentType.APPLICATION_XML.getMimeType();
     }
 
     @Override
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 91145eeba..4ca191438 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -508,6 +508,12 @@ public abstract Iterable listObjects(final String bucket, final String
      */
     public abstract Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException;
 
+    /**
+     *
+     * @param bucket
+     */
+    public abstract void deleteBucket(final String bucket) throws IOException;
+
     /**
      * Returns an Iterable of {@link Ds3Object} that have a prefix added.
      */
@@ -525,20 +531,12 @@ public abstract Iterable listObjects(final String bucket, final String
     @SafeVarargs
     public final Iterable toDs3Iterable(final Iterable objects, final Predicate... filters) {
 
-        FluentIterable fluentIterable = FluentIterable.from(objects).filter(new com.google.common.base.Predicate() {
-            @Override
-            public boolean apply(@Nullable final Contents input) {
-                return input != null;
-            }
-        });
+        FluentIterable fluentIterable = FluentIterable.from(objects).filter(input -> input != null);
 
         if (filters != null) {
             for (final Predicate filter : filters) {
-                fluentIterable = fluentIterable.filter(new com.google.common.base.Predicate() {
-                    @Override
-                    public boolean apply(@Nullable final Contents input) {
-                        return filter == null || filter.test(input); // do not filter anything if filter is null
-                    }
+                fluentIterable = fluentIterable.filter(input -> {
+                    return filter == null || filter.test(input); // do not filter anything if filter is null
                 });
             }
         }
@@ -585,4 +583,9 @@ public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInD
      * Creates a folder in the specified bucket
      */
     public abstract void createFolder(final String bucketName, final String folderName) throws IOException;
+
+    /**
+     * Returns the client being wrapped
+     */
+    public abstract Ds3Client getClient();
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
index caedd1e05..59149b9de 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java
@@ -15,12 +15,9 @@
 
 package com.spectralogic.ds3client.helpers;
 
-import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
-import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.HeadBucketRequest;
 import com.spectralogic.ds3client.commands.HeadBucketResponse;
@@ -42,20 +39,17 @@
 import com.spectralogic.ds3client.models.common.Range;
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
+import com.spectralogic.ds3client.utils.collections.StreamWrapper;
 import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
 import java.io.IOException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Collection;
-import java.util.List;
 import java.util.UUID;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 class Ds3ClientHelpersImpl extends Ds3ClientHelpers {
     private final static Logger LOG = LoggerFactory.getLogger(Ds3ClientHelpersImpl.class);
@@ -145,7 +139,7 @@ private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket,
                                                     final TransferStrategyBuilder transferStrategyBuilder)
             throws IOException
     {
-        final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(bucket, Lists.newArrayList(objectsToWrite))
+        final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(bucket, objectsToWrite)
                 .withPriority(options.getPriority())
                 .withAggregating(options.isAggregating())
                 .withForce(options.isForce())
@@ -164,19 +158,19 @@ private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket,
     }
 
     @Override
-    public Job startWriteJob(final TransferStrategy transferStrategy)
+    public Ds3ClientHelpers.Job startWriteJob(final TransferStrategy transferStrategy)
             throws IOException
     {
         return new WriteJobImpl(transferStrategy);
     }
 
     @Override
-    public Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite) throws IOException {
+    public Ds3ClientHelpers.Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite) throws IOException {
         return startWriteJobUsingStreamedBehavior(bucket, objectsToWrite, WriteJobOptions.create());
     }
 
     @Override
-    public Job startWriteJobUsingStreamedBehavior(final String bucket,
+    public Ds3ClientHelpers.Job startWriteJobUsingStreamedBehavior(final String bucket,
                                                   final Iterable objectsToWrite,
                                                   final WriteJobOptions options)
             throws IOException
@@ -190,12 +184,12 @@ public Job startWriteJobUsingStreamedBehavior(final String bucket,
     }
 
     @Override
-    public Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite) throws IOException {
+    public Ds3ClientHelpers.Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite) throws IOException {
         return startWriteJobUsingRandomAccessBehavior(bucket, objectsToWrite, WriteJobOptions.create());
     }
 
     @Override
-    public Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) throws IOException {
+    public Ds3ClientHelpers.Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) throws IOException {
         Preconditions.checkNotNull(options, "options may not be null.");
 
         final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder();
@@ -210,10 +204,8 @@ public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objectsToRead, final ReadJobOptions options)
-            throws IOException
-    {
-        final List objects = Lists.newArrayList(objectsToRead);
+    public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objects, final ReadJobOptions options)
+            throws IOException {
 
         final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request;
 
@@ -228,13 +220,13 @@ public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objects, final ReadJobOptions options) {
+    private GetBulkJobSpectraS3Request makeGetBulkJobSpectraS3Request(final String bucket, final Iterable objects, final ReadJobOptions options) {
         return new GetBulkJobSpectraS3Request(bucket, objects)
                 .withPriority(options.getPriority())
                 .withName(options.getName());
     }
 
-    private Ds3ClientHelpers.Job innerStartReadJob(final List objects,
+    private Ds3ClientHelpers.Job innerStartReadJob(final Iterable objects,
                                                    final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request,
                                                    final TransferStrategyBuilder transferStrategyBuilder)
             throws IOException
@@ -253,21 +245,19 @@ private Ds3ClientHelpers.Job innerStartReadJob(final List objects,
     }
 
     @Override
-    public Job startReadJob(final TransferStrategy transferStrategy) throws IOException {
+    public Ds3ClientHelpers.Job startReadJob(final TransferStrategy transferStrategy) throws IOException {
         return new ReadJobImpl(transferStrategy);
     }
 
     @Override
-    public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead) throws IOException {
+    public Ds3ClientHelpers.Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead) throws IOException {
         return startReadJobUsingStreamedBehavior(bucket, objectsToRead, ReadJobOptions.create());
     }
 
     @Override
-    public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException {
+    public Ds3ClientHelpers.Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objects, final ReadJobOptions options) throws IOException {
         Preconditions.checkNotNull(options, "options may not be null.");
 
-        final List objects = Lists.newArrayList(objectsToRead);
-
         final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = makeGetBulkJobSpectraS3Request(bucket, objects, options);
 
         getBulkJobSpectraS3Request.withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.IN_ORDER);
@@ -279,16 +269,14 @@ public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable
     }
 
     @Override
-    public Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead) throws IOException {
+    public Ds3ClientHelpers.Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead) throws IOException {
         return startReadJobUsingRandomAccessBehavior(bucket, objectsToRead, ReadJobOptions.create());
     }
 
     @Override
-    public Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException {
+    public Ds3ClientHelpers.Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objects, final ReadJobOptions options) throws IOException {
         Preconditions.checkNotNull(options, "options may not be null.");
 
-        final List objects = Lists.newArrayList(objectsToRead);
-
         final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = makeGetBulkJobSpectraS3Request(bucket, objects, options);
 
         getBulkJobSpectraS3Request.withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.NONE);
@@ -317,7 +305,7 @@ private Ds3ClientHelpers.Job innerStartReadAllJob(final String bucket, final Rea
         return this.startReadJob(bucket, makeBlobList(bucket), options);
     }
 
-    private final Iterable makeBlobList(final String bucket) throws IOException {
+    private Iterable makeBlobList(final String bucket) throws IOException {
         final Iterable contentsList = listObjects(bucket);
 
         return toDs3Iterable(contentsList, FolderNameFilter.filter());
@@ -360,8 +348,7 @@ public Ds3ClientHelpers.Job recoverWriteJob(final UUID jobId) throws IOException
     /**
      * Verifies that the specified job is active. If the job is not active, then a
      * JobRecoveryNotActiveException is thrown.
-     * @throws IOException
-     * @throws JobRecoveryNotActiveException
+     * @throws JobRecoveryNotActiveException This is thrown when the job is no longer active when a recovery is attempted
      */
     private void innerVerifyJobActive(final UUID jobId) throws IOException, JobRecoveryNotActiveException {
         try {
@@ -419,7 +406,7 @@ public Ds3ClientHelpers.Job recoverReadJob(final UUID jobId) throws IOException,
 
         final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder()
                 .withMasterObjectList(masterObjectList)
-                .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of()));
+                .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of()));
 
         return new ReadJobImpl(transferStrategyBuilder);
     }
@@ -431,28 +418,28 @@ private MasterObjectList masterObjectListForGetJob(final UUID jobId) throws IOEx
     }
 
     @Override
-    public Job recoverReadJobsingStreamedBehavior(final UUID jobId) throws IOException, JobRecoveryException {
+    public Ds3ClientHelpers.Job recoverReadJobsingStreamedBehavior(final UUID jobId) throws IOException, JobRecoveryException {
         innerVerifyJobActive(jobId);
 
         final MasterObjectList masterObjectList = masterObjectListForGetJob(jobId);
 
         final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder()
                 .withMasterObjectList(masterObjectList)
-                .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of()))
+                .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of()))
                 .usingStreamedTransferBehavior();
 
         return new ReadJobImpl(transferStrategyBuilder);
     }
 
     @Override
-    public Job recoverReadJobUsingRandomAccessBehavior(final UUID jobId) throws IOException, JobRecoveryException {
+    public Ds3ClientHelpers.Job recoverReadJobUsingRandomAccessBehavior(final UUID jobId) throws IOException, JobRecoveryException {
         innerVerifyJobActive(jobId);
 
         final MasterObjectList masterObjectList = masterObjectListForGetJob(jobId);
 
         final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder()
                 .withMasterObjectList(masterObjectList)
-                .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of()))
+                .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of()))
                 .usingRandomAccessTransferBehavior();
 
         return new ReadJobImpl(transferStrategyBuilder);
@@ -512,20 +499,12 @@ public Iterable listObjects(final String bucket, final String keyPrefi
     @Override
     public Iterable listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys, final int retries) {
 
-        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries, GetBucketKeyLoaderFactory.contentsFunction));
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory<>(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries, GetBucketKeyLoaderFactory.contentsFunction));
     }
 
     @Override
     public Iterable listObjectsForDirectory(final Path directory) throws IOException {
-        final ImmutableList.Builder objects = ImmutableList.builder();
-        Files.walkFileTree(directory, new SimpleFileVisitor() {
-            @Override
-            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
-                objects.add(new Ds3Object(directory.relativize(file).toString().replace("\\", "/"), Files.size(file)));
-                return FileVisitResult.CONTINUE;
-            }
-        });
-        return objects.build();
+        return FileTreeWalker.INSTANCE.walk(directory);
     }
 
     @Override
@@ -543,31 +522,26 @@ public Iterable remoteListDirectory(final String bucket, final St
     }
 
     public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException {
-        return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES, GetBucketKeyLoaderFactory.getFileSystemKeysFunction));
+        return new LazyIterable<>(new GetBucketKeyLoaderFactory<>(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES, GetBucketKeyLoaderFactory.getFileSystemKeysFunction));
+    }
+
+    @Override
+    public void deleteBucket(final String bucket) throws IOException {
+        DeleteBucket.INSTANCE.deleteBucket(this, bucket);
     }
 
+    @Override
     public Iterable addPrefixToDs3ObjectsList(final Iterable objectsList, final String prefix) {
-        final FluentIterable objectIterable = FluentIterable.from(objectsList);
+        final Stream objectIterable = StreamSupport.stream(objectsList.spliterator(), false);
 
-        return objectIterable.transform(new Function() {
-            @Nullable
-            @Override
-            public Ds3Object apply(@Nullable final Ds3Object object) {
-                return new Ds3Object( prefix + object.getName(), object.getSize());
-            }
-        });
+        return StreamWrapper.wrapStream(objectIterable.map(obj -> new Ds3Object(prefix + obj.getName(), obj.getSize())));
     }
 
+    @Override
     public Iterable removePrefixFromDs3ObjectsList(final Iterable objectsList, final String prefix) {
-        final FluentIterable objectIterable = FluentIterable.from(objectsList);
+        final Stream objectIterable = StreamSupport.stream(objectsList.spliterator(), false);
 
-        return objectIterable.transform(new Function() {
-            @Nullable
-            @Override
-            public Ds3Object apply(@Nullable final Ds3Object object) {
-                return new Ds3Object(stripLeadingPath(object.getName(), prefix), object.getSize());
-            }
-        });
+        return StreamWrapper.wrapStream(objectIterable.map(obj -> new Ds3Object(stripLeadingPath(obj.getName(), prefix), obj.getSize())));
     }
 
     /**
@@ -613,6 +587,11 @@ public void createFolder(final String bucketName, final String folderName) throw
         client.putObject(folderRequest);
     }
 
+    @Override
+    public Ds3Client getClient() {
+        return client;
+    }
+
     /**
      * Ensures that a folder names ends with a trailing forward slash. This prevents the
      * accidental creation of zero-length files when attempting to create a folder.
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java
index ab41b2127..4d7ee0386 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java
@@ -19,7 +19,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMultimap;
-import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.models.BulkObject;
 import com.spectralogic.ds3client.models.JobNode;
 import com.spectralogic.ds3client.models.Objects;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java
index a8f6ff9c3..9501f2220 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java
@@ -25,8 +25,6 @@
 import com.spectralogic.ds3client.models.BulkObject;
 import com.spectralogic.ds3client.models.MasterObjectList;
 import com.spectralogic.ds3client.models.Objects;
-import org.slf4j.LoggerFactory;
-import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
 import java.io.IOException;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java
index 89f6776ff..6296dd04a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java
@@ -78,10 +78,9 @@ static ImmutableList dedupRanges(final ImmutableList rangesForBlob
 
         Range currentRange = null;
         Range nextRange;
-        final UnmodifiableIterator rangeIterator = sortedRanges.iterator();
 
-        while (rangeIterator.hasNext()) {
-            nextRange = rangeIterator.next();
+        for (final Range sortedRange : sortedRanges) {
+            nextRange = sortedRange;
             if (currentRange == null) {
                 // This will only be called on the first iteration of the loop
                 currentRange = nextRange;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java
index 9a34369f4..e6f5d6565 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java
@@ -21,11 +21,15 @@
 import com.spectralogic.ds3client.models.Priority;
 import com.spectralogic.ds3client.models.JobChunkClientProcessingOrderGuarantee;
 import com.spectralogic.ds3client.models.WriteOptimization;
+import com.spectralogic.ds3client.utils.collections.StreamWrapper;
+
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 @JacksonXmlRootElement(localName = "Objects")
 public class Ds3ObjectList {
     @JsonProperty("Object")
-    private Iterable objects;
+    private Stream objects;
 
     @JacksonXmlProperty(isAttribute = true, namespace = "", localName = "Priority")
     private Priority priority;
@@ -40,15 +44,15 @@ public Ds3ObjectList() {
     }
 
     public Ds3ObjectList(final Iterable objects) {
-        this.objects = objects;
+        this.objects = StreamSupport.stream(objects.spliterator(), false);
     }
 
     public Iterable getObjects() {
-        return objects;
+        return StreamWrapper.wrapStream(this.objects);
     }
 
     public void setObjects(final Iterable objects) {
-        this.objects = objects;
+        this.objects = StreamSupport.stream(objects.spliterator(), false);
     }
 
     public Priority getPriority() {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java
index 346973197..ab2f78f90 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java
@@ -22,6 +22,7 @@
 import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
 import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule;
 import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
 import com.spectralogic.ds3client.models.bulk.Ds3ObjectList;
 
 import java.io.IOException;
@@ -42,6 +43,8 @@ public final class XmlOutput {
         module = new JacksonXmlModule();
         module.setDefaultUseWrapper(false);
         mapper = new XmlMapper(module);
+
+        mapper.registerModule(new Jdk8Module());
         final SimpleFilterProvider filterProvider = new SimpleFilterProvider().setFailOnUnknownId(false);
         mapper.setFilterProvider(filterProvider);
         if (isProductionBuild()) {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/JobUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/JobUtils.java
index 40655806a..3c3d6f19b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/JobUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/JobUtils.java
@@ -24,7 +24,6 @@
 import com.spectralogic.ds3client.models.*;
 
 import java.io.IOException;
-import java.security.SignatureException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
diff --git a/ds3-interfaces/build.gradle b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/StreamWrapper.java
similarity index 54%
rename from ds3-interfaces/build.gradle
rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/StreamWrapper.java
index ac8ff43ef..8fdb7adf1 100644
--- a/ds3-interfaces/build.gradle
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/StreamWrapper.java
@@ -13,30 +13,31 @@
  * ****************************************************************************
  */
 
-buildscript {
-    repositories { jcenter() }
-    dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
-    }
-}
+package com.spectralogic.ds3client.utils.collections;
 
-apply plugin: 'com.github.johnrengelman.shadow'
+import org.jetbrains.annotations.NotNull;
 
-dependencies {
-    compile 'commons-io:commons-io:2.4'
-}
+import java.util.Iterator;
+import java.util.stream.Stream;
+
+/**
+ * Wraps a Stream as an Iterable
+ */
+public class StreamWrapper implements Iterable {
 
-shadowJar {
-    relocate 'org.apache', 'ds3fatjar.org.apache'
-    dependencies {
-        exclude(dependency('org.hamcrest:hamcrest-library:1.3'))
-        exclude(dependency('org.mockito:mockito-core:1.10.19'))
-        exclude(dependency('junit:junit:4.12'))
-        exclude(dependency('org.slf4j:slf4j-api:1.7.22'))
-        exclude(dependency('org.slf4j:slf4j-simple:1.7.22'))
+    private final Stream stream;
+
+    public static  Iterable wrapStream(final Stream stream) {
+        return new StreamWrapper<>(stream);
     }
-}
 
-artifacts {
-    archives shadowJar
+    private StreamWrapper(final Stream stream) {
+        this.stream = stream;
+    }
+
+    @NotNull
+    @Override
+    public Iterator iterator() {
+        return stream.iterator();
+    }
 }
diff --git a/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/DeleteBucket.kt b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/DeleteBucket.kt
new file mode 100644
index 000000000..8d03ff69e
--- /dev/null
+++ b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/DeleteBucket.kt
@@ -0,0 +1,38 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers
+
+import com.spectralogic.ds3client.commands.DeleteBucketRequest
+import com.spectralogic.ds3client.commands.DeleteObjectsRequest
+import com.spectralogic.ds3client.utils.collections.asIterable
+import com.spectralogic.ds3client.utils.collections.take
+
+object DeleteBucket {
+    fun deleteBucket(helpers: Ds3ClientHelpers, bucket: String) {
+        deleteBucketContents(helpers, bucket)
+        helpers.client.deleteBucket(DeleteBucketRequest(bucket))
+    }
+
+    fun deleteBucketContents(helpers: Ds3ClientHelpers, bucket: String) {
+        val client = helpers.client
+        val objIterator = helpers.listObjects(bucket).iterator()
+
+        while(objIterator.hasNext()) {
+            val subIterator = objIterator.take(1_000)
+            client.deleteObjects(DeleteObjectsRequest(bucket, subIterator.asIterable()))
+        }
+    }
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker.kt b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker.kt
new file mode 100644
index 000000000..18b1d4270
--- /dev/null
+++ b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker.kt
@@ -0,0 +1,35 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers
+
+import com.spectralogic.ds3client.models.bulk.Ds3Object
+import com.spectralogic.ds3client.utils.collections.StreamWrapper
+import java.nio.file.Files
+import java.nio.file.Path
+
+object FileTreeWalker {
+    /**
+     * Walks a file system starting at {@param startDir} and returns a Lazy Iterable with all the
+     * files represented as Ds3Objects.
+     */
+    fun walk(startDir: Path) : Iterable {
+        return StreamWrapper.wrapStream(Files.walk(startDir).map {
+            Ds3Object(startDir.relativize(it).toString().replace("\\", "/"), Files.size(it))
+        }.filter {
+            !it.name.isNullOrEmpty()
+        })
+    }
+}
diff --git a/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/IterableExtensions.kt b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/IterableExtensions.kt
new file mode 100644
index 000000000..223dc15d7
--- /dev/null
+++ b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/IterableExtensions.kt
@@ -0,0 +1,24 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.utils.collections
+
+fun  Iterator.asIterable(): Iterable {
+    return IterableWrapper(this)
+}
+
+private class IterableWrapper constructor(private val iterator: Iterator): Iterable {
+    override fun iterator(): Iterator = iterator
+}
diff --git a/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator.kt b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator.kt
new file mode 100644
index 000000000..5b570c89e
--- /dev/null
+++ b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator.kt
@@ -0,0 +1,43 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.utils.collections
+
+class WindowedIterator constructor(private val iterator: Iterator, private val windowSize: Int) : Iterator by iterator {
+
+    init {
+        if (windowSize <= 0) throw IllegalArgumentException("windowSize must be larger than 0")
+    }
+
+    private var count = 0
+
+    override fun next(): T {
+        if (!hasNext()) throw NoSuchElementException("There are no new items to return")
+
+        count++
+        return iterator.next()
+    }
+
+    override fun hasNext(): Boolean {
+        return if (count >= windowSize) {
+            false
+        } else {
+            iterator.hasNext()
+        }
+    }
+}
+
+fun  Iterator.take(n: Int): Iterator = WindowedIterator(this, n)
+
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/ConnectionFixture.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/ConnectionFixture.java
index 389a30d3f..6f1dfc6c2 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/ConnectionFixture.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/ConnectionFixture.java
@@ -18,7 +18,7 @@
 import com.spectralogic.ds3client.models.common.Credentials;
 import com.spectralogic.ds3client.networking.ConnectionDetails;
 
-public class ConnectionFixture {
+public final class ConnectionFixture {
 
     public static ConnectionDetails getConnection() {
         return getConnection(8080);
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 6f70fc9af..f9c034f42 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -19,7 +19,6 @@
 import com.spectralogic.ds3client.commands.*;
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException;
-import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash;
 import com.spectralogic.ds3client.models.*;
 import com.spectralogic.ds3client.models.Objects;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
@@ -51,6 +50,7 @@
 
 import static org.hamcrest.CoreMatchers.*;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class Ds3Client_Test {
     private static final UUID MASTER_OBJECT_LIST_JOB_ID = UUID.fromString("1a85e743-ec8f-4789-afec-97e587a26936");
@@ -517,24 +517,12 @@ public void getObjectWithMetaData() throws IOException {
 
     @Test
     public void createPutJobSpectraS3() throws IOException {
-        this.runBulkTest(BulkCommand.PUT, new BulkTestDriver() {
-            @Override
-            public MasterObjectList performRestCall(final Ds3Client client, final String bucket, final List objects)
-                    throws IOException {
-                return client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucket, objects)).getMasterObjectList();
-            }
-        });
+        this.runBulkTest(BulkCommand.PUT, (client, bucket, objects) -> client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucket, objects)).getMasterObjectList());
     }
 
     @Test
     public void createGetJobSpectraS3() throws IOException {
-        this.runBulkTest(BulkCommand.GET, new BulkTestDriver() {
-            @Override
-            public MasterObjectList performRestCall(final Ds3Client client, final String bucket, final List objects)
-                    throws IOException {
-                return client.getBulkJobSpectraS3(new GetBulkJobSpectraS3Request(bucket, objects)).getMasterObjectList();
-            }
-        });
+        this.runBulkTest(BulkCommand.GET, (client, bucket, objects) -> client.getBulkJobSpectraS3(new GetBulkJobSpectraS3Request(bucket, objects)).getMasterObjectList());
     }
     
     private interface BulkTestDriver {
@@ -542,7 +530,7 @@ MasterObjectList performRestCall(final Ds3Client client, final String bucket, fi
                 throws IOException;
     }
     
-    private void runBulkTest(final BulkCommand command, final BulkTestDriver driver) throws IOException {
+    private void runBulkTest(final BulkCommand command, final Ds3Client_Test.BulkTestDriver driver) throws IOException {
         final List objects = Arrays.asList(
             new Ds3Object("file1", 256),
             new Ds3Object("file2", 1202),
@@ -939,7 +927,7 @@ public void testGettingDefaultUserAgent() {
         final String userAgent = newClient.getConnectionDetails().getUserAgent();
         final String[] userAgentFields = userAgent.split("-");
 
-        assertThat(userAgentFields.length >= 2, is(true));
+        assertThat(userAgentFields.length, is(greaterThanOrEqualTo(2)));
 
         // look for a pattern like 3.4.0, but leave open the possibility of a string like 3.4.0-SNAPSHOT
         final Pattern matchPattern = Pattern.compile("\\d+\\.\\d+\\.\\d+");
@@ -1153,7 +1141,7 @@ public void getObjectVerifyFullPayload() throws IOException {
     @Test
     public void clearSuspectBlobAzureTargetsSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_azure_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_azure_target", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .clearSuspectBlobAzureTargetsSpectraS3(new ClearSuspectBlobAzureTargetsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
@@ -1162,7 +1150,7 @@ public void clearSuspectBlobAzureTargetsSpectraS3Test() throws IOException {
     @Test
     public void clearSuspectBlobPoolsSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_pool", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_pool", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .clearSuspectBlobPoolsSpectraS3(new ClearSuspectBlobPoolsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
@@ -1171,7 +1159,7 @@ public void clearSuspectBlobPoolsSpectraS3Test() throws IOException {
     @Test
     public void clearSuspectBlobS3TargetsSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_s3_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_s3_target", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .clearSuspectBlobS3TargetsSpectraS3(new ClearSuspectBlobS3TargetsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
@@ -1180,7 +1168,7 @@ public void clearSuspectBlobS3TargetsSpectraS3Test() throws IOException {
     @Test
     public void clearSuspectBlobTapesSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_tape", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_tape", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .clearSuspectBlobTapesSpectraS3(new ClearSuspectBlobTapesSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
@@ -1189,7 +1177,7 @@ public void clearSuspectBlobTapesSpectraS3Test() throws IOException {
     @Test
     public void markSuspectBlobAzureTargetsAsDegradedSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_azure_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_azure_target", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .markSuspectBlobAzureTargetsAsDegradedSpectraS3(new MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
@@ -1198,7 +1186,7 @@ public void markSuspectBlobAzureTargetsAsDegradedSpectraS3Test() throws IOExcept
     @Test
     public void markSuspectBlobDs3TargetsAsDegradedSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_ds3_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_ds3_target", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .markSuspectBlobDs3TargetsAsDegradedSpectraS3(new MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
@@ -1207,7 +1195,7 @@ public void markSuspectBlobDs3TargetsAsDegradedSpectraS3Test() throws IOExceptio
     @Test
     public void markSuspectBlobPoolsAsDegradedSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_pool", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_pool", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .markSuspectBlobPoolsAsDegradedSpectraS3(new MarkSuspectBlobPoolsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
@@ -1216,7 +1204,7 @@ public void markSuspectBlobPoolsAsDegradedSpectraS3Test() throws IOException {
     @Test
     public void markSuspectBlobS3TargetsAsDegradedSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_s3_target", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_s3_target", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .markSuspectBlobS3TargetsAsDegradedSpectraS3(new MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
@@ -1225,7 +1213,7 @@ public void markSuspectBlobS3TargetsAsDegradedSpectraS3Test() throws IOException
     @Test
     public void markSuspectBlobTapesAsDegradedSpectraS3Test() throws IOException {
         MockNetwork
-                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_tape", new HashMap(), IDS_REQUEST_PAYLOAD)
+                .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_tape", new HashMap<>(), IDS_REQUEST_PAYLOAD)
                 .returning(204, "")
                 .asClient()
                 .markSuspectBlobTapesAsDegradedSpectraS3(new MarkSuspectBlobTapesAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST));
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockNetwork.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockNetwork.java
index 35e73bba3..ec6e384da 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockNetwork.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockNetwork.java
@@ -92,7 +92,7 @@ public MockNetwork returning(
     public MockNetwork returning(
             final int statusCode,
             final String responseContent) {
-        return returning(statusCode, responseContent, new HashMap());
+        return returning(statusCode, responseContent, new HashMap<>());
     }
     
     public Ds3Client asClient() {
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/SeekableByteChannelInputStream_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/SeekableByteChannelInputStream_Test.java
index 1eadaa633..9ed9e4e43 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/SeekableByteChannelInputStream_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/SeekableByteChannelInputStream_Test.java
@@ -100,7 +100,7 @@ private void readAndVerify(final InputStream inputStream, final byte[] expectedB
         final byte[] resultBuffer = new byte[expectedBytes.length];
         final byte[] buffer = new byte[10];
         int position = 0;
-        int bytesRead = 0;
+        int bytesRead;
         while (0 < (bytesRead = inputStream.read(buffer, 5, 5))) {
             for (int i = 0; i < bytesRead; i++) {
                 resultBuffer[position] = buffer[5 + i];
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
index 28071aec8..eafa594f0 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
@@ -47,7 +47,7 @@ public void getRequestId_WithoutRequestId_Test() {
 
     @Test
     public void getRequestId_EmptyHeaders_Test() {
-        final String result = ResponseParserUtils.getRequestId(new MockedHeaders(ImmutableMap.of()));
+        final String result = ResponseParserUtils.getRequestId(new MockedHeaders(ImmutableMap.of()));
         assertThat(result, is(nullValue()));
     }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
index abe990803..3181f7302 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java
@@ -109,7 +109,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException {
     public void testReadObjectsWithFailedGet() throws IOException, ParseException {
         final Ds3Client ds3Client = mock(Ds3Client.class);
 
-        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
+        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
 
         final GetBulkJobSpectraS3Response buildBulkGetResponse = buildBulkGetResponse();
         Mockito.when(ds3Client.getBulkJobSpectraS3(hasChunkOrdering(JobChunkClientProcessingOrderGuarantee.NONE))).thenReturn(buildBulkGetResponse);
@@ -204,7 +204,7 @@ public void testWriteObjectsWithFailedPut() throws IOException, ParseException {
         final ConnectionDetails details = mock(ConnectionDetails.class);
         Mockito.when(details.getEndpoint()).thenReturn("localhost");
 
-        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
+        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
         Mockito.when(ds3Client.getConnectionDetails()).thenReturn(details);
 
         final PutBulkJobSpectraS3Response buildBulkPutResponse = buildBulkPutResponse();
@@ -424,7 +424,7 @@ private static Ds3Client buildDs3ClientForBulk() throws IOException,
             .thenReturn(jobChunksResponse2)
             .thenReturn(jobChunksResponse3);
 
-        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
+        Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client);
         Mockito.when(ds3Client.getConnectionDetails()).thenReturn(details);
         return ds3Client;
     }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java
index 695666d4c..2e449cd00 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java
@@ -34,7 +34,7 @@
 
 import static org.mockito.Matchers.argThat;
 
-public class RequestMatchers {
+public final class RequestMatchers {
     public static GetBulkJobSpectraS3Request hasChunkOrdering(final JobChunkClientProcessingOrderGuarantee chunkOrdering) {
         return argThat(new TypeSafeMatcher() {
             @Override
@@ -225,7 +225,7 @@ public boolean matches(final Object argument) {
                         && (marker == null
                             ? null == getBucketRequest.getMarker()
                             : marker.equals(getBucketRequest.getMarker()));
-                
+
             }
         });
     }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/StreamObjectPutter.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/StreamObjectPutter.java
index 84e22db71..eb242aae6 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/StreamObjectPutter.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/StreamObjectPutter.java
@@ -27,7 +27,7 @@ public StreamObjectPutter(final byte[] buf) {
     }
 
     @Override
-    public InputStream buildInputStream(String key) {
+    public InputStream buildInputStream(final String key) {
         return new ByteArrayInputStream(buf);
     }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java
index 229b6e4f1..1dfbbdbde 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java
@@ -167,21 +167,21 @@ public void escapeSpacesInQueryParam() throws MalformedURLException {
     @Test
     public void getPortBack() throws MalformedURLException {
         final URL url = new URL("http://localhost:8080/path");
-        int port = NetUtils.getPort(url);
+        final int port = NetUtils.getPort(url);
         assertTrue(port == 8080);
     }
 
     @Test
     public void getHttpsDefaultPort() throws MalformedURLException {
         final URL url = new URL("https://localhost/path");
-        int port = NetUtils.getPort(url);
+        final int port = NetUtils.getPort(url);
         assertTrue(port == 443);
     }
 
     @Test
     public void getHttpDefaultPort() throws MalformedURLException {
         final URL url = new URL("http://localhost/path");
-        int port = NetUtils.getPort(url);
+        final int port = NetUtils.getPort(url);
         assertTrue(port == 80);
     }
 
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/Signature_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/Signature_Test.java
index 3052ef645..6e3ff31d0 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/Signature_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/Signature_Test.java
@@ -17,7 +17,6 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Multimap;
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.models.common.Credentials;
 import com.spectralogic.ds3client.models.common.SignatureDetails;
@@ -32,7 +31,6 @@ public class Signature_Test {
 
     /**
      * Example taken from the AWS S3 documentation site: http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader
-     * @throws SignatureException
      */
     @Test
     public void getSignature() throws SignatureException {
@@ -47,7 +45,6 @@ public void getSignature() throws SignatureException {
 
     /**
      * Example taken from the AWS S3 documentation site: http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader
-     * @throws SignatureException
      */
     @Test
     public void putSignature() throws SignatureException {
diff --git a/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker_Test.kt b/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker_Test.kt
new file mode 100644
index 000000000..ac1d0d7c6
--- /dev/null
+++ b/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker_Test.kt
@@ -0,0 +1,50 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers
+
+import com.google.common.collect.ImmutableList
+import com.spectralogic.ds3client.models.bulk.Ds3Object
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+
+import org.hamcrest.CoreMatchers.*
+import org.hamcrest.MatcherAssert.assertThat
+
+class FileTreeWalker_Test {
+
+    @get:Rule
+    val tempDir = TemporaryFolder()
+
+    @Test
+    fun basicFileList() {
+        tempDir.newFile("bar")
+        tempDir.newFile("baz")
+
+        val walk = FileTreeWalker.walk(tempDir.root.toPath())
+
+        val listBuilder = ImmutableList.Builder()
+
+        walk.forEach {
+            listBuilder.add(it)
+        }
+
+        val fileList = listBuilder.build()
+
+        assertThat(fileList.size, `is`(2))
+
+    }
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator_Test.kt b/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator_Test.kt
new file mode 100644
index 000000000..3a0951ac0
--- /dev/null
+++ b/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator_Test.kt
@@ -0,0 +1,76 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.utils.collections
+
+import com.google.common.collect.Lists
+import org.junit.Test
+
+import org.hamcrest.CoreMatchers.*
+import org.hamcrest.MatcherAssert.assertThat
+
+class WindowedIterator_Test {
+    @Test
+    fun iterateWholeCollection() {
+        val list = Lists.newArrayList("value", "hi", "anothervalue")
+
+        val iterator = list.iterator().take(Int.MAX_VALUE)
+
+        var count = 0
+        iterator.forEach {
+            count++
+        }
+
+        assertThat(count, `is`(3))
+    }
+
+    @Test
+    fun iteratePart() {
+        val list = Lists.newArrayList("value", "hi", "anothervalue")
+
+        val iterator = list.iterator().take(1)
+
+        var count = 0
+        iterator.forEach {
+            count++
+        }
+
+        assertThat(count, `is`(1))
+    }
+
+    @Test
+    fun iterateAfterFirstIteration() {
+        val list = Lists.newArrayList("value", "hi", "anothervalue")
+
+        val parentIterator = list.iterator()
+        val firstIterator = parentIterator.take(1)
+
+        var firstCount = 0
+        firstIterator.forEach {
+            firstCount++
+        }
+
+        assertThat(firstCount, `is`(1))
+
+        val secondIterator = parentIterator.take(Int.MAX_VALUE)
+
+        var secondCount = 0
+        secondIterator.forEach {
+            secondCount++
+        }
+
+        assertThat(secondCount, `is`(2))
+    }
+}
\ No newline at end of file
diff --git a/ds3-utils/build.gradle b/ds3-utils/build.gradle
index 6b89d6230..f8f974efa 100644
--- a/ds3-utils/build.gradle
+++ b/ds3-utils/build.gradle
@@ -14,34 +14,9 @@
  * ****************************************************************************
  */
 
-buildscript {
-    repositories { jcenter() }
-    dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
-    }
-}
-
-apply plugin: 'com.github.johnrengelman.shadow'
-
-shadowJar {
-    relocate 'org.apache', 'ds3utilfatjar.org.apache'
-    relocate 'com.google', 'ds3utilfatjar.com.google'
-
-    dependencies {
-        exclude(dependency('org.hamcrest:hamcrest-library:1.3'))
-        exclude(dependency('org.mockito:mockito-core:1.10.19'))
-        exclude(dependency('junit:junit:4.12'))
-        exclude(dependency('org.slf4j:slf4j-api:1.7.22'))
-        exclude(dependency('org.slf4j:slf4j-simple:1.7.22'))
-    }
-}
-
-artifacts {
-    archives shadowJar
-}
-
 dependencies {
-    compile     'commons-codec:commons-codec:1.10'
-    compile     'com.google.guava:guava:20.0'
+    compile "commons-codec:commons-codec:$commonscodecVersion"
+    compile "commons-io:commons-io:$commonsioVersion"
+    compile "com.google.guava:guava:$guavaVersion"
 }
 
diff --git a/ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java
similarity index 100%
rename from ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java
rename to ds3-utils/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java
diff --git a/ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java b/ds3-utils/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java
similarity index 100%
rename from ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java
rename to ds3-utils/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java
diff --git a/gradle.properties b/gradle.properties
new file mode 100644
index 000000000..8bee35caa
--- /dev/null
+++ b/gradle.properties
@@ -0,0 +1,11 @@
+commonscodecVersion=1.10
+commonsioVersion=2.4
+commonslangVersion=3.6
+guavaVersion=23.0
+hamcrestVersion=1.3
+httpclientVersion=4.5.3
+jacksonVersion=2.9.0
+jnaVersion=4.2.2
+junitVersion=4.12
+mockitoVersion=1.10.+
+slf4jVersion=1.7.25
\ No newline at end of file
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index 7b61d71aa610f3b12c0d96a3d392c2c47f5a6a78..1454129915e967f2239442034b31dae2c3787c80 100644
GIT binary patch
delta 6637
zcmZ8m2RxPk_rI=@k-hicD=RW9E8_|m5v~;_!lle|@kLg|lf6R7DqCju-XWVPBRdi$
z{!jPb|F5s_=k3HApNOFGyU2
zcoMbv#wwW7CVz9^bH{=F2lSc)4-M|;$J~F2qwf_if|v2p6!2PX5v`}!Ro(+&Wxfuz
zbOmgfIZ@IfM8#9v&uyN6i`0EQN%R~5*M
zy-5wKE6il>bQ4tX9PL<{iJbXLooCH2SG&z<(D;zd&~0C8&pgh0n<=HXAEFd$3cHNo
zW;jRQu`&NCP~Y|F+EvOLF=oZdVSa&6rZAzAuW
z0bcb%a=CRzqHlA(r6>?qdwQAzJ0Y7-13wQ8RNxC;>=VuM5@-}R_JHo&cbCSBk1gzz
z&ys{3Isig++mZn`N-8y8wYCH?Eb$EPs}zizmQ!%jy;0fae7&sf*d>k-^+IH{NN7j(
zM7k}iE9#_L_(g~xS;X!d%NaDkmO#)wd
zJ
z{h`yvDMrylrxo8@D^cMxKb=^T#3RCL)qSOPP?h`Lqa>}ZhZ0;?-QgTCr^gH@VUd;c
z+)xV(PG(htDWrRRHbI}i=0Q5MLQ4>{R|z#w(^9L%<`DdgGVDcPbT4I6YZbCK{|>jE
zsB?8ff7qo7+Xh+-_Pi3ZV_CQ4VjTn(zD3ZpqC2sACj{rKra3j@6EPtf1j51sfp9_q
zgLYa#k(m%BX6J0<;gN1eWKHwY=QN|TUW(}%naPb(>s{Ms!t~BgYm$FrgCT)D!F1sw
z3^UwYH*T8_bXQaa%uYO89+rnuuPUdF>7`X}TW*HZ66tN`i8_9mXiTlvim8O9R=cER
ztjF^Go}W=I8rOJ4yf&b4y7Bw$$Jvh3-m&YGuTl_CRCpu-)jJz&2J!LIAC6OxH7<+y
zt<=3~sdM&kYi9K}ZgahVB#H|=dVp(l
z5*@hEvD5QqA#s3x?I({lI(oi5L(z&yexk4g!fdt?bgGS
zs=$(?R20WpX6)HMWr2U3a)RsuWT@#YBB8(l^@ejofJ8r
z!Y-eQrMh;nsJyy%R}Zc;rDSh)vwZD
zQwuvb^V~0&iPgSA2B9D7>id|l}tq^k=S2v5Y)>YPc`XXL$#xZdB
z^(q3fHgHr?z0994vhW@6t7exT3nOz_2isQ_d1ZHts~3sKzSrwb*RY^kwe2_WuWJTQ
zH|@2do?hr=iy+NlTpWwm2-NRQPc9R*Uw*zZ@Nmf6gv;K`>Ra)#&&&p!VdO77>1fU3
zE2(@J(`&?T&k6P=uSXvvMw364w|pU*(c!n$e2JX*xT~m;&ZeFBWG1#%L6}J8Gs4~Q
zO+14rcYee06D|IAc8WD8%Z}psgyOWRE1VQ-UMT9y1{;C(N$%uYtr-P-Sl$?7R!~^R
z#z*VQ;7x*)DIUv4M;X8|yS~tVVZhUB=dw$(d;0ok-Oh?WgO{_2Iic_aHN>;#l9DS|
zqPM(aVn|nV`R2CUdR~<9+&AIkgmH!@08~Om3S0c?4Vyo|G+aDt{vzmy5R0oVd+9N}
z_W(6iW+=_KoZI_~EVe`$sBWBRR3O(O@RgKO5~VbJR2_3Dms~4hK8@&V991}dzSVzJ
z5oL)3|Lvk7#MvDa8*zYNGyS>HQ>y8z&d4M+BT9MsLrArZ<`xd|7l#S)iU^;jNw_=z
ztJm7@jkVn>YcI;39G#}we#Ajj`xNUdX8lmJ{)W^gl_6}~6{9@`-|lS{Go-~|?zzMW
z?~uS1v6NfWtrsB!Os3~6{F;*ukD3~*pe{o8PRmhSt4R$uGntRpYe{(@W+F-s|4x1xL^mH@$fvB>y`d`7a);WVNSAjd?$QZ6X0-zbw7Qx3?K*F|N`e!^;mtIrGq(+bI9K_c`8l0j
z9=JSz>+O_~5aGcon=YIr<5Ksq{9`It#sAnm{rtfGtM&>w*6$1hI=WPPXmTDjB@01
zh=AM9(ooxW)8yxV7%)sqULy4#p+QciQ8wnu3HyLLUy?m9v^(_&
z71vp9_j9}lX#n>`IF}8vxo$5-QPq}HuDZd4v;ae3Wp5nNt)vmMXE;pi9TK%JH_X59-iU-9{BIHp56+B$@>bf
z551xKbw~c|JAdi)NM6yz
z_$Aq5LUwd{QB7?1Z`H%fc&)wJNmF++(HyvB|LD)|6Trlj&Cz|ivo)*YVHfd$Q$KQc5{nJ6>tiw_Y1%rk22aF
zax)@W5>M)LT6150`(B-aViT-&E8~fOcjp+Ui?IHgXhpG}(=z(aN=RU_K*7HByK^Rw
zP)mD(nK){$-&>L{Wbd2sRhIZkPYqsJsWx0MCtMM4l2li)`besXD(}@P
zuu`1-tUR72pww7#a*C=KPh{iuvPoVP?vD8Qc_o76p#GvWV&L$sZ8FG~v8Z+Y%o;zV
zLfBh7x7b^^_)O2~z;(2C4Z*>)U9z}PL2H3nKjvhes#)w?NT@WL&U$>ex+I!)rQT1A
zH^$eq=I||@YItvM&5e80-&aK5JCo2V8rzWg2svMawo#N)wi2Myy6_L-kitt{WpGJP
z%7mMEhi%{DUpD!l_SHb&6F7>0lAOVh?xa);SV*b%92u!4+zX=Gf6Y~F%N(NjeW49u
z9!;?jxdVS6T4WFr{8h`?xR5{ISeQ2Ei?L!4?Qv@Tz=O!w_nqE{eytvKBCJ%0exz{T
zco#}RU&-5;npHN3(ta=_;HvNysW4%Cf$Lre-=Jh=ScI#FenCN$Di^XA)~
z5w1Wto^1OvPO5HOyPIwU(VdA0Y?je3Q+8vUB_&hl^h6p5B9}XUj2S6K)^1Fub(FBH
z?MH@%*4#nPkUFEDJ*^S7{naP>Xh7yxapNP=s;}lVhV^tcL*osrjFmM{pyQt=L_9N0
zfW!IQgTXFbA}U)0o$(2EH?u{92g49^7J&M%{RRJ`rK~e~?v~VH?iP_cdDrX`smSfs
zva0;7Z*_bhxk<&}zbrizLge%z+CGlSuKx6*gf%O%e1E=#Qm#X(Advw_b(wog-7gPp
z*?%6@u>Zr*EI>)sMyb0f0JuI6_l)Y6*TW-~;^6is=HHbrP>u$U979)DwSJo!zK@V=
z3XnRY8eHm=d-`_q+HJ}%S-BsuG}^}`VH|8Ud?K``vN_S!4M8=FPZ@?4_G}vBO~+sF
z5&JW^avQNZpn~3~3nEk>#2D=0J2*r&>@+JF=sZiTt?@D>&eDfa`uR|lCE^;yzwW5t
zV+(qq&=m1w=}EAW{+hRn@>P_`HQmIWUy2zq@O-?b1m<_#^9dbdEh~DbvmKdpC$*_6
zespC&?vD&`cDOb17*sE+O5>~3vh(q>|MH<7`n)j|SdBVVpOqRtzJiA!T$s_bG&@iv
zg@^dDx6M&D?GPMx(IYt?qv$Tom=^gS4`j#4iHWn3ZMm)I`N(&-$;(UkE^ghmF%`OV
zA%fBCWfJnjt_Uk!ecNxB(pZt$#Py2^dtL1k=OJ7tO!y}9?6(;U(>5Xb9i-gjS3f_<
zrF1Ob&^$sVv9Oc7vAlI@)sL!B3e>w6^hrd+Hz@WhPxtsI??-+|YIHd~3_ke$Z{uY6
z%x=b)X+X27d~PlhbS{4)N}kKN%8QlXlk0phVN%y++~aNZU`pTgLwLydStD5zEpiE2
zjj@ifihZ@|Wwj~l_QZ!HxKSs*hL59;-tc`lIT7%~K@Apn#eDK1*ixKJN%ac~JVl=1
z`AMvd-z?`RnTD(XP|mgJu}ktrMbusw>0Jo=ys>L87x!tB)*p9c*G+EF^VKq+i@k|!
zzh>IHe`y(iq*Qusm=&+`pqR}Ybyk8Md$oB@`25i52~OU@2i~%IV`+!P>)bc(E0Q!3Wv|VXPNH)4pU~FT7f5pL61_gC7pkiTibr-DDGr%
zJF9d8JInWlH9ZKv)KIlLPOz-s0w~Xdr?rz9{a_C?c!N~C3#G)hC&kO@U!^I%8$^V?
z$TUh(2{C&l@w!#oiG}IWG0O5r>yg#@@5wPe@wqH;XweH^5`RA@_1#_Xy4!nr=sJ2l
zw0E(0$9!Hsde%Z5bd^SZNBF~iTD87I_2J<qzV(t(
ze6)Asif)n%>Z^u9ky?eswfiFU#fc?arUL0pg2|*nU=A(d_J*Wl!0|P9aK!JvhJnwd#o=P8w+_2lwFYmlV%7S%
z7GilTacqPb+|8m>EN`-A2TR#Bu;HRTPII;x%Ey5~@^B##IbhF|6`%kvRH%7$W4-IQ
zTfmBJR<2>G_;NUw`u$iQ>x@D`J{88CSI!tVq{x?0>{E(I!&rTUsY4hFAZvx;vHCOW
z09LJ6xbVR|Um%?D8Fi%w%NFzu{EY*pf&87o0d=iX{~N&p-&^I;)&QwCF$`hfCW9d|
z+LSOvX(tnK+(r+4>g2;PNV=GyCIpxPOBV~rpP2ud!;ymty9y$tL92h4`~~bJh7$-M
zxybPc{AW`0cTn>e3W1~&{r}hoSn8regU=tse}k9k{~t_t{!juf?KEgvyUFboBv^hF
z7aZYkjAqbiQFRpW7UQrNy31h{&lL<;L2L7f4{JF@0mz|iH5IkT{VFh!xDF}gp
z+Wxh;Ww2lZ;8HiHHUNt*W}F*n2i}Tb{JR7B4kjq24v^K(1*kAG0M9!xx^0gfh1b9^
zFoDN|?_ZiB5M~$%mge}Alsmrt3vr-}N<+Sio{ckkC85#_p
zbRf!_8d&Sa26v&235sihea4BlxEwNU#RkfP<2nd8c>P@z_JCWD1jiq@{w%Pr(2E><
z$J|`O&+Q&ebD#`+Aojf!5YUf_gg4)kku|tUCK3ol;T%x#9tdOvSOzX}{4u-6
zP5Mm}JkC0xZHaThC1;?_kp)m3<;LXmc|Q}>+Xc-d^CJJBrN2$+zcHV51;GM;Bjeo1
z1by=e;Pl`Iiu$P0)-|dLe<*>Cq!-M^c{fu$(ZEFqe1Lz56AiW^S02p)-4wj=-y*-{
ziv?4O{I?6Jj?k@q1uZ-V1EF|se>_jnLOH|yXzh~Dx2m;4?Ra2A)i?*d7l;NHy7B@R
zeb_8h4l+UMg3)~aK>gH09Bv5f7SeQ!Z6wAkeU_pTV
z9UZ`i?HDXhbY%k|cm@Q^o&$@wIg_
zO+;(t?`6RFFE+vi6;DC)JVv-Myuo26D04c<;|54aSTQ{P0Vb$>CYl#BAcWyHg1pjf
zkjDl58o=~3b8nZV7htNoz~;<%&O`Gm0QHg!_&kI?tI_*0gV*JPX@
zgJnqqrcVBx@ReqO?UCZ2^21nu3}%9~6-XM?L<^FiwcP^W0TA%b3}HVfxY!1;v~vEh
nKnTGGR-pi$zbOS(b2qy%=QOMw-X22mO%
z1^kv>{y#t7bLPx*?mYLocV_O)Iq&=}#&65Vr+lP@Pe2O-k&uAM{4Og-Q*x1<_G3rS
zccP_q=$L-FZn@%t{*jJ-F{A=^0&L$0!}WWD*uLtL0V(OF(~#_`}PJj6IO!`MQELu^V!V3JecJ)e>`K$-Y1c+C-rBIZEYkedFk0
zU({9hBRV!?R`CO|^>`5%P%9R!M>cq_OYBX#Sy>`Z4tMcyGetUFiLRPuw=&^>o+3V0
z5#449?nO0Qn32~x+>fp+cbKKTca+3-cqnmC)Cg@BGHR}Ijc@e$QPVnWhG-Nijc)6OUB{o@
zq1$-+YWqpoGe-HTuhRM%?d4yoeS$~+sEfg_83j==AKX2b`PeheH
zAc-8<5_C?si;4-XeEG-ASW>!rGbD2#5!@YhkZnt*lO`o}&nJ;_vwU%lvQdvk_)32>
zkAkv{F}i0Q#-n#&QmlT1+0;w;<$h%0j~|SoLzQDoMMbo*OBH@5t*EH9VbfO2GEa+v
zM4jwb)boR^kVO3)xhlkcsYt$Nc0PNy%iDpeBqPo
zCVoaC@DsWl7r4~gxX|^A^glYC&KH`wPv3Nvt|poQS0Ye>i)UrSiziNqwE^ZtGq}MB
z`A9If_P!lX!x=AtKvPU05C^9ASP=qd_9kv_X*#5SwB!CK>DBK=AVeJW7x~cTXcP(b
z>O7idt~@%*xI&;>-uweP;_+phEn6!Y5%Mm_k;bx5EDqneZJ6IXjELJnzgs(uKZ(Bh
z(Qzf}Ps=-TmANHrx8FBMxDTc#M_EULmb8tho8}SC8;Cx{{!rxf@e_~~G8XatM!Idg
z?ol91dVGfnQSgg5g!l=Y0v9&KGv+Iw@6G7qZy?=Vy`GF5<+7ag+=iX#b&x}*tc=CV
z7_C42xqzwm*pF;69l`E?oo#YkLK1#o+hCgt_;1_u7dMl*Z4XM`3^MIYw`9zQOSYJ^
zhY1pAo>0iS`yt!3x2_YN&>JB>MaP7<0(V;kJX^|UnMYnZ^~^qyIBI5T;R<%@ot1Lk
zFB6?-dwP`poWoD`Phi%@~5Ac&xzdA_KcJYkPL!ttI(`#Rr8&~=)jde+({`{Zk01Cmbe
zRwb*-p6#BWzA&*eOQ)9`3a**MT@jKgyHsSqYYiVPxeE1qdMJ|+9HzpH48;-$COQrp
z6eeX%R4ijT9`^Jo>p1F_DW%{c#8pzuO7r*2-8fQQO5hW|Dm~o~5&2fzWzDpY++U)^
z3w7V}3X#~`T!kdm9ItOxcy)iS{v;DMfbLQ5Pfm7qh-^%;QwTkbT|4+;ui)XvFIchC
znpo+!9o5GlE;+9|?NRkO2(|F7kGk-sx!r(ot*ZzSFRnPXpk
z?Yo|afe6vk!EU9@2t=e=?9T=N=VO&Zm%VQ#?1jE}=vVUz01cxri0Lt@>q>F)b3M41
z{C%b8Xv<#g{gUpd-TD-IbQ3`XRB3p2XCxUduODLkZZTIwsktIsV87QP#_^i^q8OBi
zBg`rvx-^`#_~pIhkHI`m(QoL-F!WDCkKTwihNQ0?lW*NazM~MllH=(EkMupaB7WzU
zhA`OF%#D<)q(pbSnaA9W^{snTe;l8ppj70prMXzHbLFey1I8F3PS{HE4b3D*P2vUJ
z{#E%!k$9)b1e5x*mg}|~CC0r)=HqU$ULRb%BL80DMW^eM*~6fzH7dof+z(fp{A7kmlV-rxdTNtlxKX
z4-CniRGubrayP2D0O-Qd6qPM%mvC-JF{D7nB&~0b$~$^rQg1!(#+1Q>-B@lP~k<
zJ@2AH(KQ=#!xj-%p9%#!pe#I2L!J7YFz}rptb?@oOm${5fnZe7e1}}tJnKBn=XYo5
za6u8l@PztB_kl^r&pjiGgAXgb(Ik
zUdm148ursQ;Y7{E{A9WAZ&bKhVg#9JK6ol?VOU_Dejj1c*~z>a0+(Inj|>;mxY9{V
zPub0!J)vBp9mlE?kz#r`HAn8MSm3C)!eR1UE~hS8yzMS?4@*BX*|i-$p{cHAs4MEy
zLz|CGd*yx9d3&5yBlVK!=n#KpS)qm_L6+;D>FS&Tw|LUPLnNhs%i`d7`!PMq+s$_$
zf9GivOddn@dZzR~LhD=-M_pI(87PbKwiN2SGUM(y|3EP3_)GEr&9w(rZ`gLW8&dBv
z=Vg6Ov9btZO<`2e^to=QDfG@Pw<$iUV^48y$+|F0VLDALkjqUut^j?Y{y|u&Se-*p
z%V0puL-b=f-AeWc-TvBK#j!kxCwt<86Se)5`6p`yh~#8b^&ENEIN?rp_3xrjBwBWD
z#q42$+As^#n=ASZ#Rh6n;$P`bRpO<_@|C-4Z*IMtqSn^Rn5r(dfGSO>byOX9>{iDO
zZW2}${tVQ(mv5dp$`yS8D=jw|p&jQBcS=yKqD_8p#k=NH^gs%^zYE<3X~o8!Si4Vd
zbMucpMzl^BqSfcD;x9$m8)=DDxP*o!TNc!ukY0{4j`*^-^abzn=C>#Le+s6L#A%|7
z2Ew=s$zmqr?Nrhnc3sMT;=Odp_x0?wGeDTudR4~+YTj+BFZ0i3@Pt3JmM+g~t|_e@KmOY7Z6%$tLxY0{?+29|}}dA5k97TsPIdE+Yh`@Gi@IR%U3pi*7^
z!Ed79U!?3Z8AD7$Kjz}SKBAa=#yjs-NyV`m`niN(!(jfXVEzF4g~u+Eti{X5;h@hm
zt!my|pZ-pg0)Cm8Nd95=*TKafi&9&E*;1MmLNauioJhxV<29C)#|KCM#6NMbg#PYz
zf_SSUcH*<%+O^+zSig$0(_-Kiab(hMxMQfj7qHyRtD#=C_V=2HTE%a)=_h|*X!@sK
zucBN&yZP%!B_eYv1L>aYCdR9!FE@TjMvdDYaYZ%%YB|brD)qc~Ksf$v2W?_0$)K62
z^v>whn5P(9PeF+v;j28T=fHM?K_;cTd)cF`cEsb<2P|Ov7w8b1KIfa{8F)5jgIc8<
zcXT|KQ-dvlvUG_Zd(Qr5X9*vJgPaSS4VnIUS%Ae?WJyrf-l#zGOv+r-i;KVUQ`-IiuDz^0?g{p`L~$wGJd
zYLV{?Xu^;|TT%LuMfDXyGxwOV4ssn47E(Gyc`a;c>JllRF|@*AZJ6`<=3&U=wjlY8
z0C`pCo!$7oWf`AOj+dz;mI>{Z*KKXRXk-OUH1NiyU%0EzPHZlLYYk*YnWh>8R?_Xn
z#>8aI*TnT+*VI2N`?2GcvzkAAaq4l~YuIRB^W<{a^6QmkGV}>rl~p_6fp9g1;N9&=
z1Od7AMq|4X9yD_HkhSWhfqU@slAG%G$eknjlbJxHHo6_YyN9kQLT3@}x+qfylW+aj
zUtT((l`?$2DkBkl*$l*t9d#xb>*i@GIgn|-LvjqiY1Z+YsO2{GSdiH_WM%H{yR{#S
z%5Dmt34-rj)HgyAH6!3-tmkgf6lX_ce3g^eY`riD;pKMtg`wwAFI7lq)w8$u1a
zBHu0eIVSA7dV_NP%QYx@y+!@XF^M!&OU=thGV~5lnB!NK>mEUS2B@h8uuuCRGW7IR
zO#r?hy>pa{c6UhsE7HUqDEzMQ1NQu+36Z`qQK&~5FOn+$gQO183j4F*Hqj_ws%P4{
z`&pxE>)RToiK}yGrE6(`s$(q;F&*GhtnloVs!e<+9)`R&UiA;Ww}jGht?$
z!D0Ed3Q^fx+}4Nqyhwj{3nn%OZy&be@#E#CIA5qG75>R>i8U4z{&g97xmHp{m+lMiExFWg@NZ|PsbIQTKPMs6zl=_g
zE4nuJ)xY^^6}oOvyfyyDLKf-9vflK9zi&*1E3)cn&FKf+IIDPh3&zL>yDBsumO)Kx
z8jwP7fNRK(ByB3$3>n`j
zjF94bP5yQXO+zvbXZx_pbU+fgVLKe>%UJ$8)I1>aHm8ChEGAR_TJ&em%=_va@>lfl
zrm<9)b#i?@?u;q-=kND{E+A%JC9z*3GK)n!Sj}`7v+zSS8x=`lTjH{T#R`$FhK5{y
z5&prAiUh%14+p@p&L9LzH_Fwcf=pt;{6ZE<5*(4XdD7qE%XU?Fm%A*
z;wlzY=_O2qapJPJHw&Ukm4!Dq;(<})#lCmV4BraaB~-Z1^rPka!_
zj~sXu0jo0er#g!RPt!O{TPAt=49Q1soS{won6o%iF|23mB^JSFlUnakA#k-sDgw7U
zB{+#Nk1NC^^It21XfTf@moV*-5+0Zq^zTguNbVH*?@(91lW5h{yoPT0%+`}z~*u8N#I`+7<5VaOy?|~AZDW0w=gz(z@(cI>s%LV
zyplx*0`W3p+BxSAb*!^JCj)@^->QDatX`=h)_$Rj8!OOv;fZWC=8zl^fk6D{1T>AY
zM8{rYz(|({;P1tyJ0Cb?#EK!ZVA|z#4%f_pcfCUFr9S-!ie=XqPjVfjd6w(gFqtZd^0Vv
zI#U2Q`YCXB|3L`Y&I!wXKPZe96*jVn%)^K+V;avnQ6^U`an+p!kQkuAl2h#s2J`Vi
zpghdkx^a$7=!PYqraL(d#gfDCh}rvL$mAHZU3`B
zh+`huIRx}G{{$9HM`5!VXGhG!+&U+);fGDD{4E=xJcNrD
zm|M+YMGXR>Fjc{Qj?5o`A&cT7ZDCdl9fNfi9D?I)e?LROhsgl*s2p~Fvb7|8GMKsb
zV2+8@IidbktmEbY5isRJ2RIC#Ex!y#z$^n$Z5BA)60C^6mt$fqChd=yE987OHrW^=
zCjk45yAA<^Aqei%;;MAHAa
z;>(vWY|CNxR|(@Jbk3o<5#zuDKpYtU)u<4r_%)i&gf8Q<;lS*>dn;z&Z(*lG^|wPC
zbJ2n@y9i=CCkSc>jygE_{&yFXVBW`1uk8PSPh;+b|M#i21B0WSltQP8VYZ;C{dDaA
E00F&l(f|Me

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 258c82709..c57087b3b 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
-#Mon Jun 05 12:51:36 MDT 2017
+#Fri Aug 25 15:48:59 MDT 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip

From 20ff029c930b021246eb0c528b93aa0831d70ad3 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Wed, 30 Aug 2017 14:34:33 -0600
Subject: [PATCH 097/229] =?UTF-8?q?Adding=20the=20bintray=20plugin=20to=20?=
 =?UTF-8?q?the=20project=20so=20that=20it=20can=20be=20built=20and=20?=
 =?UTF-8?q?=E2=80=A6=20(#506)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Adding the bintray plugin to the project so that it can be built and pushed automatically

* Updating the travis build to use the new project name
---
 .travis.yml                   |  2 +-
 build.gradle                  | 11 +++++++----
 ds3-interfaces/build.gradle   |  1 +
 ds3-metadata/build.gradle     |  9 ++-------
 ds3-sdk/build.gradle          |  9 ++-------
 ds3-utils/build.gradle        |  3 ++-
 gradle/scripts/publish.gradle | 23 +++++++++++++++++++++++
 settings.gradle               |  1 +
 8 files changed, 39 insertions(+), 20 deletions(-)
 create mode 100644 ds3-interfaces/build.gradle
 create mode 100644 gradle/scripts/publish.gradle

diff --git a/.travis.yml b/.travis.yml
index 8d5b0b1e1..b39a2f0a2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
 language: java
 jdk:
   - oraclejdk8
-script: ./gradlew -S clean jar ds3-sdk:test ds3-utils:test ds3-metadata:test
+script: ./gradlew -S clean jar ds3_java_sdk:test ds3-utils:test ds3-metadata:test
diff --git a/build.gradle b/build.gradle
index 7d6f0212a..a24cca91c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -14,14 +14,17 @@
  */
 
 buildscript {
-    ext.kotlin_version = '1.1.4-2'
+    ext.kotlin_version = '1.1.4-3'
 
     repositories {
         mavenCentral()
+        jcenter()
     }
 
     dependencies {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.7.3'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
     }
 }
 
@@ -58,7 +61,7 @@ task wrapper(type: Wrapper) {
     gradleVersion = '4.1'
 }
 
-project(':ds3-sdk') {
+project(':ds3_java_sdk') {
     dependencies {
         compile project(':ds3-interfaces')
         compile project(':ds3-utils')
@@ -74,14 +77,14 @@ project(':ds3-metadata') {
 
 project(':ds3-sdk-integration') {
     dependencies {
-        compile project(':ds3-sdk')
+        compile project(':ds3_java_sdk')
         compile project(':ds3-metadata')
     }
 }
 
 project(':ds3-sdk-samples') {
     dependencies {
-        compile project(':ds3-sdk')
+        compile project(':ds3_java_sdk')
     }
 }
 
diff --git a/ds3-interfaces/build.gradle b/ds3-interfaces/build.gradle
new file mode 100644
index 000000000..a82804d4d
--- /dev/null
+++ b/ds3-interfaces/build.gradle
@@ -0,0 +1 @@
+apply from: "$rootDir/gradle/scripts/publish.gradle"
diff --git a/ds3-metadata/build.gradle b/ds3-metadata/build.gradle
index 6002069a0..19ee26af8 100644
--- a/ds3-metadata/build.gradle
+++ b/ds3-metadata/build.gradle
@@ -13,15 +13,10 @@
  * ****************************************************************************
  */
 
-buildscript {
-    repositories { jcenter() }
-    dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
-    }
-}
-
 apply plugin: 'com.github.johnrengelman.shadow'
 
+apply from: "$rootDir/gradle/scripts/publish.gradle"
+
 shadowJar {
     relocate 'org.apache', 'ds3metafatjar.org.apache'
     relocate 'com.google', 'ds3metafatjar.com.google'
diff --git a/ds3-sdk/build.gradle b/ds3-sdk/build.gradle
index 5b74d8b55..a683f7683 100644
--- a/ds3-sdk/build.gradle
+++ b/ds3-sdk/build.gradle
@@ -16,15 +16,10 @@
 import java.nio.file.Files
 import java.nio.file.Path
 
-buildscript {
-    repositories { jcenter() }
-    dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
-    }
-}
-
 apply plugin: 'com.github.johnrengelman.shadow'
 
+apply from: "$rootDir/gradle/scripts/publish.gradle"
+
 shadowJar {
     relocate 'com.google', 'ds3fatjar.com.google'
     relocate 'org.jetbrains', 'ds3fatjar.org.jetbrains'
diff --git a/ds3-utils/build.gradle b/ds3-utils/build.gradle
index f8f974efa..219664d5c 100644
--- a/ds3-utils/build.gradle
+++ b/ds3-utils/build.gradle
@@ -14,9 +14,10 @@
  * ****************************************************************************
  */
 
+apply from: "$rootDir/gradle/scripts/publish.gradle"
+
 dependencies {
     compile "commons-codec:commons-codec:$commonscodecVersion"
     compile "commons-io:commons-io:$commonsioVersion"
     compile "com.google.guava:guava:$guavaVersion"
 }
-
diff --git a/gradle/scripts/publish.gradle b/gradle/scripts/publish.gradle
new file mode 100644
index 000000000..7a3b70027
--- /dev/null
+++ b/gradle/scripts/publish.gradle
@@ -0,0 +1,23 @@
+apply plugin: 'com.jfrog.bintray'
+
+bintray {
+    user = System.getenv('BINTRAY_USER')
+    key = System.getenv('BINTRAY_KEY')
+
+    configurations = ['archives']
+
+    //dryRun = true
+    publish = true
+
+    pkg {
+        name = "$project.name"
+        repo = "ds3"
+        userOrg = "spectralogic"
+        websiteUrl = "https://github.com/SpectraLogic/ds3_java_sdk"
+        vcsUrl = "https://github.com/SpectraLogic/ds3_java_sdk.git"
+        licenses = ['Apache-2.0']
+        version {
+            name = "$project.version"
+        }
+    }
+}
\ No newline at end of file
diff --git a/settings.gradle b/settings.gradle
index d0a85a6f1..b7a6f649c 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -16,3 +16,4 @@
 include 'ds3-sdk-integration', 'ds3-sdk', 'ds3-sdk-samples', 'ds3-interfaces', 'ds3-metadata', 'ds3-utils'
 rootProject.name = 'ds3-java-sdk'
 
+project(":ds3-sdk").name = "ds3_java_sdk"

From d0adbd6716a0a91f28cdff0ecbbe6e40bd75df66 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Wed, 30 Aug 2017 15:22:48 -0600
Subject: [PATCH 098/229] Update README.md

---
 README.md | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index b87ca2001..08d4f07a5 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ If using Intellij, simply import the project as a Gradle project.
 
 ## Install
 
-To install the latest `ds3_java_sdk` either download the latest release jar file from the [Releases](../../releases) page or clone the repository with `git clone https://github.com/SpectraLogic/ds3_java_sdk.git`, cd to `ds3_java_sdk` and run `./gradlew clean ds3-sdk:install` to install the sdk into your local maven repository.  It is compatible with Java 7 and above.
+To install the latest `ds3_java_sdk` either download the latest release jar file from the [Releases](../../releases) page or clone the repository with `git clone https://github.com/SpectraLogic/ds3_java_sdk.git`, cd to `ds3_java_sdk` and run `./gradlew clean ds3_java_sdk:install` to install the sdk into your local maven repository.  It is compatible with Java 8.
 
 The SDK can also be included directly into a Maven or Gradle build. There is also a fatjar version that you can optionally use with the `all` classifier, take a look at the commented out code in the examples below.  To include the SDK  into maven add the following to the `pom.xml` file:
 
@@ -40,8 +40,8 @@ The SDK can also be included directly into a Maven or Gradle build. There is als
       ...
       
         com.spectralogic.ds3
-        ds3-sdk
-        4.0.0
+        ds3_java_sdk
+        3.5.2
         
       
     ...
@@ -64,8 +64,8 @@ repositories {
 
 dependencies {
     ...
-    compile 'com.spectralogic.ds3:ds3-sdk:4.0.0'
-    // compile 'com.spectralogic.ds3:ds3-sdk:4.0.0:all'
+    compile 'com.spectralogic.ds3:ds3_java_sdk:3.5.2'
+    // compile 'com.spectralogic.ds3:ds3_java_sdk:3.5.2:all'
     ...
 }
 
@@ -93,4 +93,16 @@ bridges, please see [SLF4j.org](http://www.slf4j.org/manual.html).
 
 In addition to unit tests in the main `ds3-sdk` module, there are additional integration tests in the `ds3-integration` module.  Please see the integration [README](ds3-sdk-integration/README.md) for details on running the tests.  To just run the SDK's unit tests use:
 
-    ./gradlew clean ds3-sdk:test
+    ./gradlew clean ds3_java_sdk:test
+
+## Publishing
+
+Before a user can publish the SDK they must first have [Bintray](https://bintray.com/spectralogic) API credentials and access to our organization to upload.  Once a user has access they can set their credentials on the command line by issuing:
+
+```shell
+
+$ export BINTRAY_USER="userName"
+$ export BINTRAY_KEY="api_key"
+```
+
+After the credentials have been set, to publish the SDK to [Bintray](https://bintray.com/spectralogic), issue the following from the command line run: `./gradlew clean bintrayUpload`

From 77fe3ea00366ed893c65ec1059fd9d2a32beeb74 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Thu, 31 Aug 2017 09:04:53 -0600
Subject: [PATCH 099/229] Reverting a change that modified the sdk in bintray
 that would require users to change the artifact name in their build.

---
 README.md                     | 10 +++++-----
 build.gradle                  |  6 +++---
 gradle/scripts/publish.gradle |  4 ++--
 settings.gradle               |  2 --
 4 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 08d4f07a5..9b2d09b9d 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ If using Intellij, simply import the project as a Gradle project.
 
 ## Install
 
-To install the latest `ds3_java_sdk` either download the latest release jar file from the [Releases](../../releases) page or clone the repository with `git clone https://github.com/SpectraLogic/ds3_java_sdk.git`, cd to `ds3_java_sdk` and run `./gradlew clean ds3_java_sdk:install` to install the sdk into your local maven repository.  It is compatible with Java 8.
+To install the latest `ds3_java_sdk` either download the latest release jar file from the [Releases](../../releases) page or clone the repository with `git clone https://github.com/SpectraLogic/ds3_java_sdk.git`, cd to `ds3_java_sdk` and run `./gradlew clean ds3-sdk:install` to install the sdk into your local maven repository.  It is compatible with Java 8.
 
 The SDK can also be included directly into a Maven or Gradle build. There is also a fatjar version that you can optionally use with the `all` classifier, take a look at the commented out code in the examples below.  To include the SDK  into maven add the following to the `pom.xml` file:
 
@@ -40,7 +40,7 @@ The SDK can also be included directly into a Maven or Gradle build. There is als
       ...
       
         com.spectralogic.ds3
-        ds3_java_sdk
+        ds3-sdk
         3.5.2
         
       
@@ -64,8 +64,8 @@ repositories {
 
 dependencies {
     ...
-    compile 'com.spectralogic.ds3:ds3_java_sdk:3.5.2'
-    // compile 'com.spectralogic.ds3:ds3_java_sdk:3.5.2:all'
+    compile 'com.spectralogic.ds3:ds3-sdk:3.5.2'
+    // compile 'com.spectralogic.ds3:ds3-sdk:3.5.2:all'
     ...
 }
 
@@ -93,7 +93,7 @@ bridges, please see [SLF4j.org](http://www.slf4j.org/manual.html).
 
 In addition to unit tests in the main `ds3-sdk` module, there are additional integration tests in the `ds3-integration` module.  Please see the integration [README](ds3-sdk-integration/README.md) for details on running the tests.  To just run the SDK's unit tests use:
 
-    ./gradlew clean ds3_java_sdk:test
+    ./gradlew clean ds3-sdk:test
 
 ## Publishing
 
diff --git a/build.gradle b/build.gradle
index a24cca91c..6d255998b 100644
--- a/build.gradle
+++ b/build.gradle
@@ -61,7 +61,7 @@ task wrapper(type: Wrapper) {
     gradleVersion = '4.1'
 }
 
-project(':ds3_java_sdk') {
+project(':ds3-sdk') {
     dependencies {
         compile project(':ds3-interfaces')
         compile project(':ds3-utils')
@@ -77,14 +77,14 @@ project(':ds3-metadata') {
 
 project(':ds3-sdk-integration') {
     dependencies {
-        compile project(':ds3_java_sdk')
+        compile project(':ds3-sdk')
         compile project(':ds3-metadata')
     }
 }
 
 project(':ds3-sdk-samples') {
     dependencies {
-        compile project(':ds3_java_sdk')
+        compile project(':ds3-sdk')
     }
 }
 
diff --git a/gradle/scripts/publish.gradle b/gradle/scripts/publish.gradle
index 7a3b70027..4fc4ecd85 100644
--- a/gradle/scripts/publish.gradle
+++ b/gradle/scripts/publish.gradle
@@ -6,7 +6,7 @@ bintray {
 
     configurations = ['archives']
 
-    //dryRun = true
+    dryRun = true
     publish = true
 
     pkg {
@@ -20,4 +20,4 @@ bintray {
             name = "$project.version"
         }
     }
-}
\ No newline at end of file
+}
diff --git a/settings.gradle b/settings.gradle
index b7a6f649c..5e0c648b0 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -15,5 +15,3 @@
 
 include 'ds3-sdk-integration', 'ds3-sdk', 'ds3-sdk-samples', 'ds3-interfaces', 'ds3-metadata', 'ds3-utils'
 rootProject.name = 'ds3-java-sdk'
-
-project(":ds3-sdk").name = "ds3_java_sdk"

From bef5b30e15645196f92cd5650ab1a98844b921d0 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Thu, 31 Aug 2017 09:06:23 -0600
Subject: [PATCH 100/229] commenting out the dryRun flag for the bintrayUpload
 task

---
 gradle/scripts/publish.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle/scripts/publish.gradle b/gradle/scripts/publish.gradle
index 4fc4ecd85..833862f54 100644
--- a/gradle/scripts/publish.gradle
+++ b/gradle/scripts/publish.gradle
@@ -6,7 +6,7 @@ bintray {
 
     configurations = ['archives']
 
-    dryRun = true
+    //dryRun = true
     publish = true
 
     pkg {

From 62f813952cce970d806500e3e8bb7c4b860aa322 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Thu, 31 Aug 2017 09:10:45 -0600
Subject: [PATCH 101/229] reverting module name in travis build.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index b39a2f0a2..8d5b0b1e1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
 language: java
 jdk:
   - oraclejdk8
-script: ./gradlew -S clean jar ds3_java_sdk:test ds3-utils:test ds3-metadata:test
+script: ./gradlew -S clean jar ds3-sdk:test ds3-utils:test ds3-metadata:test

From 6b0ec0b68f8cb9449ecce0021a608b48e75da8d2 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 1 Sep 2017 11:43:40 -0600
Subject: [PATCH 102/229] Ensuring streams and channels are not null

---
 .../com/spectralogic/ds3client/Ds3Client.java   |  1 +
 .../ds3client/commands/GetObjectRequest.java    | 17 ++++++++++++-----
 .../commands/PutMultiPartUploadPartRequest.java | 14 ++++++++++----
 .../ds3client/commands/PutObjectRequest.java    | 17 ++++++++++++-----
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 ...otificationRegistrationSpectraS3Request.java |  1 -
 16 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index b81e82ce0..559dfc55e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -34,6 +34,7 @@
 public interface Ds3Client extends Closeable {
 
     ConnectionDetails getConnectionDetails();
+
     
     
     AbortMultiPartUploadResponse abortMultiPartUpload(final AbortMultiPartUploadRequest request)
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
index 9d1f17ca0..68c269fcc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
@@ -29,6 +29,8 @@
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import javax.annotation.Nonnull;
+import com.google.common.base.Preconditions;
 import com.spectralogic.ds3client.models.ChecksumType;
 
 public class GetObjectRequest extends AbstractRequest {
@@ -52,7 +54,8 @@ public class GetObjectRequest extends AbstractRequest {
     
     /** @deprecated use {@link #GetObjectRequest(String, String, WritableByteChannel, UUID, long)} instead */
     @Deprecated
-    public GetObjectRequest(final String bucketName, final String objectName, final WritableByteChannel channel) {
+    public GetObjectRequest(final String bucketName, final String objectName, @Nonnull final WritableByteChannel channel) {
+        Preconditions.checkNotNull(channel, "Channel may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.channel = channel;
@@ -61,7 +64,8 @@ public GetObjectRequest(final String bucketName, final String objectName, final
     }
 
     
-    public GetObjectRequest(final String bucketName, final String objectName, final WritableByteChannel channel, final UUID job, final long offset) {
+    public GetObjectRequest(final String bucketName, final String objectName, @Nonnull final WritableByteChannel channel, final UUID job, final long offset) {
+        Preconditions.checkNotNull(channel, "Channel may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.job = job.toString();
@@ -76,7 +80,8 @@ public GetObjectRequest(final String bucketName, final String objectName, final
     }
 
     
-    public GetObjectRequest(final String bucketName, final String objectName, final WritableByteChannel channel, final String job, final long offset) {
+    public GetObjectRequest(final String bucketName, final String objectName, @Nonnull final WritableByteChannel channel, final String job, final long offset) {
+        Preconditions.checkNotNull(channel, "Channel may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.job = job;
@@ -91,7 +96,8 @@ public GetObjectRequest(final String bucketName, final String objectName, final
     }
 
     
-    public GetObjectRequest(final String bucketName, final String objectName, final UUID job, final long offset, final OutputStream stream) {
+    public GetObjectRequest(final String bucketName, final String objectName, final UUID job, final long offset, @Nonnull final OutputStream stream) {
+        Preconditions.checkNotNull(stream, "Stream may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.job = job.toString();
@@ -106,7 +112,8 @@ public GetObjectRequest(final String bucketName, final String objectName, final
     }
 
     
-    public GetObjectRequest(final String bucketName, final String objectName, final String job, final long offset, final OutputStream stream) {
+    public GetObjectRequest(final String bucketName, final String objectName, final String job, final long offset, @Nonnull final OutputStream stream) {
+        Preconditions.checkNotNull(stream, "Stream may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.job = job;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest.java
index e3340d670..3e5e3c9a5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest.java
@@ -20,6 +20,8 @@
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import javax.annotation.Nonnull;
+import com.google.common.base.Preconditions;
 import com.spectralogic.ds3client.utils.SeekableByteChannelInputStream;
 import java.nio.channels.SeekableByteChannel;
 import java.io.InputStream;
@@ -45,7 +47,8 @@ public class PutMultiPartUploadPartRequest extends AbstractRequest {
     // Constructor
     
     
-    public PutMultiPartUploadPartRequest(final String bucketName, final String objectName, final SeekableByteChannel channel, final int partNumber, final long size, final UUID uploadId) {
+    public PutMultiPartUploadPartRequest(final String bucketName, final String objectName, @Nonnull final SeekableByteChannel channel, final int partNumber, final long size, final UUID uploadId) {
+        Preconditions.checkNotNull(channel, "Channel may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.partNumber = partNumber;
@@ -61,7 +64,8 @@ public PutMultiPartUploadPartRequest(final String bucketName, final String objec
     }
 
     
-    public PutMultiPartUploadPartRequest(final String bucketName, final String objectName, final SeekableByteChannel channel, final int partNumber, final long size, final String uploadId) {
+    public PutMultiPartUploadPartRequest(final String bucketName, final String objectName, @Nonnull final SeekableByteChannel channel, final int partNumber, final long size, final String uploadId) {
+        Preconditions.checkNotNull(channel, "Channel may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.partNumber = partNumber;
@@ -77,7 +81,8 @@ public PutMultiPartUploadPartRequest(final String bucketName, final String objec
     }
 
     
-    public PutMultiPartUploadPartRequest(final String bucketName, final String objectName, final int partNumber, final long size, final InputStream stream, final UUID uploadId) {
+    public PutMultiPartUploadPartRequest(final String bucketName, final String objectName, final int partNumber, final long size, @Nonnull final InputStream stream, final UUID uploadId) {
+        Preconditions.checkNotNull(stream, "Stream may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.partNumber = partNumber;
@@ -92,7 +97,8 @@ public PutMultiPartUploadPartRequest(final String bucketName, final String objec
     }
 
     
-    public PutMultiPartUploadPartRequest(final String bucketName, final String objectName, final int partNumber, final long size, final InputStream stream, final String uploadId) {
+    public PutMultiPartUploadPartRequest(final String bucketName, final String objectName, final int partNumber, final long size, @Nonnull final InputStream stream, final String uploadId) {
+        Preconditions.checkNotNull(stream, "Stream may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.partNumber = partNumber;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutObjectRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutObjectRequest.java
index db69c7c4a..c66185a6b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutObjectRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/PutObjectRequest.java
@@ -26,6 +26,8 @@
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import javax.annotation.Nonnull;
+import com.google.common.base.Preconditions;
 import com.spectralogic.ds3client.models.ChecksumType;
 public class PutObjectRequest extends AbstractRequest {
 
@@ -53,7 +55,8 @@ public class PutObjectRequest extends AbstractRequest {
     
     /** @deprecated use {@link #PutObjectRequest(String, String, SeekableByteChannel, UUID, long, long)} instead */
     @Deprecated
-    public PutObjectRequest(final String bucketName, final String objectName, final SeekableByteChannel channel, final long size) {
+    public PutObjectRequest(final String bucketName, final String objectName, @Nonnull final SeekableByteChannel channel, final long size) {
+        Preconditions.checkNotNull(channel, "Channel may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.size = size;
@@ -64,7 +67,8 @@ public PutObjectRequest(final String bucketName, final String objectName, final
     }
 
     
-    public PutObjectRequest(final String bucketName, final String objectName, final SeekableByteChannel channel, final UUID job, final long offset, final long size) {
+    public PutObjectRequest(final String bucketName, final String objectName, @Nonnull final SeekableByteChannel channel, final UUID job, final long offset, final long size) {
+        Preconditions.checkNotNull(channel, "Channel may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.size = size;
@@ -81,7 +85,8 @@ public PutObjectRequest(final String bucketName, final String objectName, final
     }
 
     
-    public PutObjectRequest(final String bucketName, final String objectName, final SeekableByteChannel channel, final String job, final long offset, final long size) {
+    public PutObjectRequest(final String bucketName, final String objectName, @Nonnull final SeekableByteChannel channel, final String job, final long offset, final long size) {
+        Preconditions.checkNotNull(channel, "Channel may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.size = size;
@@ -98,7 +103,8 @@ public PutObjectRequest(final String bucketName, final String objectName, final
     }
 
     
-    public PutObjectRequest(final String bucketName, final String objectName, final UUID job, final long offset, final long size, final InputStream stream) {
+    public PutObjectRequest(final String bucketName, final String objectName, final UUID job, final long offset, final long size, @Nonnull final InputStream stream) {
+        Preconditions.checkNotNull(stream, "Stream may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.size = size;
@@ -114,7 +120,8 @@ public PutObjectRequest(final String bucketName, final String objectName, final
     }
 
     
-    public PutObjectRequest(final String bucketName, final String objectName, final String job, final long offset, final long size, final InputStream stream) {
+    public PutObjectRequest(final String bucketName, final String objectName, final String job, final long offset, final long size, @Nonnull final InputStream stream) {
+        Preconditions.checkNotNull(stream, "Stream may not be null.");
         this.bucketName = bucketName;
         this.objectName = objectName;
         this.size = size;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutDs3TargetFailureNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutDs3TargetFailureNotificationRegistrationSpectraS3Request.java
index 548c7faf3..25c7a6c43 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutDs3TargetFailureNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutDs3TargetFailureNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutDs3TargetFailureNotificationRegistrationSpectraS3Request extends
     
     public PutDs3TargetFailureNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCompletedNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCompletedNotificationRegistrationSpectraS3Request.java
index b4dcdb438..97ce73d2e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCompletedNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCompletedNotificationRegistrationSpectraS3Request.java
@@ -38,7 +38,6 @@ public class PutJobCompletedNotificationRegistrationSpectraS3Request extends Abs
     
     public PutJobCompletedNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCreatedNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCreatedNotificationRegistrationSpectraS3Request.java
index 5782cc444..f996fc33d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCreatedNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCreatedNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutJobCreatedNotificationRegistrationSpectraS3Request extends Abstr
     
     public PutJobCreatedNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCreationFailedNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCreationFailedNotificationRegistrationSpectraS3Request.java
index 08285353d..2a3473c57 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCreationFailedNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutJobCreationFailedNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutJobCreationFailedNotificationRegistrationSpectraS3Request extend
     
     public PutJobCreationFailedNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectCachedNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectCachedNotificationRegistrationSpectraS3Request.java
index 7bf3a9d94..3621cbb8d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectCachedNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectCachedNotificationRegistrationSpectraS3Request.java
@@ -38,7 +38,6 @@ public class PutObjectCachedNotificationRegistrationSpectraS3Request extends Abs
     
     public PutObjectCachedNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectLostNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectLostNotificationRegistrationSpectraS3Request.java
index bef37e1e2..2bfc5a9fe 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectLostNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectLostNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutObjectLostNotificationRegistrationSpectraS3Request extends Abstr
     
     public PutObjectLostNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectPersistedNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectPersistedNotificationRegistrationSpectraS3Request.java
index d94928346..1a7b11431 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectPersistedNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutObjectPersistedNotificationRegistrationSpectraS3Request.java
@@ -38,7 +38,6 @@ public class PutObjectPersistedNotificationRegistrationSpectraS3Request extends
     
     public PutObjectPersistedNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutPoolFailureNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutPoolFailureNotificationRegistrationSpectraS3Request.java
index e9be01db0..8af8a90f8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutPoolFailureNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutPoolFailureNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutPoolFailureNotificationRegistrationSpectraS3Request extends Abst
     
     public PutPoolFailureNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutStorageDomainFailureNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutStorageDomainFailureNotificationRegistrationSpectraS3Request.java
index 571a01fd8..48b0ae4fc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutStorageDomainFailureNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutStorageDomainFailureNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutStorageDomainFailureNotificationRegistrationSpectraS3Request ext
     
     public PutStorageDomainFailureNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutSystemFailureNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutSystemFailureNotificationRegistrationSpectraS3Request.java
index b6e71d63d..92987afc6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutSystemFailureNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutSystemFailureNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutSystemFailureNotificationRegistrationSpectraS3Request extends Ab
     
     public PutSystemFailureNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutTapeFailureNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutTapeFailureNotificationRegistrationSpectraS3Request.java
index 121dbc3c4..5dcf107ed 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutTapeFailureNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutTapeFailureNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutTapeFailureNotificationRegistrationSpectraS3Request extends Abst
     
     public PutTapeFailureNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutTapePartitionFailureNotificationRegistrationSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutTapePartitionFailureNotificationRegistrationSpectraS3Request.java
index cb4d67fee..91fb37f65 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutTapePartitionFailureNotificationRegistrationSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/notifications/PutTapePartitionFailureNotificationRegistrationSpectraS3Request.java
@@ -35,7 +35,6 @@ public class PutTapePartitionFailureNotificationRegistrationSpectraS3Request ext
     
     public PutTapePartitionFailureNotificationRegistrationSpectraS3Request(final String notificationEndPoint) {
         super(notificationEndPoint);
-
         
 
     }

From fc961a3b7a342e98bc6fb6fea01bf1f0bf3da340 Mon Sep 17 00:00:00 2001
From: Denver 
Date: Wed, 6 Sep 2017 09:34:52 -0600
Subject: [PATCH 103/229] Update Dockerfile to Java8

---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 00d7d6b08..5ce225695 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM greyrocksoftware/javastandalone7
+FROM greyrocksoftware/javastandalone
 
 # Set the locale
 RUN locale-gen en_US.UTF-8

From 8c4a74cedd512c10d54debb963b3db919c670c1d Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Thu, 7 Sep 2017 14:02:16 -0600
Subject: [PATCH 104/229] Negative tests for null streams and channels

---
 .../ds3client/Ds3Client_Test.java             | 70 +++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index f9c034f42..dcdfcc857 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -1554,4 +1554,74 @@ public void getBlobsOnDs3TargetTest() throws IOException {
                 .asClient()
                 .getBlobsOnDs3TargetSpectraS3(new GetBlobsOnDs3TargetSpectraS3Request(target));
     }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void getObjectNullChannelDeprecatedConstructorTest() {
+        new GetObjectRequest("BucketName", "ObjectName", null);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void getObjectNullChannelUuidConstructorTest() {
+        new GetObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(), 0);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void getObjectNullChannelStringConstructorTest() {
+        new GetObjectRequest("BucketName", "ObjectName", null, "JobId", 0);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void getObjectNullStreamUuidConstructorTest() {
+        new GetObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, null);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void getObjectNullStreamStringConstructorTest() {
+        new GetObjectRequest("BucketName", "ObjectName", "JobId", 0, null);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putObjectDeprecatedConstructorTest() {
+        new PutObjectRequest("BucketName", "ObjectName", null, 0);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putObjectNullChannelUuidConstructorTest() {
+        new PutObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(),0, 0);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putObjectNullChannelStringConstructorTest() {
+        new PutObjectRequest("BucketName", "ObjectName", null, "JobId",0, 0);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putObjectNullStreamUuidConstructorTest() {
+        new PutObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, 0, null);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putObjectNullStreamStringConstructorTest() {
+        new PutObjectRequest("BucketName", "ObjectName", "JobId", 0, 0, null);
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putMultiPartUploadNullChannelUuidConstructorTest() {
+        new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, UUID.randomUUID());
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putMultiPartUploadNullChannelStringConstructorTest() {
+        new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, "UploadId");
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putMultiPartUploadNullStreamUuidConstructorTest() {
+        new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, UUID.randomUUID());
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void putMultiPartUploadNullStreamStringConstructorTest() {
+        new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, "UploadId");
+    }
 }

From 67361548061311f409714b0e5697a0e5eca823fb Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Thu, 7 Sep 2017 15:16:16 -0600
Subject: [PATCH 105/229] Handling both preconditions and annotation error
 thrown by IntelliJ in negative tests

---
 .../ds3client/Ds3Client_Test.java             | 169 +++++++++++++++---
 1 file changed, 141 insertions(+), 28 deletions(-)

diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index dcdfcc857..e8ec6702d 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -51,6 +51,7 @@
 import static org.hamcrest.CoreMatchers.*;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.junit.Assert.fail;
 
 public class Ds3Client_Test {
     private static final UUID MASTER_OBJECT_LIST_JOB_ID = UUID.fromString("1a85e743-ec8f-4789-afec-97e587a26936");
@@ -1555,73 +1556,185 @@ public void getBlobsOnDs3TargetTest() throws IOException {
                 .getBlobsOnDs3TargetSpectraS3(new GetBlobsOnDs3TargetSpectraS3Request(target));
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void getObjectNullChannelDeprecatedConstructorTest() {
-        new GetObjectRequest("BucketName", "ObjectName", null);
+        try {
+            new GetObjectRequest("BucketName", "ObjectName", null);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Channel may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void getObjectNullChannelUuidConstructorTest() {
-        new GetObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(), 0);
+        try {
+            new GetObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(), 0);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Channel may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void getObjectNullChannelStringConstructorTest() {
-        new GetObjectRequest("BucketName", "ObjectName", null, "JobId", 0);
+        try {
+            new GetObjectRequest("BucketName", "ObjectName", null, "JobId", 0);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Channel may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void getObjectNullStreamUuidConstructorTest() {
-        new GetObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, null);
+        try {
+            new GetObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, null);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Stream may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void getObjectNullStreamStringConstructorTest() {
-        new GetObjectRequest("BucketName", "ObjectName", "JobId", 0, null);
+        try {
+            new GetObjectRequest("BucketName", "ObjectName", "JobId", 0, null);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Stream may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putObjectDeprecatedConstructorTest() {
-        new PutObjectRequest("BucketName", "ObjectName", null, 0);
+        try {
+            new PutObjectRequest("BucketName", "ObjectName", null, 0);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Channel may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putObjectNullChannelUuidConstructorTest() {
-        new PutObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(),0, 0);
+        try {
+            new PutObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(),0, 0);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Channel may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putObjectNullChannelStringConstructorTest() {
-        new PutObjectRequest("BucketName", "ObjectName", null, "JobId",0, 0);
+        try {
+            new PutObjectRequest("BucketName", "ObjectName", null, "JobId",0, 0);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Channel may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putObjectNullStreamUuidConstructorTest() {
-        new PutObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, 0, null);
+        try {
+            new PutObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, 0, null);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Stream may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putObjectNullStreamStringConstructorTest() {
-        new PutObjectRequest("BucketName", "ObjectName", "JobId", 0, 0, null);
+        try {
+            new PutObjectRequest("BucketName", "ObjectName", "JobId", 0, 0, null);
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Stream may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putMultiPartUploadNullChannelUuidConstructorTest() {
-        new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, UUID.randomUUID());
+        try {
+            new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, UUID.randomUUID());
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Channel may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putMultiPartUploadNullChannelStringConstructorTest() {
-        new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, "UploadId");
+        try {
+            new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, "UploadId");
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Channel may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putMultiPartUploadNullStreamUuidConstructorTest() {
-        new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, UUID.randomUUID());
+        try {
+            new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, UUID.randomUUID());
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Stream may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest. must not be null"));
+        }
     }
 
-    @Test (expected = IllegalArgumentException.class)
+    @Test
     public void putMultiPartUploadNullStreamStringConstructorTest() {
-        new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, "UploadId");
+        try {
+            new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, "UploadId");
+            fail();
+        } catch (final NullPointerException e) {
+            assertThat(e.getMessage(), is("Stream may not be null."));
+        } catch (final IllegalArgumentException e) {
+            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
+            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest. must not be null"));
+        }
     }
 }

From d6a4536172e50c2837a36c11f82ff9bff2e988cd Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Thu, 7 Sep 2017 16:58:49 -0600
Subject: [PATCH 106/229] Improved tests with lambda

---
 .../ds3client/Ds3Client_Test.java             | 154 ++++--------------
 1 file changed, 34 insertions(+), 120 deletions(-)

diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index e8ec6702d..7c83007fb 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -1558,183 +1558,97 @@ public void getBlobsOnDs3TargetTest() throws IOException {
 
     @Test
     public void getObjectNullChannelDeprecatedConstructorTest() {
-        try {
-            new GetObjectRequest("BucketName", "ObjectName", null);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Channel may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Channel",
+                () -> new GetObjectRequest("BucketName", "ObjectName", null));
     }
 
     @Test
     public void getObjectNullChannelUuidConstructorTest() {
-        try {
-            new GetObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(), 0);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Channel may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Channel",
+                () -> new GetObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(), 0));
     }
 
     @Test
     public void getObjectNullChannelStringConstructorTest() {
-        try {
-            new GetObjectRequest("BucketName", "ObjectName", null, "JobId", 0);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Channel may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Channel",
+                () -> new GetObjectRequest("BucketName", "ObjectName", null, "JobId", 0));
     }
 
     @Test
     public void getObjectNullStreamUuidConstructorTest() {
-        try {
-            new GetObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, null);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Stream may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Stream",
+                () -> new GetObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, null));
     }
 
     @Test
     public void getObjectNullStreamStringConstructorTest() {
-        try {
-            new GetObjectRequest("BucketName", "ObjectName", "JobId", 0, null);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Stream may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/GetObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Stream",
+                () -> new GetObjectRequest("BucketName", "ObjectName", "JobId", 0, null));
     }
 
     @Test
     public void putObjectDeprecatedConstructorTest() {
-        try {
-            new PutObjectRequest("BucketName", "ObjectName", null, 0);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Channel may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Channel",
+                () -> new PutObjectRequest("BucketName", "ObjectName", null, 0));
     }
 
     @Test
     public void putObjectNullChannelUuidConstructorTest() {
-        try {
-            new PutObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(),0, 0);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Channel may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Channel",
+                () -> new PutObjectRequest("BucketName", "ObjectName", null, UUID.randomUUID(),0, 0));
     }
 
     @Test
     public void putObjectNullChannelStringConstructorTest() {
-        try {
-            new PutObjectRequest("BucketName", "ObjectName", null, "JobId",0, 0);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Channel may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Channel",
+                () -> new PutObjectRequest("BucketName", "ObjectName", null, "JobId",0, 0));
     }
 
     @Test
     public void putObjectNullStreamUuidConstructorTest() {
-        try {
-            new PutObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, 0, null);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Stream may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Stream",
+                () -> new PutObjectRequest("BucketName", "ObjectName", UUID.randomUUID(), 0, 0, null));
     }
 
     @Test
     public void putObjectNullStreamStringConstructorTest() {
-        try {
-            new PutObjectRequest("BucketName", "ObjectName", "JobId", 0, 0, null);
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Stream may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/PutObjectRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Stream",
+                () -> new PutObjectRequest("BucketName", "ObjectName", "JobId", 0, 0, null));
     }
 
     @Test
     public void putMultiPartUploadNullChannelUuidConstructorTest() {
-        try {
-            new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, UUID.randomUUID());
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Channel may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Channel",
+                () -> new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, UUID.randomUUID()));
     }
 
     @Test
     public void putMultiPartUploadNullChannelStringConstructorTest() {
-        try {
-            new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, "UploadId");
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Channel may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'channel' of com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Channel",
+                () -> new PutMultiPartUploadPartRequest("BucketName", "ObjectName", null, 0, 0, "UploadId"));
     }
 
     @Test
     public void putMultiPartUploadNullStreamUuidConstructorTest() {
-        try {
-            new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, UUID.randomUUID());
-            fail();
-        } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Stream may not be null."));
-        } catch (final IllegalArgumentException e) {
-            // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest. must not be null"));
-        }
+        testNonnullStreamChannelExceptions("Stream",
+                () -> new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, UUID.randomUUID()));
     }
 
     @Test
     public void putMultiPartUploadNullStreamStringConstructorTest() {
+        testNonnullStreamChannelExceptions("Stream",
+                () -> new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, "UploadId"));
+    }
+
+    private void testNonnullStreamChannelExceptions(final String paramName, final Runnable runnable) {
         try {
-            new PutMultiPartUploadPartRequest("BucketName", "ObjectName", 0, 0, null, "UploadId");
+            runnable.run();
             fail();
         } catch (final NullPointerException e) {
-            assertThat(e.getMessage(), is("Stream may not be null."));
+            assertThat(e.getMessage(), is(paramName + " may not be null."));
         } catch (final IllegalArgumentException e) {
             // IntelliJ IDEA throws an IllegalArgumentException if parameters are annotated as non-null are passed null values
-            assertThat(e.getMessage(), is("Argument for @Nonnull parameter 'stream' of com/spectralogic/ds3client/commands/PutMultiPartUploadPartRequest. must not be null"));
+            assertThat(e.getMessage(), startsWith("Argument for @Nonnull parameter '" + paramName.toLowerCase() + "'"));
         }
     }
 }

From 9fe72e5950606dca07f62628c3c3fe0be0d057f4 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 18 Sep 2017 11:06:48 -0600
Subject: [PATCH 107/229] Updated stale contract used for generation

---
 contract/4_0_0_contract.xml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/contract/4_0_0_contract.xml b/contract/4_0_0_contract.xml
index 939620807..8fd0958f5 100644
--- a/contract/4_0_0_contract.xml
+++ b/contract/4_0_0_contract.xml
@@ -4937,6 +4937,7 @@
                     
                         
                         
+                        
                         
                         
                         
@@ -4960,7 +4961,7 @@
                         
                     
                 
-                1.965E74D83E54746B6877B79D92E8768A
+                1.033F928F2F6A0DE70086DFE3C999B9B6
             
             
                 
@@ -7936,7 +7937,7 @@
                         
                     
                     
-                        410
+                        404
                         
                             
                         
@@ -12977,6 +12978,12 @@
                             
                         
                     
+                    
+                        
+                            
+                            
+                        
+                    
                 
             
             

From 0ec884ded43e65709b82cf5c5ec867187e6c0972 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Mon, 16 Oct 2017 17:02:15 -0600
Subject: [PATCH 108/229] Fixing a bug when reading files during put job with
 streamer strategy. Fixing some fractured english in error messages. A few
 cleanup details like transforming anonymous class implementations to lambdas.
 Updating kotlin version.

---
 build.gradle                                  |  2 +-
 .../integration/PutJobManagement_Test.java    | 72 ++++++++++++++++
 .../ContentLengthNotMatchException.java       |  2 +-
 .../PutSequentialBlobStrategy.java            | 13 ++-
 .../SeekableByteChannelDecorator.java         | 84 ++++++++++++++++---
 .../SequentialChannelStrategy.java            |  2 +-
 .../GetJobTransferMethod.java                 |  7 ++
 .../PutJobTransferMethod.java                 |  2 +
 .../networking/NetworkClientImpl.java         | 15 +---
 9 files changed, 166 insertions(+), 33 deletions(-)

diff --git a/build.gradle b/build.gradle
index 6d255998b..eacfd74e4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -14,7 +14,7 @@
  */
 
 buildscript {
-    ext.kotlin_version = '1.1.4-3'
+    ext.kotlin_version = '1.1.51'
 
     repositories {
         mavenCentral()
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
index 067bf9fb0..1eefab26e 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
@@ -52,7 +52,9 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedInputStream;
+import java.io.DataOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
@@ -66,9 +68,12 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
+import static com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request.MIN_UPLOAD_SIZE_IN_BYTES;
 import static com.spectralogic.ds3client.integration.Util.RESOURCE_BASE_NAME;
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
 import static org.hamcrest.Matchers.*;
@@ -2059,4 +2064,71 @@ public void testThatNonExistentFileDoesNotStopPutJob() throws IOException {
         assertTrue(caughtNoSuchFileException.get());
         assertTrue(getJobRan.get());
     }
+
+    @Test
+    public void testStreamedPutJobWithBlobbedFile() throws Exception {
+        final int chunkSize = MIN_UPLOAD_SIZE_IN_BYTES;
+        final long biggerThanAChunkSize = chunkSize * 2L + 1024;
+
+        final int numIntsInBiggerThanAChunkSize = (int)biggerThanAChunkSize / 4;
+
+        final String originalFileName = "Gracie.bin";
+        final String movedFileName = "Gracie.bak";
+
+        try {
+            final DataOutputStream originalFileStream = new DataOutputStream(new FileOutputStream(originalFileName));
+
+            byte[] bytes = new byte[4];
+
+            for (int i = 0; i < numIntsInBiggerThanAChunkSize; ++i) {
+                bytes[0] = (byte)i;
+                bytes[1] = (byte)(i >> 8);
+                bytes[2] = (byte)(i >> 16);
+                bytes[3] = (byte)(i >> 24);
+                originalFileStream.write(bytes);
+            }
+
+            originalFileStream.close();
+
+            final Ds3Object ds3Object = new Ds3Object();
+            ds3Object.setName(originalFileName);
+            ds3Object.setSize(biggerThanAChunkSize);
+
+            final AtomicLong numBytesTransferred = new AtomicLong(0);
+
+            final WriteJobOptions writeJobOptions = WriteJobOptions.create();
+            writeJobOptions.withMaxUploadSize(chunkSize);
+
+            final Ds3ClientHelpers.Job writeJob = HELPERS.startWriteJobUsingStreamedBehavior(BUCKET_NAME, Collections.singletonList(ds3Object), writeJobOptions);
+            writeJob.attachDataTransferredListener(numBytesTransferred::addAndGet);
+
+            final CountDownLatch writeCountDownLatch = new CountDownLatch(1);
+
+            writeJob.attachObjectCompletedListener(name -> writeCountDownLatch.countDown());
+
+            writeJob.transfer(new FileObjectPutter(Paths.get(".")));
+
+            writeCountDownLatch.await();
+
+            assertEquals(biggerThanAChunkSize, numBytesTransferred.get());
+
+            Files.move(Paths.get(originalFileName), Paths.get(movedFileName));
+
+            final CountDownLatch readCountdownLatch = new CountDownLatch(1);
+
+            final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME, Collections.singletonList(ds3Object));
+            readJob.withMaxParallelRequests(1);
+            readJob.attachObjectCompletedListener(name -> readCountdownLatch.countDown());
+            readJob.transfer(new FileObjectGetter(Paths.get(".")));
+
+            readCountdownLatch.await();
+
+            assertTrue(FileUtils.contentEquals(new File(movedFileName), new File(originalFileName)));
+        } finally {
+            deleteAllContents(client, BUCKET_NAME);
+
+            Files.deleteIfExists(Paths.get(originalFileName));
+            Files.deleteIfExists(Paths.get(movedFileName));
+        }
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/ContentLengthNotMatchException.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/ContentLengthNotMatchException.java
index 66a14f8a1..6d3e0e932 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/ContentLengthNotMatchException.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/exceptions/ContentLengthNotMatchException.java
@@ -22,7 +22,7 @@ public class ContentLengthNotMatchException extends IOException {
     private final long contentLength;
     private final long totalBytes;
     public ContentLengthNotMatchException(final String fileName, final long contentLength, final long totalBytes) {
-        super(String.format("The Content length for %s (%d) not match the number of byte read (%d)", fileName, contentLength, totalBytes));
+        super(String.format("The Content length for %s (%d) does not match the number of bytes read (%d)", fileName, contentLength, totalBytes));
 
         this.fileName = fileName;
         this.contentLength = contentLength;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
index 35478a302..78730b435 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
@@ -57,19 +57,16 @@ public PutSequentialBlobStrategy(final Ds3Client client,
     public Iterable getWork() throws IOException, InterruptedException {
         final Objects nextChunk = allocateChunk(chunksThatContainBlobs.next());
 
-        LOG.debug("Allocating chunk: {}", nextChunk.getChunkId().toString());
+        LOG.debug("==> Allocating chunk: {}", nextChunk.getChunkId().toString());
         return FluentIterable.from(nextChunk.getObjects())
-                .filter(new Predicate() {
-                    @Override
-                    public boolean apply(@Nullable final BulkObject input) {
-                        return !input.getInCache();
-                    }
-                })
+                .filter(input -> !input.getInCache())
                 .transform(new Function() {
                     @Nullable
                     @Override
                     public JobPart apply(@Nullable final BulkObject blob) {
-                        return new JobPart(client(), blob);
+                        final JobPart jobPart = new JobPart(client(), blob);
+                        LOG.debug("==> JobPart: {}", jobPart);
+                        return jobPart;
 
                         // TODO: When we get to the point where BP enables clustering, we'll want to be able to get the
                         // client connection info correct for the server on which a chunk resides. StrategyUtils.getClient
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index 2acdc4dab..f3771df0b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -21,56 +21,118 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.SeekableByteChannel;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * An instance of {@link SeekableByteChannel} used to decorate another SeekableByteChannel in the
  * situation where we re-use the same channel for more than 1 blob.  This subclass prevents closing
  * a channel when there are other blobs still referencing the shared channel.
  */
 class SeekableByteChannelDecorator implements SeekableByteChannel {
-    private final SeekableByteChannel seekableByteChannel;
+    private static final Logger LOG = LoggerFactory.getLogger(SeekableByteChannelDecorator.class);
 
-    SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel) {
-        Preconditions.checkNotNull(seekableByteChannel, "seekableByteChannel may not be null");
+    private final Object lock = new Object();
+
+    private final SeekableByteChannel seekableByteChannel;
+    private final long initialOffset;
+    private final long length;
+    private long position;
+
+    SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel, final long initialOffset, final long length) {
+        Preconditions.checkNotNull(seekableByteChannel, "seekableByteChannel may not be null.");
+        Preconditions.checkArgument(initialOffset >= 0, "initialOffset must be >= 0.");
+        Preconditions.checkArgument(length >= 0, "length must be >= 0.");
         this.seekableByteChannel = seekableByteChannel;
+        this.initialOffset = initialOffset;
+        this.position = initialOffset;
+        this.length = length;
+
+        LOG.debug("==> initialOffset: {}, position: {}, length: {}", initialOffset, position, length);
     }
 
-    protected SeekableByteChannel wrappedSeekableByteChannel() {
+    SeekableByteChannel wrappedSeekableByteChannel() {
         return seekableByteChannel;
     }
 
     @Override
     public int read(final ByteBuffer dst) throws IOException {
-        return seekableByteChannel.read(dst);
+        synchronized (lock) {
+            final long remainingInChannel = length - position;
+            LOG.debug("==> remainingInChannel: {},", remainingInChannel);
+            final long numBytesWeCanRead = Math.min(dst.remaining(), remainingInChannel);
+            LOG.debug("==> numBytesWeCanRead: {},", numBytesWeCanRead);
+
+            if (numBytesWeCanRead <= 0) {
+                return 0;
+            }
+
+            final int numBytesRead;
+
+            if (numBytesWeCanRead != dst.remaining()) {
+                final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[(int) numBytesWeCanRead]);
+                numBytesRead = seekableByteChannel.read(byteBuffer);
+                byteBuffer.flip();
+                dst.put(byteBuffer);
+            } else {
+                numBytesRead = seekableByteChannel.read(dst);
+            }
+
+            position += numBytesRead;
+
+            LOG.debug("==> numBytesRead: {}, position: {}", numBytesRead, position);
+
+            return numBytesRead;
+        }
     }
 
     @Override
     public int write(final ByteBuffer src) throws IOException {
-        return seekableByteChannel.write(src);
+        synchronized (lock) {
+            return seekableByteChannel.write(src);
+        }
     }
 
     @Override
     public long position() throws IOException {
-        return seekableByteChannel.position();
+        synchronized (lock) {
+            LOG.debug("get position: {}, seekableByteChannel.position: {}", position, seekableByteChannel.position());
+            return seekableByteChannel.position();
+        }
     }
 
     @Override
     public SeekableByteChannel position(final long newPosition) throws IOException {
-        return seekableByteChannel.position(newPosition);
+        synchronized (lock) {
+            final long lastPossiblePosition = length - 1;
+            position = Math.min(newPosition, lastPossiblePosition);
+            seekableByteChannel.position(initialOffset + position);
+
+            LOG.debug("==> set position: {}, seekableByteChannel.position: {}", position, seekableByteChannel.position());
+
+            return this;
+        }
     }
 
     @Override
     public long size() throws IOException {
-        return seekableByteChannel.size();
+        synchronized (lock) {
+            return seekableByteChannel.size();
+        }
     }
 
     @Override
     public SeekableByteChannel truncate(final long size) throws IOException {
-        return seekableByteChannel.truncate(size);
+        synchronized (lock) {
+            return seekableByteChannel.truncate(size);
+        }
     }
 
     @Override
     public boolean isOpen() {
-        return seekableByteChannel.isOpen();
+        synchronized (lock) {
+            return seekableByteChannel.isOpen();
+        }
     }
 
     @Override
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java
index 23a672967..d3b7974f0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SequentialChannelStrategy.java
@@ -88,7 +88,7 @@ private SeekableByteChannel makeNewChannel(final BulkObject blob) throws IOExcep
         channelPreparer.prepareChannel(blob.getName(), objectChannelBuilder);
 
         final SeekableByteChannel seekableByteChannel = channelStrategyDelegate.acquireChannelForBlob(blob);
-        final SeekableByteChannelDecorator seekableByteChannelDecorator = new SeekableByteChannelDecorator(seekableByteChannel);
+        final SeekableByteChannelDecorator seekableByteChannelDecorator = new SeekableByteChannelDecorator(seekableByteChannel, blob.getOffset(), blob.getLength());
 
         blobNameChannelMap.put(blob.getName(), seekableByteChannelDecorator);
         return seekableByteChannelDecorator;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
index 3a174145d..ecc64dd96 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
@@ -29,10 +29,15 @@
 import java.io.IOException;
 import java.nio.channels.SeekableByteChannel;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * The HTTP GET implementation class that retrieves a blob from a Black Pearl.
  */
 public class GetJobTransferMethod implements TransferMethod {
+    private static final Logger LOG = LoggerFactory.getLogger(GetJobTransferMethod.class);
+
     private final ChannelStrategy channelStrategy;
     private final String bucketName;
     private final String jobId;
@@ -67,6 +72,8 @@ public void transferJobPart(final JobPart jobPart) throws IOException {
 
         final BulkObject blob = jobPart.getBlob();
 
+        LOG.debug("==> transferJobPart: {}", blob);
+
         channelStrategy.releaseChannelForBlob(seekableByteChannel, blob);
 
         eventDispatcher.emitChecksumEvent(blob, getObjectResponse.getChecksumType(), getObjectResponse.getChecksum());
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
index 29520216c..48e6fb27a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
@@ -73,6 +73,8 @@ public PutJobTransferMethod(final ChannelStrategy channelStrategy,
     public void transferJobPart(final JobPart jobPart) throws IOException {
         final BulkObject blob = jobPart.getBlob();
 
+        LOG.debug("==> Transferring: {}", blob);
+
         final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(blob);
 
         jobPart.getClient().putObject(makePutObjectRequest(seekableByteChannel, jobPart));
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java
index 938be7c24..c1f6aba8a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/networking/NetworkClientImpl.java
@@ -59,8 +59,6 @@
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SignatureException;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
 import java.util.Map;
 
 import static com.spectralogic.ds3client.utils.Signature.canonicalizeAmzHeaders;
@@ -93,7 +91,7 @@ public NetworkClientImpl(final ConnectionDetails connectionDetails) {
         this(connectionDetails, createDefaultClient(connectionDetails));
     }
 
-    public NetworkClientImpl(final ConnectionDetails connectionDetails, final CloseableHttpClient client) {
+    private NetworkClientImpl(final ConnectionDetails connectionDetails, final CloseableHttpClient client) {
         if (connectionDetails == null) throw new AssertionError("ConnectionDetails cannot be null");
         if (client == null) throw new AssertionError("CloseableHttpClient cannot be null");
         try {
@@ -125,12 +123,7 @@ private static CloseableHttpClient createDefaultClient(final ConnectionDetails c
     private static CloseableHttpClient createInsecureSslHttpClient() throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException {
         final SSLContext sslContext = new SSLContextBuilder()
                 .useProtocol(INSECURE_SSL_PROTOCOL)
-                .loadTrustMaterial(null, new TrustStrategy() {
-            @Override
-            public boolean isTrusted(final X509Certificate[] chain, final String authType) throws CertificateException {
-                return true;
-            }
-        }).build();
+                .loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true).build();
         final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new NoopHostnameVerifier());
 
         final Registry socketFactoryRegistry = RegistryBuilder.create()
@@ -215,7 +208,7 @@ private class RequestExecutor implements Closeable {
         private final ChecksumType.Type checksumType;
         private final CloseableHttpClient client;
 
-        public RequestExecutor(final CloseableHttpClient client, final HttpHost host, final Ds3Request ds3Request) throws IOException {
+        RequestExecutor(final CloseableHttpClient client, final HttpHost host, final Ds3Request ds3Request) throws IOException {
             this.client = client;
             this.ds3Request = ds3Request;
             this.host = host;
@@ -230,7 +223,7 @@ public RequestExecutor(final CloseableHttpClient client, final HttpHost host, fi
             this.hash = this.buildHash();
         }
 
-        public CloseableHttpResponse execute() throws IOException {
+        CloseableHttpResponse execute() throws IOException {
             if (this.content != null) {
                 this.content.reset();
             }

From 6f336923046d409b0dbd3921e8e1a35334845a63 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 17 Oct 2017 11:42:07 -0600
Subject: [PATCH 109/229] Fixing a bug when reading files during put job with
 streamer strategy. The root of the bug is an inconsistency in positioning
 within a stream depending on the path taken in getting a stream.

---
 .../integration/GetJobManagement_Test.java    | 71 +++++++++++++++++++
 .../integration/PutJobManagement_Test.java    |  1 -
 .../PutSequentialBlobStrategy.java            |  2 -
 .../SeekableByteChannelDecorator.java         | 36 ++++++----
 .../GetJobTransferMethod.java                 |  2 -
 .../PutJobTransferMethod.java                 |  2 -
 6 files changed, 95 insertions(+), 19 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
index 5af9f2c41..5790c994e 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java
@@ -41,6 +41,7 @@
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
+import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy;
 import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryBehavior;
 import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryDelayBehavior;
@@ -83,7 +84,9 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
+import java.io.DataOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -101,11 +104,15 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
+import static com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request.MIN_UPLOAD_SIZE_IN_BYTES;
 import static com.spectralogic.ds3client.integration.Util.RESOURCE_BASE_NAME;
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
 import static com.spectralogic.ds3client.integration.Util.deleteBucketContents;
@@ -1449,4 +1456,68 @@ public void testThatFifoIsNotProcessed() throws IOException, InterruptedExceptio
 
         assertTrue(caughtException.get());
     }
+
+    @Test
+    public void testStreamedGetJobWithBlobbedFile() throws Exception {
+        final int chunkSize = MIN_UPLOAD_SIZE_IN_BYTES;
+        final long biggerThanAChunkSize = chunkSize * 2L + 1024;
+
+        final int numIntsInBiggerThanAChunkSize = (int)biggerThanAChunkSize / 4;
+
+        final String originalFileName = "Gracie.bin";
+        final String movedFileName = "Gracie.bak";
+
+        try {
+            final DataOutputStream originalFileStream = new DataOutputStream(new FileOutputStream(originalFileName));
+
+            byte[] bytes = new byte[4];
+
+            for (int i = 0; i < numIntsInBiggerThanAChunkSize; ++i) {
+                bytes[0] = (byte)i;
+                bytes[1] = (byte)(i >> 8);
+                bytes[2] = (byte)(i >> 16);
+                bytes[3] = (byte)(i >> 24);
+                originalFileStream.write(bytes);
+            }
+
+            originalFileStream.close();
+
+            final Ds3Object ds3Object = new Ds3Object();
+            ds3Object.setName(originalFileName);
+            ds3Object.setSize(biggerThanAChunkSize);
+
+            final AtomicLong numBytesTransferred = new AtomicLong(0);
+
+            final WriteJobOptions writeJobOptions = WriteJobOptions.create();
+            writeJobOptions.withMaxUploadSize(chunkSize);
+
+            final Ds3ClientHelpers.Job writeJob = HELPERS.startWriteJob(BUCKET_NAME, Collections.singletonList(ds3Object), writeJobOptions);
+            writeJob.attachDataTransferredListener(numBytesTransferred::addAndGet);
+
+            final CountDownLatch writeCountDownLatch = new CountDownLatch(1);
+
+            writeJob.attachObjectCompletedListener(name -> writeCountDownLatch.countDown());
+
+            writeJob.transfer(new FileObjectPutter(Paths.get(".")));
+
+            writeCountDownLatch.await();
+
+            assertEquals(biggerThanAChunkSize, numBytesTransferred.get());
+
+            Files.move(Paths.get(originalFileName), Paths.get(movedFileName));
+
+            final CountDownLatch readCountdownLatch = new CountDownLatch(1);
+
+            final Ds3ClientHelpers.Job readJob = HELPERS.startReadJobUsingStreamedBehavior(BUCKET_NAME, Collections.singletonList(ds3Object));
+            readJob.attachObjectCompletedListener(name -> readCountdownLatch.countDown());
+            readJob.transfer(new FileObjectGetter(Paths.get(".")));
+
+            readCountdownLatch.await();
+
+            assertTrue(FileUtils.contentEquals(new File(movedFileName), new File(originalFileName)));
+        } finally {
+            Files.deleteIfExists(Paths.get(originalFileName));
+            Files.deleteIfExists(Paths.get(movedFileName));
+        }
+    }
 }
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
index 1eefab26e..cfdad62ab 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
@@ -2117,7 +2117,6 @@ public void testStreamedPutJobWithBlobbedFile() throws Exception {
             final CountDownLatch readCountdownLatch = new CountDownLatch(1);
 
             final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME, Collections.singletonList(ds3Object));
-            readJob.withMaxParallelRequests(1);
             readJob.attachObjectCompletedListener(name -> readCountdownLatch.countDown());
             readJob.transfer(new FileObjectGetter(Paths.get(".")));
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
index 78730b435..c9c06006a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
@@ -57,7 +57,6 @@ public PutSequentialBlobStrategy(final Ds3Client client,
     public Iterable getWork() throws IOException, InterruptedException {
         final Objects nextChunk = allocateChunk(chunksThatContainBlobs.next());
 
-        LOG.debug("==> Allocating chunk: {}", nextChunk.getChunkId().toString());
         return FluentIterable.from(nextChunk.getObjects())
                 .filter(input -> !input.getInCache())
                 .transform(new Function() {
@@ -65,7 +64,6 @@ public Iterable getWork() throws IOException, InterruptedException {
                     @Override
                     public JobPart apply(@Nullable final BulkObject blob) {
                         final JobPart jobPart = new JobPart(client(), blob);
-                        LOG.debug("==> JobPart: {}", jobPart);
                         return jobPart;
 
                         // TODO: When we get to the point where BP enables clustering, we'll want to be able to get the
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index f3771df0b..16a9f7217 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -37,18 +37,17 @@ class SeekableByteChannelDecorator implements SeekableByteChannel {
     private final SeekableByteChannel seekableByteChannel;
     private final long initialOffset;
     private final long length;
-    private long position;
+    private long position = 0;
 
-    SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel, final long initialOffset, final long length) {
+    SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel, final long initialOffset, final long length) throws IOException {
         Preconditions.checkNotNull(seekableByteChannel, "seekableByteChannel may not be null.");
         Preconditions.checkArgument(initialOffset >= 0, "initialOffset must be >= 0.");
         Preconditions.checkArgument(length >= 0, "length must be >= 0.");
         this.seekableByteChannel = seekableByteChannel;
         this.initialOffset = initialOffset;
-        this.position = initialOffset;
         this.length = length;
 
-        LOG.debug("==> initialOffset: {}, position: {}, length: {}", initialOffset, position, length);
+        seekableByteChannel.position(initialOffset);
     }
 
     SeekableByteChannel wrappedSeekableByteChannel() {
@@ -59,9 +58,7 @@ SeekableByteChannel wrappedSeekableByteChannel() {
     public int read(final ByteBuffer dst) throws IOException {
         synchronized (lock) {
             final long remainingInChannel = length - position;
-            LOG.debug("==> remainingInChannel: {},", remainingInChannel);
             final long numBytesWeCanRead = Math.min(dst.remaining(), remainingInChannel);
-            LOG.debug("==> numBytesWeCanRead: {},", numBytesWeCanRead);
 
             if (numBytesWeCanRead <= 0) {
                 return 0;
@@ -80,8 +77,6 @@ public int read(final ByteBuffer dst) throws IOException {
 
             position += numBytesRead;
 
-            LOG.debug("==> numBytesRead: {}, position: {}", numBytesRead, position);
-
             return numBytesRead;
         }
     }
@@ -89,14 +84,33 @@ public int read(final ByteBuffer dst) throws IOException {
     @Override
     public int write(final ByteBuffer src) throws IOException {
         synchronized (lock) {
-            return seekableByteChannel.write(src);
+            final long remainingInChannel = length - position;
+            final long numBytesWeCanWrite = Math.min(src.remaining(), remainingInChannel);
+
+            if (numBytesWeCanWrite <= 0) {
+                return 0;
+            }
+
+            final int numBytesWritten;
+
+            if (numBytesWeCanWrite != src.remaining()) {
+                final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[(int) numBytesWeCanWrite]);
+                byteBuffer.put(src);
+                byteBuffer.flip();
+                numBytesWritten = seekableByteChannel.write(byteBuffer);
+            } else {
+                numBytesWritten = seekableByteChannel.write(src);
+            }
+
+            position += numBytesWritten;
+
+            return numBytesWritten;
         }
     }
 
     @Override
     public long position() throws IOException {
         synchronized (lock) {
-            LOG.debug("get position: {}, seekableByteChannel.position: {}", position, seekableByteChannel.position());
             return seekableByteChannel.position();
         }
     }
@@ -108,8 +122,6 @@ public SeekableByteChannel position(final long newPosition) throws IOException {
             position = Math.min(newPosition, lastPossiblePosition);
             seekableByteChannel.position(initialOffset + position);
 
-            LOG.debug("==> set position: {}, seekableByteChannel.position: {}", position, seekableByteChannel.position());
-
             return this;
         }
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
index ecc64dd96..8d1642ef6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
@@ -72,8 +72,6 @@ public void transferJobPart(final JobPart jobPart) throws IOException {
 
         final BulkObject blob = jobPart.getBlob();
 
-        LOG.debug("==> transferJobPart: {}", blob);
-
         channelStrategy.releaseChannelForBlob(seekableByteChannel, blob);
 
         eventDispatcher.emitChecksumEvent(blob, getObjectResponse.getChecksumType(), getObjectResponse.getChecksum());
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
index 48e6fb27a..29520216c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/PutJobTransferMethod.java
@@ -73,8 +73,6 @@ public PutJobTransferMethod(final ChannelStrategy channelStrategy,
     public void transferJobPart(final JobPart jobPart) throws IOException {
         final BulkObject blob = jobPart.getBlob();
 
-        LOG.debug("==> Transferring: {}", blob);
-
         final SeekableByteChannel seekableByteChannel = channelStrategy.acquireChannelForBlob(blob);
 
         jobPart.getClient().putObject(makePutObjectRequest(seekableByteChannel, jobPart));

From fd0eded331ad5af5c9ed3d850c35775c9f3f820a Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 17 Oct 2017 11:48:51 -0600
Subject: [PATCH 110/229] Removing a logging artifact.

---
 .../strategy/blobstrategy/PutSequentialBlobStrategy.java       | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
index c9c06006a..163e1d90c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/PutSequentialBlobStrategy.java
@@ -63,8 +63,7 @@ public Iterable getWork() throws IOException, InterruptedException {
                     @Nullable
                     @Override
                     public JobPart apply(@Nullable final BulkObject blob) {
-                        final JobPart jobPart = new JobPart(client(), blob);
-                        return jobPart;
+                        return new JobPart(client(), blob);
 
                         // TODO: When we get to the point where BP enables clustering, we'll want to be able to get the
                         // client connection info correct for the server on which a chunk resides. StrategyUtils.getClient

From a1211d909611fd97637f3041a28f4afa92eb7d3c Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 17 Oct 2017 11:52:20 -0600
Subject: [PATCH 111/229] Removing a logging artifact.

---
 .../channelstrategy/SeekableByteChannelDecorator.java        | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index 16a9f7217..02193bacc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -21,17 +21,12 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.SeekableByteChannel;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 /**
  * An instance of {@link SeekableByteChannel} used to decorate another SeekableByteChannel in the
  * situation where we re-use the same channel for more than 1 blob.  This subclass prevents closing
  * a channel when there are other blobs still referencing the shared channel.
  */
 class SeekableByteChannelDecorator implements SeekableByteChannel {
-    private static final Logger LOG = LoggerFactory.getLogger(SeekableByteChannelDecorator.class);
-
     private final Object lock = new Object();
 
     private final SeekableByteChannel seekableByteChannel;

From f1aa355ca0009562bee2bad8f4db0f7fa456190b Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 17 Oct 2017 11:55:29 -0600
Subject: [PATCH 112/229] Removing a logging artifact.

---
 .../strategy/transferstrategy/GetJobTransferMethod.java      | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
index 8d1642ef6..3a174145d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
@@ -29,15 +29,10 @@
 import java.io.IOException;
 import java.nio.channels.SeekableByteChannel;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 /**
  * The HTTP GET implementation class that retrieves a blob from a Black Pearl.
  */
 public class GetJobTransferMethod implements TransferMethod {
-    private static final Logger LOG = LoggerFactory.getLogger(GetJobTransferMethod.class);
-
     private final ChannelStrategy channelStrategy;
     private final String bucketName;
     private final String jobId;

From 18924425ae1f21e7422d6f3d1a92d085ad595da7 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 18 Oct 2017 11:02:47 -0600
Subject: [PATCH 113/229] Renaming offset & length to better reflect they're
 tied to the blob.

---
 .../SeekableByteChannelDecorator.java         | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index 02193bacc..a606ab18b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -30,19 +30,19 @@ class SeekableByteChannelDecorator implements SeekableByteChannel {
     private final Object lock = new Object();
 
     private final SeekableByteChannel seekableByteChannel;
-    private final long initialOffset;
-    private final long length;
+    private final long blobOffset;
+    private final long blobLength;
     private long position = 0;
 
-    SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel, final long initialOffset, final long length) throws IOException {
+    SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel, final long blobOffset, final long blobLength) throws IOException {
         Preconditions.checkNotNull(seekableByteChannel, "seekableByteChannel may not be null.");
-        Preconditions.checkArgument(initialOffset >= 0, "initialOffset must be >= 0.");
-        Preconditions.checkArgument(length >= 0, "length must be >= 0.");
+        Preconditions.checkArgument(blobOffset >= 0, "blobOffset must be >= 0.");
+        Preconditions.checkArgument(blobLength >= 0, "length must be >= 0.");
         this.seekableByteChannel = seekableByteChannel;
-        this.initialOffset = initialOffset;
-        this.length = length;
+        this.blobOffset = blobOffset;
+        this.blobLength = blobLength;
 
-        seekableByteChannel.position(initialOffset);
+        seekableByteChannel.position(blobOffset);
     }
 
     SeekableByteChannel wrappedSeekableByteChannel() {
@@ -52,7 +52,7 @@ SeekableByteChannel wrappedSeekableByteChannel() {
     @Override
     public int read(final ByteBuffer dst) throws IOException {
         synchronized (lock) {
-            final long remainingInChannel = length - position;
+            final long remainingInChannel = blobLength - position;
             final long numBytesWeCanRead = Math.min(dst.remaining(), remainingInChannel);
 
             if (numBytesWeCanRead <= 0) {
@@ -79,7 +79,7 @@ public int read(final ByteBuffer dst) throws IOException {
     @Override
     public int write(final ByteBuffer src) throws IOException {
         synchronized (lock) {
-            final long remainingInChannel = length - position;
+            final long remainingInChannel = blobLength - position;
             final long numBytesWeCanWrite = Math.min(src.remaining(), remainingInChannel);
 
             if (numBytesWeCanWrite <= 0) {
@@ -113,9 +113,9 @@ public long position() throws IOException {
     @Override
     public SeekableByteChannel position(final long newPosition) throws IOException {
         synchronized (lock) {
-            final long lastPossiblePosition = length - 1;
+            final long lastPossiblePosition = blobLength - 1;
             position = Math.min(newPosition, lastPossiblePosition);
-            seekableByteChannel.position(initialOffset + position);
+            seekableByteChannel.position(blobOffset + position);
 
             return this;
         }

From 497ddc37f558dbd271de86a710161dedd4b9436c Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 18 Oct 2017 11:07:23 -0600
Subject: [PATCH 114/229] Renaming offset & length to better reflect they're
 tied to the blob.

---
 .../strategy/channelstrategy/SeekableByteChannelDecorator.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index a606ab18b..53ade55c1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -37,7 +37,7 @@ class SeekableByteChannelDecorator implements SeekableByteChannel {
     SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel, final long blobOffset, final long blobLength) throws IOException {
         Preconditions.checkNotNull(seekableByteChannel, "seekableByteChannel may not be null.");
         Preconditions.checkArgument(blobOffset >= 0, "blobOffset must be >= 0.");
-        Preconditions.checkArgument(blobLength >= 0, "length must be >= 0.");
+        Preconditions.checkArgument(blobLength >= 0, "blobLength must be >= 0.");
         this.seekableByteChannel = seekableByteChannel;
         this.blobOffset = blobOffset;
         this.blobLength = blobLength;

From dec2c3fbb8aae0d02d6f397e8be8ce59848a507b Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 18 Oct 2017 11:12:33 -0600
Subject: [PATCH 115/229] Documentation

---
 .../strategy/channelstrategy/SeekableByteChannelDecorator.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index 53ade55c1..6e8e518c9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -25,6 +25,9 @@
  * An instance of {@link SeekableByteChannel} used to decorate another SeekableByteChannel in the
  * situation where we re-use the same channel for more than 1 blob.  This subclass prevents closing
  * a channel when there are other blobs still referencing the shared channel.
+ *
+ * This class positions the content of a blob within the bounds of a channel that may be capable
+ * of containing more than one blob.
  */
 class SeekableByteChannelDecorator implements SeekableByteChannel {
     private final Object lock = new Object();

From 0bc1044af4e96027a6d3163b605de84d12f2ab81 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 18 Oct 2017 11:27:47 -0600
Subject: [PATCH 116/229] Renaming stuff to make it more clear what they
 represent.

---
 .../channelstrategy/SeekableByteChannelDecorator.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index 6e8e518c9..6e86b567b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -55,8 +55,8 @@ SeekableByteChannel wrappedSeekableByteChannel() {
     @Override
     public int read(final ByteBuffer dst) throws IOException {
         synchronized (lock) {
-            final long remainingInChannel = blobLength - position;
-            final long numBytesWeCanRead = Math.min(dst.remaining(), remainingInChannel);
+            final long remainingInWindow = blobLength - position;
+            final long numBytesWeCanRead = Math.min(dst.remaining(), remainingInWindow);
 
             if (numBytesWeCanRead <= 0) {
                 return 0;
@@ -82,8 +82,8 @@ public int read(final ByteBuffer dst) throws IOException {
     @Override
     public int write(final ByteBuffer src) throws IOException {
         synchronized (lock) {
-            final long remainingInChannel = blobLength - position;
-            final long numBytesWeCanWrite = Math.min(src.remaining(), remainingInChannel);
+            final long remainingInWindow = blobLength - position;
+            final long numBytesWeCanWrite = Math.min(src.remaining(), remainingInWindow);
 
             if (numBytesWeCanWrite <= 0) {
                 return 0;

From c0a0b31f4f9ef908d5440601ea7806605a3d26df Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 18 Oct 2017 12:14:12 -0600
Subject: [PATCH 117/229] Fixing a typo

---
 SETUP.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SETUP.md b/SETUP.md
index 82c281663..ae0f77fde 100644
--- a/SETUP.md
+++ b/SETUP.md
@@ -13,7 +13,7 @@ If using Eclipse:
 * In the results that are returned install `Gradle Integration for Eclipse (4.4) 3.7.1.RELEASE` or current version.  Clicking `Install` will take you to the 'Confirm Selected Features' dialog.  (De-select the two optional Spring checkboxes on older versions).  `Gradle IDE` and `org.gradle.toolingapi.feature` should both be selected.  Accept any dialogs that popup.
 * After Gradle has been installed into eclipse and git has been installed, you should be able to clone the repo and import that project into eclipse
 
-If using Intelllij:
+If using Intellij:
 * Open Intellij and select `Import Project`
 * Find the `build.gradle` file contained at the root of the project and select it
 * Accept the defaults

From 2959126b2939c619eea3ae54089320f0dc14d899 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 18 Oct 2017 12:39:53 -0600
Subject: [PATCH 118/229] Renaming stuff to make it more clear what they
 represent.

---
 .../channelstrategy/SeekableByteChannelDecorator.java         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index 6e86b567b..4f6c5b4ca 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -116,8 +116,8 @@ public long position() throws IOException {
     @Override
     public SeekableByteChannel position(final long newPosition) throws IOException {
         synchronized (lock) {
-            final long lastPossiblePosition = blobLength - 1;
-            position = Math.min(newPosition, lastPossiblePosition);
+            final long greatestPossiblePosition = blobLength - 1;
+            position = Math.min(newPosition, greatestPossiblePosition);
             seekableByteChannel.position(blobOffset + position);
 
             return this;

From 57feccbefa10f4e3d97f362eed0c7a4c91ec74eb Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 18 Oct 2017 13:39:12 -0600
Subject: [PATCH 119/229] Renaming stuff to make it more clear what they
 represent.

---
 .../SeekableByteChannelDecorator.java              | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
index 4f6c5b4ca..9196caff4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/channelstrategy/SeekableByteChannelDecorator.java
@@ -35,7 +35,7 @@ class SeekableByteChannelDecorator implements SeekableByteChannel {
     private final SeekableByteChannel seekableByteChannel;
     private final long blobOffset;
     private final long blobLength;
-    private long position = 0;
+    private long nextAvailableByteOffset = 0;
 
     SeekableByteChannelDecorator(final SeekableByteChannel seekableByteChannel, final long blobOffset, final long blobLength) throws IOException {
         Preconditions.checkNotNull(seekableByteChannel, "seekableByteChannel may not be null.");
@@ -55,7 +55,7 @@ SeekableByteChannel wrappedSeekableByteChannel() {
     @Override
     public int read(final ByteBuffer dst) throws IOException {
         synchronized (lock) {
-            final long remainingInWindow = blobLength - position;
+            final long remainingInWindow = blobLength - nextAvailableByteOffset;
             final long numBytesWeCanRead = Math.min(dst.remaining(), remainingInWindow);
 
             if (numBytesWeCanRead <= 0) {
@@ -73,7 +73,7 @@ public int read(final ByteBuffer dst) throws IOException {
                 numBytesRead = seekableByteChannel.read(dst);
             }
 
-            position += numBytesRead;
+            nextAvailableByteOffset += numBytesRead;
 
             return numBytesRead;
         }
@@ -82,7 +82,7 @@ public int read(final ByteBuffer dst) throws IOException {
     @Override
     public int write(final ByteBuffer src) throws IOException {
         synchronized (lock) {
-            final long remainingInWindow = blobLength - position;
+            final long remainingInWindow = blobLength - nextAvailableByteOffset;
             final long numBytesWeCanWrite = Math.min(src.remaining(), remainingInWindow);
 
             if (numBytesWeCanWrite <= 0) {
@@ -100,7 +100,7 @@ public int write(final ByteBuffer src) throws IOException {
                 numBytesWritten = seekableByteChannel.write(src);
             }
 
-            position += numBytesWritten;
+            nextAvailableByteOffset += numBytesWritten;
 
             return numBytesWritten;
         }
@@ -117,8 +117,8 @@ public long position() throws IOException {
     public SeekableByteChannel position(final long newPosition) throws IOException {
         synchronized (lock) {
             final long greatestPossiblePosition = blobLength - 1;
-            position = Math.min(newPosition, greatestPossiblePosition);
-            seekableByteChannel.position(blobOffset + position);
+            nextAvailableByteOffset = Math.min(newPosition, greatestPossiblePosition);
+            seekableByteChannel.position(blobOffset + nextAvailableByteOffset);
 
             return this;
         }

From dae1f50e45ed644b91dc404ebfabcb33926e2825 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 5 Dec 2017 13:40:36 -0700
Subject: [PATCH 120/229] Partial bob ranges are not relative to blobs, as I
 had been told.

---
 .../GetJobNetworkFailureRetryDecorator.java      | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java
index b80ca778c..737151477 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobNetworkFailureRetryDecorator.java
@@ -16,6 +16,7 @@
 package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
 
 import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMultimap;
 import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException;
@@ -25,6 +26,7 @@
 import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy;
 import com.spectralogic.ds3client.models.BulkObject;
 import com.spectralogic.ds3client.models.common.Range;
+import com.spectralogic.ds3client.utils.Guard;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -125,12 +127,24 @@ private ImmutableCollection initializeRanges(final BulkObject blob, final
 
         if (ranges == null) {
             final long numBytesTransferred = 0;
-            ranges = updateRanges(ranges, numBytesTransferred, blob.getLength());
+            ranges = adjustRangesForBlobOffset(updateRanges(ranges, numBytesTransferred, blob.getLength()), blob);
         }
 
         return ranges;
     }
 
+    private ImmutableCollection adjustRangesForBlobOffset(final ImmutableCollection ranges, final BulkObject blob) {
+        if (Guard.isNullOrEmpty(ranges) || ranges.size() > 1) {
+            return ranges;
+        }
+
+        final Range firstRange = ranges.iterator().next();
+
+        final long blobOffset = blob.getOffset();
+
+        return ImmutableList.of(new Range(firstRange.getStart() + blobOffset, firstRange.getEnd() + blobOffset));
+    }
+
     private ImmutableCollection updateRanges(final ImmutableCollection ranges,
                                                     final long numBytesTransferred,
                                                     final Long numBytesToTransfer)

From 3a545c5898a5ebe9559992c0dc4567ec2f1333c7 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 6 Dec 2017 14:29:44 -0700
Subject: [PATCH 121/229] Fix for
 https://jira.spectralogic.com/browse/JSDK-268. Putting git commit hash into
 ds3-sdk.properties file.

---
 build.gradle                                  |  4 ++
 ds3-sdk/build.gradle                          |  3 ++
 .../ds3client/utils/PropertyUtils.java        | 40 ++++++++++++++-----
 .../ds3client/Ds3Client_Test.java             |  9 +++++
 4 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/build.gradle b/build.gradle
index eacfd74e4..1b9e50da4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -29,6 +29,10 @@ buildscript {
 }
 
 
+plugins {
+    id "com.palantir.git-version" version "0.10.0"
+}
+
 allprojects {
     group = 'com.spectralogic.ds3'
     version = '4.0.1'
diff --git a/ds3-sdk/build.gradle b/ds3-sdk/build.gradle
index a683f7683..ec84ec7dc 100644
--- a/ds3-sdk/build.gradle
+++ b/ds3-sdk/build.gradle
@@ -58,10 +58,13 @@ task genConfigProperties {
             Files.createDirectories(configPath)
         }
 
+        def gitInfo = versionDetails()
+
         configFile.withWriter{out ->
             out.writeLine("productionBuild=" + productionBuild.toString())
             out.writeLine("version=" + version.toString())
             out.writeLine("build.date=" + new Date().toString())
+            out.writeLine("git.commitHash=" + gitInfo.gitHashFull)
         }
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
index 243693ba5..93fbf58e9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/PropertyUtils.java
@@ -26,9 +26,11 @@ public final class PropertyUtils {
     private static final Logger LOG = LoggerFactory.getLogger(PropertyUtils.class);
 
     private static final String SDK_VERSION_PROPERTY_NAME = "version";
+    private static final String GIT_COMMIT_HASH_PROPERTY_NAME = "git.commitHash";
     private static final String PROPERTIES_FILE_NAME = "ds3_sdk.properties";
 
     private static final AtomicReference versionProperty = new AtomicReference<>("");
+    private static final AtomicReference gitCommitHashProperty = new AtomicReference<>("");
 
     private PropertyUtils() {}
 
@@ -37,29 +39,47 @@ private PropertyUtils() {}
      * @return The sdk version, if we can find one, an empty string otherwise.
      */
     public static String getSdkVersion() {
-        if ( ! Guard.isStringNullOrEmpty(versionProperty.get())) {
-            return versionProperty.get();
+        return getPropertyFromPropertiesFile(SDK_VERSION_PROPERTY_NAME, versionProperty);
+    }
+
+    private static String getPropertyFromPropertiesFile(final String propertyName, final AtomicReference propertyValue) {
+        if ( ! Guard.isStringNullOrEmpty(propertyValue.get())) {
+            return propertyValue.get();
+        }
+
+        final String propertyFromPropertiesFile = getPropertyFromPropertiesFile(propertyName);
+
+        if ( ! Guard.isStringNullOrEmpty(propertyFromPropertiesFile)) {
+            propertyValue.set(propertyFromPropertiesFile);
         }
 
+        return propertyValue.get();
+    }
+
+    private static String getPropertyFromPropertiesFile(final String propertyName) {
         final Properties properties = new Properties();
 
         try (final InputStream propertiesStream = PropertyUtils.class.getClassLoader().getResourceAsStream(PROPERTIES_FILE_NAME)) {
             if (propertiesStream != null) {
                 properties.load(propertiesStream);
-                final Object versionPropertyObject = properties.get(SDK_VERSION_PROPERTY_NAME);
-
-                if (versionPropertyObject != null) {
-                    final String versionFromPropFile = properties.get(SDK_VERSION_PROPERTY_NAME).toString();
+                final Object propertyObject = properties.get(propertyName);
 
-                    if ( ! Guard.isStringNullOrEmpty(versionFromPropFile)) {
-                        versionProperty.set(versionFromPropFile);
-                    }
+                if (propertyObject != null) {
+                    return properties.get(propertyName).toString();
                 }
             }
         } catch (final Throwable t) {
             LOG.warn("Could not read properties file.", t);
         }
 
-        return versionProperty.get();
+        return "";
+    }
+
+    /**
+     * Get the git commit hash from a properties file.
+     * @return The git commit hash, if we can find it, an empty string otherwise.
+     */
+    public static String getGitCommitHash() {
+        return getPropertyFromPropertiesFile(GIT_COMMIT_HASH_PROPERTY_NAME, gitCommitHashProperty);
     }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 7c83007fb..a9809752b 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -31,6 +31,7 @@
 import com.spectralogic.ds3client.networking.FailedRequestUsingMgmtPortException;
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
+import com.spectralogic.ds3client.utils.PropertyUtils;
 import com.spectralogic.ds3client.utils.ResourceUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -51,6 +52,7 @@
 import static org.hamcrest.CoreMatchers.*;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 public class Ds3Client_Test {
@@ -937,6 +939,13 @@ public void testGettingDefaultUserAgent() {
         assertThat(matcher.find(), is(true));
     }
 
+    @Test
+    public void testGettingGitCommitHash() {
+        final String gitCommitHash = PropertyUtils.getGitCommitHash();
+
+        assertEquals(40, gitCommitHash.length());
+    }
+
     @Test
     public void VerifySystemHealthSpectraS3() throws IOException {
         final String responsePayload = "0";

From 94c772aa3842bbb4949cc7b12e0f4f9c6705d5d9 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Thu, 7 Dec 2017 11:15:37 -0700
Subject: [PATCH 122/229] Adding ltfs header prefix to metadata.

---
 .../ds3client/commands/MetadataImpl_Test.java | 24 +++++++++++++++++++
 .../commands/interfaces/MetadataImpl.java     | 20 ++++++++++++----
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/MetadataImpl_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/MetadataImpl_Test.java
index 8d3686352..053ec0444 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/MetadataImpl_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/MetadataImpl_Test.java
@@ -68,6 +68,30 @@ public void getMultiValueWithSpace() {
         assertThat(value.get(2), is("value3"));
     }
 
+    @Test
+    public void getSingleValueLtfs() {
+        final String userGuid = "060a2b340101010101010f0013-000000-709e29c2d1e20085-e7610015b2a9-a854";
+        final Metadata metadata = genMetadata(new BasicHeader("x-spectra-ltfs-user.guid", userGuid));
+        final List userGuids = metadata.get("user.guid");
+        assertThat(userGuids, is(notNullValue()));
+        assertFalse(userGuids.isEmpty());
+        assertThat(userGuids.size(), is(1));
+        assertThat(userGuids.get(0), is(userGuid));
+    }
+
+    @Test
+    public void getTwoValuesLtfs() {
+        final String userGuid1 = "060a2b340101010101010f0013-000000-709e29c2d1e20085-e7610015b2a9-a854";
+        final String userGuid2 = "160a2b340101010101010f0013-000000-709e29c2d1e20085-e7610015b2a9-a854";
+        final Metadata metadata = genMetadata(new BasicHeader("x-spectra-ltfs-user.guid", userGuid1 + "," + userGuid2));
+        final List userGuids = metadata.get("user.guid");
+        assertThat(userGuids, is(notNullValue()));
+        assertFalse(userGuids.isEmpty());
+        assertThat(userGuids.size(), is(2));
+        assertThat(userGuids.get(0), is(userGuid1));
+        assertThat(userGuids.get(1), is(userGuid2));
+    }
+
     private Metadata genMetadata(final Header... headers) {
 
         final ImmutableMultimap.Builder mapBuilder = ImmutableMultimap.builder();
diff --git a/ds3-utils/src/main/java/com/spectralogic/ds3client/commands/interfaces/MetadataImpl.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/commands/interfaces/MetadataImpl.java
index f5d6cc440..6fef3de88 100644
--- a/ds3-utils/src/main/java/com/spectralogic/ds3client/commands/interfaces/MetadataImpl.java
+++ b/ds3-utils/src/main/java/com/spectralogic/ds3client/commands/interfaces/MetadataImpl.java
@@ -30,7 +30,8 @@
 public class MetadataImpl implements Metadata {
 
     private static final String X_AMZ_META = "x-amz-meta-";
-    final ImmutableMultimap metadata;
+    private static final String LTFS_METADATA_PREFIX = "x-spectra-ltfs-";
+    private final ImmutableMultimap metadata;
 
     public MetadataImpl(final Headers headers) {
         this.metadata  = genMetadata(headers);
@@ -45,17 +46,27 @@ private static ImmutableMultimap genMetadata(final Headers heade
         final ImmutableMultimap.Builder mapBuilder = ImmutableMultimap.builder();
 
         for (final String key : headers.keys()) {
-            if (key.startsWith(X_AMZ_META)) {
-                final String name = toDecodedString(key.substring(X_AMZ_META.length()));
+            final String keyWithoutPrefix = metadataKey(key);
 
+            if ( ! keyWithoutPrefix.isEmpty()) {
                 final List values = getValues(headers, key);
-                mapBuilder.putAll(name, values);
+                mapBuilder.putAll(keyWithoutPrefix, values);
             }
         }
 
         return mapBuilder.build();
     }
 
+    private static String metadataKey(final String key) {
+        if (key.startsWith(X_AMZ_META)) {
+            return toDecodedString(key.substring(X_AMZ_META.length()));
+        } else if (key.startsWith(LTFS_METADATA_PREFIX)) {
+            return toDecodedString(key.substring(LTFS_METADATA_PREFIX.length()));
+        }
+
+        return "";
+    }
+
     private static List getValues(final Headers headers, final String key) {
         final List valueList = headers.get(key);
         final List returnList = new ArrayList<>(valueList.size());
@@ -83,3 +94,4 @@ public Set keys() {
         return metadata.keySet();
     }
 }
+

From 72815b4da8ed976f0b70656be62ef4f2e25b07e0 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 18 Dec 2017 14:19:13 -0700
Subject: [PATCH 123/229] Updated TapeType to String

---
 ...BucketCapacitySummarySpectraS3Request.java |  7 ++--
 ...DomainCapacitySummarySpectraS3Request.java |  7 ++--
 ...tStorageDomainMembersSpectraS3Request.java |  7 ++--
 ...SystemCapacitySummarySpectraS3Request.java |  8 ++---
 ...TapeDensityDirectivesSpectraS3Request.java |  7 ++--
 .../spectrads3/GetTapesSpectraS3Request.java  |  7 ++--
 ...tTapeDensityDirectiveSpectraS3Request.java |  9 +++--
 ...peStorageDomainMemberSpectraS3Request.java |  9 +++--
 .../models/DetailedTapePartition.java         |  6 ++--
 .../models/NamedDetailedTapePartition.java    |  6 ++--
 .../ds3client/models/StorageDomainMember.java |  7 ++--
 .../spectralogic/ds3client/models/Tape.java   |  6 ++--
 .../models/TapeDensityDirective.java          |  7 ++--
 .../ds3client/models/TapeType.java            | 34 -------------------
 14 files changed, 44 insertions(+), 83 deletions(-)
 delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeType.java

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBucketCapacitySummarySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBucketCapacitySummarySpectraS3Request.java
index 4a0b74fe0..295065e9e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBucketCapacitySummarySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBucketCapacitySummarySpectraS3Request.java
@@ -24,7 +24,6 @@
 import com.spectralogic.ds3client.models.PoolState;
 import com.spectralogic.ds3client.models.PoolType;
 import com.spectralogic.ds3client.models.TapeState;
-import com.spectralogic.ds3client.models.TapeType;
 
 public class GetBucketCapacitySummarySpectraS3Request extends AbstractRequest {
 
@@ -42,7 +41,7 @@ public class GetBucketCapacitySummarySpectraS3Request extends AbstractRequest {
 
     private TapeState tapeState;
 
-    private TapeType tapeType;
+    private String tapeType;
 
     // Constructor
     
@@ -96,7 +95,7 @@ public GetBucketCapacitySummarySpectraS3Request withTapeState(final TapeState ta
     }
 
 
-    public GetBucketCapacitySummarySpectraS3Request withTapeType(final TapeType tapeType) {
+    public GetBucketCapacitySummarySpectraS3Request withTapeType(final String tapeType) {
         this.tapeType = tapeType;
         this.updateQueryParam("tape_type", tapeType);
         return this;
@@ -144,7 +143,7 @@ public TapeState getTapeState() {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainCapacitySummarySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainCapacitySummarySpectraS3Request.java
index 939d0881b..fae8aa5fa 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainCapacitySummarySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainCapacitySummarySpectraS3Request.java
@@ -24,7 +24,6 @@
 import com.spectralogic.ds3client.models.PoolState;
 import com.spectralogic.ds3client.models.PoolType;
 import com.spectralogic.ds3client.models.TapeState;
-import com.spectralogic.ds3client.models.TapeType;
 
 public class GetStorageDomainCapacitySummarySpectraS3Request extends AbstractRequest {
 
@@ -40,7 +39,7 @@ public class GetStorageDomainCapacitySummarySpectraS3Request extends AbstractReq
 
     private TapeState tapeState;
 
-    private TapeType tapeType;
+    private String tapeType;
 
     // Constructor
     
@@ -88,7 +87,7 @@ public GetStorageDomainCapacitySummarySpectraS3Request withTapeState(final TapeS
     }
 
 
-    public GetStorageDomainCapacitySummarySpectraS3Request withTapeType(final TapeType tapeType) {
+    public GetStorageDomainCapacitySummarySpectraS3Request withTapeType(final String tapeType) {
         this.tapeType = tapeType;
         this.updateQueryParam("tape_type", tapeType);
         return this;
@@ -131,7 +130,7 @@ public TapeState getTapeState() {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainMembersSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainMembersSpectraS3Request.java
index 158978d0f..4cf77f4a9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainMembersSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetStorageDomainMembersSpectraS3Request.java
@@ -21,7 +21,6 @@
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 import com.spectralogic.ds3client.models.StorageDomainMemberState;
-import com.spectralogic.ds3client.models.TapeType;
 import com.spectralogic.ds3client.models.WritePreferenceLevel;
 
 public class GetStorageDomainMembersSpectraS3Request extends AbstractPaginationRequest {
@@ -44,7 +43,7 @@ public class GetStorageDomainMembersSpectraS3Request extends AbstractPaginationR
 
     private String tapePartitionId;
 
-    private TapeType tapeType;
+    private String tapeType;
 
     private WritePreferenceLevel writePreference;
 
@@ -143,7 +142,7 @@ public GetStorageDomainMembersSpectraS3Request withTapePartitionId(final String
     }
 
 
-    public GetStorageDomainMembersSpectraS3Request withTapeType(final TapeType tapeType) {
+    public GetStorageDomainMembersSpectraS3Request withTapeType(final String tapeType) {
         this.tapeType = tapeType;
         this.updateQueryParam("tape_type", tapeType);
         return this;
@@ -208,7 +207,7 @@ public String getTapePartitionId() {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetSystemCapacitySummarySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetSystemCapacitySummarySpectraS3Request.java
index 20d144d05..ead04f093 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetSystemCapacitySummarySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetSystemCapacitySummarySpectraS3Request.java
@@ -22,7 +22,7 @@
 import com.spectralogic.ds3client.models.PoolState;
 import com.spectralogic.ds3client.models.PoolType;
 import com.spectralogic.ds3client.models.TapeState;
-import com.spectralogic.ds3client.models.TapeType;
+import com.google.common.net.UrlEscapers;
 
 public class GetSystemCapacitySummarySpectraS3Request extends AbstractRequest {
 
@@ -36,7 +36,7 @@ public class GetSystemCapacitySummarySpectraS3Request extends AbstractRequest {
 
     private TapeState tapeState;
 
-    private TapeType tapeType;
+    private String tapeType;
 
     // Constructor
     
@@ -73,7 +73,7 @@ public GetSystemCapacitySummarySpectraS3Request withTapeState(final TapeState ta
     }
 
 
-    public GetSystemCapacitySummarySpectraS3Request withTapeType(final TapeType tapeType) {
+    public GetSystemCapacitySummarySpectraS3Request withTapeType(final String tapeType) {
         this.tapeType = tapeType;
         this.updateQueryParam("tape_type", tapeType);
         return this;
@@ -111,7 +111,7 @@ public TapeState getTapeState() {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapeDensityDirectivesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapeDensityDirectivesSpectraS3Request.java
index 11d69f24e..a272c07ba 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapeDensityDirectivesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapeDensityDirectivesSpectraS3Request.java
@@ -21,7 +21,6 @@
 import com.spectralogic.ds3client.models.TapeDriveType;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
-import com.spectralogic.ds3client.models.TapeType;
 
 public class GetTapeDensityDirectivesSpectraS3Request extends AbstractPaginationRequest {
 
@@ -39,7 +38,7 @@ public class GetTapeDensityDirectivesSpectraS3Request extends AbstractPagination
 
     private String partitionId;
 
-    private TapeType tapeType;
+    private String tapeType;
 
     // Constructor
     
@@ -108,7 +107,7 @@ public GetTapeDensityDirectivesSpectraS3Request withPartitionId(final String par
     }
 
 
-    public GetTapeDensityDirectivesSpectraS3Request withTapeType(final TapeType tapeType) {
+    public GetTapeDensityDirectivesSpectraS3Request withTapeType(final String tapeType) {
         this.tapeType = tapeType;
         this.updateQueryParam("tape_type", tapeType);
         return this;
@@ -156,7 +155,7 @@ public String getPartitionId() {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
index 6ced563a9..5a9330fc9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
@@ -23,7 +23,6 @@
 import java.util.Date;
 import java.util.UUID;
 import com.spectralogic.ds3client.models.TapeState;
-import com.spectralogic.ds3client.models.TapeType;
 import com.spectralogic.ds3client.models.Priority;
 
 public class GetTapesSpectraS3Request extends AbstractPaginationRequest {
@@ -68,7 +67,7 @@ public class GetTapesSpectraS3Request extends AbstractPaginationRequest {
 
     private String storageDomainId;
 
-    private TapeType type;
+    private String type;
 
     private Priority verifyPending;
 
@@ -239,7 +238,7 @@ public GetTapesSpectraS3Request withStorageDomainId(final String storageDomainId
     }
 
 
-    public GetTapesSpectraS3Request withType(final TapeType type) {
+    public GetTapesSpectraS3Request withType(final String type) {
         this.type = type;
         this.updateQueryParam("type", type);
         return this;
@@ -366,7 +365,7 @@ public String getStorageDomainId() {
     }
 
 
-    public TapeType getType() {
+    public String getType() {
         return this.type;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeDensityDirectiveSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeDensityDirectiveSpectraS3Request.java
index 37cf90a3c..1e1a33b01 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeDensityDirectiveSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeDensityDirectiveSpectraS3Request.java
@@ -21,7 +21,6 @@
 import com.spectralogic.ds3client.models.TapeDriveType;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
-import com.spectralogic.ds3client.models.TapeType;
 
 public class PutTapeDensityDirectiveSpectraS3Request extends AbstractRequest {
 
@@ -31,12 +30,12 @@ public class PutTapeDensityDirectiveSpectraS3Request extends AbstractRequest {
 
     private final String partitionId;
 
-    private final TapeType tapeType;
+    private final String tapeType;
 
     // Constructor
     
     
-    public PutTapeDensityDirectiveSpectraS3Request(final TapeDriveType density, final UUID partitionId, final TapeType tapeType) {
+    public PutTapeDensityDirectiveSpectraS3Request(final TapeDriveType density, final UUID partitionId, final String tapeType) {
         this.density = density;
         this.partitionId = partitionId.toString();
         this.tapeType = tapeType;
@@ -50,7 +49,7 @@ public PutTapeDensityDirectiveSpectraS3Request(final TapeDriveType density, fina
     }
 
     
-    public PutTapeDensityDirectiveSpectraS3Request(final TapeDriveType density, final String partitionId, final TapeType tapeType) {
+    public PutTapeDensityDirectiveSpectraS3Request(final TapeDriveType density, final String partitionId, final String tapeType) {
         this.density = density;
         this.partitionId = partitionId;
         this.tapeType = tapeType;
@@ -84,7 +83,7 @@ public String getPartitionId() {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
index e3806bbb0..c74c01fd6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
@@ -20,7 +20,6 @@
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
-import com.spectralogic.ds3client.models.TapeType;
 import com.spectralogic.ds3client.models.WritePreferenceLevel;
 
 public class PutTapeStorageDomainMemberSpectraS3Request extends AbstractRequest {
@@ -31,14 +30,14 @@ public class PutTapeStorageDomainMemberSpectraS3Request extends AbstractRequest
 
     private final String tapePartitionId;
 
-    private final TapeType tapeType;
+    private final String tapeType;
 
     private WritePreferenceLevel writePreference;
 
     // Constructor
     
     
-    public PutTapeStorageDomainMemberSpectraS3Request(final UUID storageDomainId, final UUID tapePartitionId, final TapeType tapeType) {
+    public PutTapeStorageDomainMemberSpectraS3Request(final UUID storageDomainId, final UUID tapePartitionId, final String tapeType) {
         this.storageDomainId = storageDomainId.toString();
         this.tapePartitionId = tapePartitionId.toString();
         this.tapeType = tapeType;
@@ -52,7 +51,7 @@ public PutTapeStorageDomainMemberSpectraS3Request(final UUID storageDomainId, fi
     }
 
     
-    public PutTapeStorageDomainMemberSpectraS3Request(final String storageDomainId, final String tapePartitionId, final TapeType tapeType) {
+    public PutTapeStorageDomainMemberSpectraS3Request(final String storageDomainId, final String tapePartitionId, final String tapeType) {
         this.storageDomainId = storageDomainId;
         this.tapePartitionId = tapePartitionId;
         this.tapeType = tapeType;
@@ -93,7 +92,7 @@ public String getTapePartitionId() {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
index 8ed233ab8..75923d9a4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
@@ -64,7 +64,7 @@ public class DetailedTapePartition {
 
     @JsonProperty("TapeTypes")
     @JacksonXmlElementWrapper(useWrapping = false)
-    private List tapeTypes = new ArrayList<>();
+    private List tapeTypes = new ArrayList<>();
 
     // Constructor
     public DetailedTapePartition() {
@@ -172,11 +172,11 @@ public void setState(final TapePartitionState state) {
     }
 
 
-    public List getTapeTypes() {
+    public List getTapeTypes() {
         return this.tapeTypes;
     }
 
-    public void setTapeTypes(final List tapeTypes) {
+    public void setTapeTypes(final List tapeTypes) {
         this.tapeTypes = tapeTypes;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
index 1f0420295..b92b34261 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
@@ -64,7 +64,7 @@ public class NamedDetailedTapePartition {
 
     @JsonProperty("TapeTypes")
     @JacksonXmlElementWrapper(useWrapping = false)
-    private List tapeTypes = new ArrayList<>();
+    private List tapeTypes = new ArrayList<>();
 
     // Constructor
     public NamedDetailedTapePartition() {
@@ -172,11 +172,11 @@ public void setState(final TapePartitionState state) {
     }
 
 
-    public List getTapeTypes() {
+    public List getTapeTypes() {
         return this.tapeTypes;
     }
 
-    public void setTapeTypes(final List tapeTypes) {
+    public void setTapeTypes(final List tapeTypes) {
         this.tapeTypes = tapeTypes;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
index 432233560..d1a835ca7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
+import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class StorageDomainMember {
@@ -40,7 +41,7 @@ public class StorageDomainMember {
     private UUID tapePartitionId;
 
     @JsonProperty("TapeType")
-    private TapeType tapeType;
+    private String tapeType;
 
     @JsonProperty("WritePreference")
     private WritePreferenceLevel writePreference;
@@ -97,11 +98,11 @@ public void setTapePartitionId(final UUID tapePartitionId) {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
-    public void setTapeType(final TapeType tapeType) {
+    public void setTapeType(final String tapeType) {
         this.tapeType = tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
index 7c74e8f93..19a62d58b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
@@ -97,7 +97,7 @@ public class Tape {
     private Long totalRawCapacity;
 
     @JsonProperty("Type")
-    private TapeType type;
+    private String type;
 
     @JsonProperty("VerifyPending")
     private Priority verifyPending;
@@ -319,11 +319,11 @@ public void setTotalRawCapacity(final Long totalRawCapacity) {
     }
 
 
-    public TapeType getType() {
+    public String getType() {
         return this.type;
     }
 
-    public void setType(final TapeType type) {
+    public void setType(final String type) {
         this.type = type;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDensityDirective.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDensityDirective.java
index 5c74edf4d..3a4ed6ff6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDensityDirective.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDensityDirective.java
@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
+import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class TapeDensityDirective {
@@ -34,7 +35,7 @@ public class TapeDensityDirective {
     private UUID partitionId;
 
     @JsonProperty("TapeType")
-    private TapeType tapeType;
+    private String tapeType;
 
     // Constructor
     public TapeDensityDirective() {
@@ -70,11 +71,11 @@ public void setPartitionId(final UUID partitionId) {
     }
 
 
-    public TapeType getTapeType() {
+    public String getTapeType() {
         return this.tapeType;
     }
 
-    public void setTapeType(final TapeType tapeType) {
+    public void setTapeType(final String tapeType) {
         this.tapeType = tapeType;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeType.java
deleted file mode 100644
index e20bf41a5..000000000
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeType.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * ******************************************************************************
- *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
- *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
- *   this file except in compliance with the License. A copy of the License is located at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *   or in the "license" file accompanying this file.
- *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
- *   specific language governing permissions and limitations under the License.
- * ****************************************************************************
- */
-
-// This code is auto-generated, do not modify
-package com.spectralogic.ds3client.models;
-
-public enum TapeType {
-    LTO5,
-    LTO6,
-    LTO7,
-    LTO8,
-    LTO_CLEANING_TAPE,
-    TS_JC,
-    TS_JY,
-    TS_JK,
-    TS_JD,
-    TS_JZ,
-    TS_JL,
-    TS_CLEANING_TAPE,
-    UNKNOWN,
-    FORBIDDEN
-}
\ No newline at end of file

From 83cf12864ad94d21f1da01c9763e37384779b0da Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 18 Dec 2017 15:32:34 -0700
Subject: [PATCH 124/229] Removing unnecessary imports in models

---
 .../main/java/com/spectralogic/ds3client/models/ActiveJob.java | 1 -
 .../java/com/spectralogic/ds3client/models/AzureTarget.java    | 2 --
 .../spectralogic/ds3client/models/AzureTargetBucketName.java   | 1 -
 .../com/spectralogic/ds3client/models/AzureTargetFailure.java  | 1 -
 .../models/AzureTargetFailureNotificationRegistration.java     | 2 --
 .../src/main/java/com/spectralogic/ds3client/models/Blob.java  | 1 -
 .../ds3client/models/BlobStoreTaskInformation.java             | 1 -
 .../main/java/com/spectralogic/ds3client/models/Bucket.java    | 2 --
 .../java/com/spectralogic/ds3client/models/BucketDetails.java  | 1 -
 .../com/spectralogic/ds3client/models/BuildInformation.java    | 1 -
 .../java/com/spectralogic/ds3client/models/BulkObject.java     | 1 -
 .../com/spectralogic/ds3client/models/CacheFilesystem.java     | 3 ---
 .../ds3client/models/CacheFilesystemInformation.java           | 1 -
 .../java/com/spectralogic/ds3client/models/CanceledJob.java    | 1 -
 .../ds3client/models/CompleteMultipartUploadResult.java        | 1 -
 .../java/com/spectralogic/ds3client/models/CompletedJob.java   | 1 -
 .../main/java/com/spectralogic/ds3client/models/Contents.java  | 1 -
 .../com/spectralogic/ds3client/models/DataPathBackend.java     | 1 -
 .../com/spectralogic/ds3client/models/DataPersistenceRule.java | 1 -
 .../java/com/spectralogic/ds3client/models/DataPolicy.java     | 2 --
 .../com/spectralogic/ds3client/models/DeleteObjectError.java   | 1 -
 .../com/spectralogic/ds3client/models/DetailedS3Object.java    | 2 --
 .../com/spectralogic/ds3client/models/DetailedTapeFailure.java | 1 -
 .../spectralogic/ds3client/models/DetailedTapePartition.java   | 1 -
 .../spectralogic/ds3client/models/Ds3DataReplicationRule.java  | 1 -
 .../main/java/com/spectralogic/ds3client/models/Ds3Target.java | 2 --
 .../com/spectralogic/ds3client/models/Ds3TargetFailure.java    | 1 -
 .../models/Ds3TargetFailureNotificationRegistration.java       | 2 --
 .../src/main/java/com/spectralogic/ds3client/models/Error.java | 1 -
 .../java/com/spectralogic/ds3client/models/FeatureKey.java     | 2 --
 .../src/main/java/com/spectralogic/ds3client/models/Group.java | 1 -
 .../ds3client/models/InitiateMultipartUploadResult.java        | 1 -
 .../src/main/java/com/spectralogic/ds3client/models/Job.java   | 1 -
 .../ds3client/models/JobCompletedNotificationRegistration.java | 2 --
 .../ds3client/models/JobCreatedNotificationRegistration.java   | 2 --
 .../models/JobCreationFailedNotificationRegistration.java      | 2 --
 .../main/java/com/spectralogic/ds3client/models/JobNode.java   | 2 --
 .../com/spectralogic/ds3client/models/ListBucketResult.java    | 3 +--
 .../ds3client/models/ListMultiPartUploadsResult.java           | 1 -
 .../com/spectralogic/ds3client/models/ListPartsResult.java     | 2 --
 .../com/spectralogic/ds3client/models/MasterObjectList.java    | 1 -
 .../com/spectralogic/ds3client/models/MultiPartUpload.java     | 1 -
 .../com/spectralogic/ds3client/models/MultiPartUploadPart.java | 1 -
 .../ds3client/models/NamedDetailedTapePartition.java           | 1 -
 .../src/main/java/com/spectralogic/ds3client/models/Node.java  | 2 --
 .../src/main/java/com/spectralogic/ds3client/models/Pool.java  | 1 -
 .../java/com/spectralogic/ds3client/models/PoolFailure.java    | 1 -
 .../ds3client/models/PoolFailureNotificationRegistration.java  | 2 --
 .../java/com/spectralogic/ds3client/models/PoolPartition.java  | 1 -
 .../main/java/com/spectralogic/ds3client/models/S3Object.java  | 1 -
 .../models/S3ObjectCachedNotificationRegistration.java         | 2 --
 .../ds3client/models/S3ObjectLostNotificationRegistration.java | 2 --
 .../models/S3ObjectPersistedNotificationRegistration.java      | 2 --
 .../com/spectralogic/ds3client/models/S3ObjectToDelete.java    | 1 -
 .../main/java/com/spectralogic/ds3client/models/S3Target.java  | 2 --
 .../com/spectralogic/ds3client/models/S3TargetBucketName.java  | 1 -
 .../com/spectralogic/ds3client/models/S3TargetFailure.java     | 1 -
 .../models/S3TargetFailureNotificationRegistration.java        | 2 --
 .../java/com/spectralogic/ds3client/models/SpectraUser.java    | 1 -
 .../java/com/spectralogic/ds3client/models/StorageDomain.java  | 3 ---
 .../spectralogic/ds3client/models/StorageDomainFailure.java    | 1 -
 .../models/StorageDomainFailureNotificationRegistration.java   | 2 --
 .../com/spectralogic/ds3client/models/StorageDomainMember.java | 1 -
 .../java/com/spectralogic/ds3client/models/SystemFailure.java  | 1 -
 .../models/SystemFailureNotificationRegistration.java          | 2 --
 .../com/spectralogic/ds3client/models/SystemInformation.java   | 1 -
 .../src/main/java/com/spectralogic/ds3client/models/Tape.java  | 2 --
 .../spectralogic/ds3client/models/TapeDensityDirective.java    | 1 -
 .../main/java/com/spectralogic/ds3client/models/TapeDrive.java | 1 -
 .../java/com/spectralogic/ds3client/models/TapeFailure.java    | 1 -
 .../ds3client/models/TapeFailureNotificationRegistration.java  | 2 --
 .../java/com/spectralogic/ds3client/models/TapeLibrary.java    | 1 -
 .../java/com/spectralogic/ds3client/models/TapePartition.java  | 1 -
 .../spectralogic/ds3client/models/TapePartitionFailure.java    | 1 -
 .../models/TapePartitionFailureNotificationRegistration.java   | 2 --
 .../src/main/java/com/spectralogic/ds3client/models/User.java  | 1 -
 76 files changed, 1 insertion(+), 106 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
index 72c7af5b8..de63306c6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
@@ -20,7 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
 import java.util.Date;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class ActiveJob {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTarget.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTarget.java
index f9069a666..75eaa8838 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTarget.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTarget.java
@@ -18,8 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
-import java.lang.Integer;
 import java.util.UUID;
 import java.util.Date;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetBucketName.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetBucketName.java
index 8ab1de76f..66fca00c5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetBucketName.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetBucketName.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class AzureTargetBucketName {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetFailure.java
index 7bd738b55..c3bf2ec2c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetFailure.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetFailureNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetFailureNotificationRegistration.java
index 6b1b93558..6ada00fb8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetFailureNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/AzureTargetFailureNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class AzureTargetFailureNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Blob.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Blob.java
index 722dd01ed..5fa84e9fc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Blob.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Blob.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BlobStoreTaskInformation.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BlobStoreTaskInformation.java
index 0b261ff0c..27e5149ce 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BlobStoreTaskInformation.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BlobStoreTaskInformation.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Bucket.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Bucket.java
index 24b00336c..f8ecb7544 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Bucket.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Bucket.java
@@ -21,8 +21,6 @@
 import java.util.Date;
 import java.util.UUID;
 import java.lang.Boolean;
-import java.lang.Long;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class Bucket {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BucketDetails.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BucketDetails.java
index 00d731b6c..696f1ebc5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BucketDetails.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BucketDetails.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class BucketDetails {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BuildInformation.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BuildInformation.java
index e2bace3b9..90afff4d7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BuildInformation.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BuildInformation.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class BuildInformation {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BulkObject.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BulkObject.java
index 17d77e3bf..e2d214aa2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BulkObject.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BulkObject.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
 import java.util.UUID;
 import java.lang.Boolean;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CacheFilesystem.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CacheFilesystem.java
index 5658acc11..1643e2fbf 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CacheFilesystem.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CacheFilesystem.java
@@ -19,9 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.Long;
-import java.lang.Double;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class CacheFilesystem {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CacheFilesystemInformation.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CacheFilesystemInformation.java
index d58ba2ef4..fdc8cb99f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CacheFilesystemInformation.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CacheFilesystemInformation.java
@@ -21,7 +21,6 @@
 import java.util.List;
 import java.util.ArrayList;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class CacheFilesystemInformation {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CanceledJob.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CanceledJob.java
index eb3735415..61ea73bd5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CanceledJob.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CanceledJob.java
@@ -20,7 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
 import java.util.Date;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class CanceledJob {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CompleteMultipartUploadResult.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CompleteMultipartUploadResult.java
index a9ca5e847..1553821a1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CompleteMultipartUploadResult.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CompleteMultipartUploadResult.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "CompleteMultipartUploadResult")
 public class CompleteMultipartUploadResult {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CompletedJob.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CompletedJob.java
index 6ba3e045a..dd91c4885 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CompletedJob.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/CompletedJob.java
@@ -20,7 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
 import java.util.Date;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class CompletedJob {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Contents.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Contents.java
index 20cdd84be..9f5f97dd4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Contents.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Contents.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.Date;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
index 4af0ee42b..04465a50a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.Integer;
 import java.util.UUID;
 import java.util.Date;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPersistenceRule.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPersistenceRule.java
index 637c85794..26c2d07c8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPersistenceRule.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPersistenceRule.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class DataPersistenceRule {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
index 2ede6aa7d..3e7ad9e22 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
@@ -19,9 +19,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.Long;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class DataPolicy {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DeleteObjectError.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DeleteObjectError.java
index c9baec5c0..3335493e1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DeleteObjectError.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DeleteObjectError.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class DeleteObjectError {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedS3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedS3Object.java
index 87d0706d0..2228e4f59 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedS3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedS3Object.java
@@ -18,10 +18,8 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.Integer;
 import java.util.UUID;
 import java.util.Date;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Object")
 public class DetailedS3Object {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapeFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapeFailure.java
index 1e33d6a32..6274b7b45 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapeFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapeFailure.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
index 75923d9a4..b57fc32bf 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
@@ -21,7 +21,6 @@
 import java.util.List;
 import java.util.ArrayList;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3DataReplicationRule.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3DataReplicationRule.java
index f28feaf64..71a888869 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3DataReplicationRule.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3DataReplicationRule.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class Ds3DataReplicationRule {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3Target.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3Target.java
index 8a95adaae..c5a1c4d16 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3Target.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3Target.java
@@ -18,8 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
-import java.lang.Integer;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3TargetFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3TargetFailure.java
index 86fb07d63..c9ba760d2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3TargetFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3TargetFailure.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3TargetFailureNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3TargetFailureNotificationRegistration.java
index 2bebff42d..b81406561 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3TargetFailureNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Ds3TargetFailureNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class Ds3TargetFailureNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Error.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Error.java
index 45a6727b2..3f1cfc2bd 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Error.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Error.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Error")
 @JsonIgnoreProperties(ignoreUnknown = true)
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FeatureKey.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FeatureKey.java
index 9c52da106..08b1425cf 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FeatureKey.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/FeatureKey.java
@@ -18,8 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.Long;
-import java.lang.String;
 import java.util.Date;
 import java.util.UUID;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Group.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Group.java
index f36f19008..031a97a93 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Group.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Group.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class Group {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/InitiateMultipartUploadResult.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/InitiateMultipartUploadResult.java
index 73c8f772e..d02625df8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/InitiateMultipartUploadResult.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/InitiateMultipartUploadResult.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "InitiateMultipartUploadResult")
 public class InitiateMultipartUploadResult {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Job.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Job.java
index 49458a687..801229ec3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Job.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Job.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
-import java.lang.String;
 import java.lang.Boolean;
 import java.util.UUID;
 import java.util.List;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCompletedNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCompletedNotificationRegistration.java
index 8e5277f2d..a8f3be4ce 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCompletedNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCompletedNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class JobCompletedNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCreatedNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCreatedNotificationRegistration.java
index 17386fcf3..9e0dec98d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCreatedNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCreatedNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class JobCreatedNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCreationFailedNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCreationFailedNotificationRegistration.java
index 613eac38f..655111c5a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCreationFailedNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobCreationFailedNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class JobCreationFailedNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobNode.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobNode.java
index 6e21b0c76..c5b6f5fbb 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobNode.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobNode.java
@@ -18,9 +18,7 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
-import java.lang.Integer;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListBucketResult.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListBucketResult.java
index d74203176..96b1bf0d6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListBucketResult.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListBucketResult.java
@@ -18,11 +18,10 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
+import java.util.Date;
 import java.util.List;
 import java.util.ArrayList;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
-import java.util.Date;
 import com.spectralogic.ds3client.models.common.CommonPrefixes;
 
 @JacksonXmlRootElement(namespace = "ListBucketResult")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListMultiPartUploadsResult.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListMultiPartUploadsResult.java
index 2dcd4a937..d4b2e437d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListMultiPartUploadsResult.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListMultiPartUploadsResult.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.List;
 import java.util.ArrayList;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListPartsResult.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListPartsResult.java
index 481bc03f3..d9fed0e8a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListPartsResult.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListPartsResult.java
@@ -18,8 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
-import java.lang.Integer;
 import java.util.List;
 import java.util.ArrayList;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MasterObjectList.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MasterObjectList.java
index f0e97e4c3..a464dc2a3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MasterObjectList.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MasterObjectList.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
-import java.lang.String;
 import java.lang.Boolean;
 import java.util.UUID;
 import java.util.List;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MultiPartUpload.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MultiPartUpload.java
index 78d94c508..620bfbdf8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MultiPartUpload.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MultiPartUpload.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MultiPartUploadPart.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MultiPartUploadPart.java
index f0ed0052f..d3b8d1a1b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MultiPartUploadPart.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/MultiPartUploadPart.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.Date;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
index b92b34261..5d79e7ce6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
@@ -21,7 +21,6 @@
 import java.util.List;
 import java.util.ArrayList;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "TapePartition")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Node.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Node.java
index 9320b4b67..3e8a24f24 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Node.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Node.java
@@ -18,8 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.Integer;
-import java.lang.String;
 import java.util.UUID;
 import java.util.Date;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Pool.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Pool.java
index fe7a29b60..00581f1b8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Pool.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Pool.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 import java.util.Date;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailure.java
index 31e05bc8d..e90483c57 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailure.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailureNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailureNotificationRegistration.java
index b7b9a3d4c..f742b02b2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailureNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolFailureNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class PoolFailureNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolPartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolPartition.java
index 26537b4f0..25c25ce70 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolPartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/PoolPartition.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class PoolPartition {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Object.java
index 6de63d90b..39ff6894a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Object.java
@@ -21,7 +21,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
 import java.util.Date;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class S3Object {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectCachedNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectCachedNotificationRegistration.java
index f94ca3adc..7ff562736 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectCachedNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectCachedNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class S3ObjectCachedNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectLostNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectLostNotificationRegistration.java
index b3beee4c5..588a7409a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectLostNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectLostNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class S3ObjectLostNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectPersistedNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectPersistedNotificationRegistration.java
index 6894c1066..873560130 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectPersistedNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectPersistedNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class S3ObjectPersistedNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectToDelete.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectToDelete.java
index b4621e603..b7dd8574f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectToDelete.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectToDelete.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class S3ObjectToDelete {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Target.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Target.java
index c2519b706..0566318af 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Target.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Target.java
@@ -18,8 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
-import java.lang.Integer;
 import java.util.UUID;
 import java.util.Date;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetBucketName.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetBucketName.java
index 97e633c49..f28e1fb85 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetBucketName.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetBucketName.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class S3TargetBucketName {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetFailure.java
index 7d3601597..cef28bc32 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetFailure.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetFailureNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetFailureNotificationRegistration.java
index 27ffaac6d..05577f437 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetFailureNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3TargetFailureNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class S3TargetFailureNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SpectraUser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SpectraUser.java
index eb238b089..9dd159b0c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SpectraUser.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SpectraUser.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomain.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomain.java
index c11ac5384..20dd93491 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomain.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomain.java
@@ -18,10 +18,7 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.Long;
-import java.lang.String;
 import java.util.UUID;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class StorageDomain {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainFailure.java
index c8ccabf56..4db76d579 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainFailure.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainFailureNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainFailureNotificationRegistration.java
index c4819a161..d7ab9143f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainFailureNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainFailureNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class StorageDomainFailureNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
index d1a835ca7..971f592c7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class StorageDomainMember {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemFailure.java
index 569b5cb3c..972723c82 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemFailure.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemFailureNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemFailureNotificationRegistration.java
index 7a5d02f58..09b60dde0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemFailureNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemFailureNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class SystemFailureNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemInformation.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemInformation.java
index f516a01a6..78ef776a4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemInformation.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SystemInformation.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
index 19a62d58b..4b6a50990 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
@@ -18,8 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.Long;
-import java.lang.String;
 import java.util.UUID;
 import java.util.Date;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDensityDirective.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDensityDirective.java
index 3a4ed6ff6..0afacf9f8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDensityDirective.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDensityDirective.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class TapeDensityDirective {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
index 063918953..fa9f0e1f9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.UUID;
 import java.util.Date;
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailure.java
index d1190ffcb..02ff544f8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailure.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class TapeFailure {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailureNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailureNotificationRegistration.java
index 4a8b5f95e..e1e7da7dd 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailureNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeFailureNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class TapeFailureNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeLibrary.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeLibrary.java
index 65a72e6f3..03b145661 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeLibrary.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeLibrary.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.UUID;
-import java.lang.String;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class TapeLibrary {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
index d7dd9e19c..eac412bee 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailure.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailure.java
index 87f83ef0c..d74e26662 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailure.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailure.java
@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureNotificationRegistration.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureNotificationRegistration.java
index d52b5144e..63ac9b2c8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureNotificationRegistration.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureNotificationRegistration.java
@@ -20,8 +20,6 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import java.util.Date;
 import java.util.UUID;
-import java.lang.String;
-import java.lang.Integer;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class TapePartitionFailureNotificationRegistration {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/User.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/User.java
index a7e68fb9d..bf0332850 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/User.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/User.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
-import java.lang.String;
 import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")

From a99edb3292daf5abee47d29efba5f9a7c4c9fe2e Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 24 Jan 2018 14:33:06 -0700
Subject: [PATCH 125/229] Fix for
 https://jira.spectralogic.com/browse/BPBROWSER-408 Seems that building on a
 machine that has both java 8 & 9 JDKs using the java 8 javac causes java 9
 jars to get pulled in.

---
 .../com/spectralogic/ds3client/helpers/ExceptionClassifier.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java
index 00be54730..e21ba26b7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java
@@ -22,7 +22,7 @@ public final class ExceptionClassifier {
     private ExceptionClassifier() { }
 
     public static boolean isRecoverableException(final Throwable t) {
-        if (t instanceof UnrecoverableIOException || t instanceof FileSystemException || t instanceof SecurityException) {
+        if (t instanceof UnrecoverableIOException || t instanceof FileSystemException || t instanceof SecurityException || t instanceof NoSuchMethodException) {
             return false;
         }
 

From 5ba02434d2944f596caa66d14a57ec9b708c6549 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Mon, 12 Feb 2018 14:11:40 -0700
Subject: [PATCH 126/229] Make sure we put the ds3_sdk.properties file in the
 shadow jar.

---
 ds3-sdk/build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ds3-sdk/build.gradle b/ds3-sdk/build.gradle
index ec84ec7dc..24a20c7aa 100644
--- a/ds3-sdk/build.gradle
+++ b/ds3-sdk/build.gradle
@@ -82,6 +82,7 @@ jar {
 }
 
 jar.dependsOn genConfigProperties
+shadowJar.dependsOn genConfigProperties
 
 dependencies {
     compile     "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"

From 39fb1adea93d2b6cc383661ffca00620365a3dc9 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Tue, 13 Feb 2018 14:53:00 -0700
Subject: [PATCH 127/229] Fix a thread leak when Job.startJob is
 called in a loop.

---
 .../AbstractTransferStrategy.java             | 63 +++++++++----------
 1 file changed, 31 insertions(+), 32 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index f42a195a4..0b5df66d0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -15,7 +15,6 @@
 
 package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
 
-import com.google.common.base.Predicate;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.ListeningExecutorService;
@@ -41,8 +40,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
-
 /**
  * An implementation of {@link TransferStrategy} that provides a default implementation {@code transfer}
  * implementation.
@@ -109,9 +106,21 @@ public AbstractTransferStrategy withTransferMethod(final TransferMethod transfer
     public void transfer() throws IOException {
         cachedException.set(null);
 
+        try {
+            transferAllJobBlobs();
+        } finally {
+            close();
+        }
+
+        if (cachedException.get() != null) {
+            throw cachedException.get();
+        }
+    }
+
+    private void transferAllJobBlobs() throws IOException {
         final AtomicInteger numBlobsRemaining = new AtomicInteger(jobState.numBlobsInJob());
 
-        while ( ! Thread.currentThread().isInterrupted() && numBlobsRemaining.get() > 0) {
+        while (!Thread.currentThread().isInterrupted() && numBlobsRemaining.get() > 0) {
             try {
                 final Iterable jobParts = jobPartsNotAlreadyTransferred();
 
@@ -129,25 +138,17 @@ public void transfer() throws IOException {
                 throw e;
             } catch (final InterruptedException | NoSuchElementException e) {
                 Thread.currentThread().interrupt();
-            }  catch (final Throwable t) {
+            } catch (final Throwable t) {
                 emitFailureAndSetCachedException(t);
             }
         }
-
-        if (cachedException.get() != null) {
-            throw cachedException.get();
-        }
     }
 
+
     private Iterable jobPartsNotAlreadyTransferred() throws IOException, InterruptedException {
         return FluentIterable
                 .from(blobStrategy.getWork())
-                .filter(new Predicate() {
-                    @Override
-                    public boolean apply(@Nullable final JobPart jobPart) {
-                        return jobState.contains(jobPart.getBlob());
-                    }
-                });
+                .filter(jobPart -> jobState.contains(jobPart.getBlob()));
     }
 
     private void emitFailureAndSetCachedException(final Throwable t) {
@@ -169,24 +170,22 @@ private void transferJobParts(final Iterable jobParts,
                                   final CountDownLatch countDownLatch,
                                   final AtomicInteger numBlobsTransferred) throws IOException {
         for (final JobPart jobPart : jobParts) {
-            executorService.submit(new Callable() {
-                @Override
-                public Void call() throws Exception {
-                    try {
-                        transferMethod.transferJobPart(jobPart);
-                    } catch (final RuntimeException e) {
-                        emitFailureAndSetCachedException(e);
-                        throw e;
-                    } catch (final Exception e) {
-                        emitFailureAndSetCachedException(e);
-                        throw new RuntimeException(e);
-                    } finally {
-                        jobState.blobTransferredOrFailed(jobPart.getBlob());
-                        numBlobsTransferred.decrementAndGet();
-                        countDownLatch.countDown();
-                        return null;
-                    }
+            executorService.submit((Callable) () -> {
+                try {
+                    transferMethod.transferJobPart(jobPart);
+                } catch (final RuntimeException e) {
+                    emitFailureAndSetCachedException(e);
+                    throw e;
+                } catch (final Exception e) {
+                    emitFailureAndSetCachedException(e);
+                    throw new RuntimeException(e);
+                } finally {
+                    jobState.blobTransferredOrFailed(jobPart.getBlob());
+                    numBlobsTransferred.decrementAndGet();
+                    countDownLatch.countDown();
                 }
+
+                return null;
             });
         }
     }

From 37f9cf41e20c849469faac85eb58b8590d31d248 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 10:18:45 -0700
Subject: [PATCH 128/229] Fix a thread leak when Job.startJob is
 called in a loop.

---
 .../ds3client/integration/Resources_Test.java | 200 ++++++++++++++++++
 1 file changed, 200 insertions(+)
 create mode 100644 ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
new file mode 100644
index 000000000..197f22d7a
--- /dev/null
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
@@ -0,0 +1,200 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.integration;
+
+import com.google.common.collect.ImmutableList;
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.spectrads3.PutBucketSpectraS3Request;
+import com.spectralogic.ds3client.helpers.DeleteBucket;
+import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
+import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.util.UUID;
+
+public class Resources_Test {
+    private static final String BASE_BUCKET_NAME = "putLotsaBuckets";
+    private static final String BUCKET_DATA_POLICY_NAME = "Memory Test";
+    private static final int NUM_BUCKETS = 10;
+    private static final int NUM_OBJECTS = 1024;
+    private static final int OBJECT_SIZE = 1024 * 10;
+    private static final String BASE_FILE_NAME = "file_";
+    private static final String DATA = "Gracie";
+    private static final int DATA_BUFFER_SIZE = 1024;
+    private static final int NUM_THREADS = 1024;
+    private static final Ds3Client ds3Client = Util.fromEnv();
+    private static TempStorageIds envStorageIds;
+    private static UUID envDataPolicyId;
+    private static final String POLICY_PREFIX = Resources_Test.class.getSimpleName();
+
+    @BeforeClass
+    public static void startup() throws IOException {
+        envDataPolicyId = TempStorageUtil.setupDataPolicy(POLICY_PREFIX, false, ChecksumType.Type.MD5, ds3Client);
+        envStorageIds = TempStorageUtil.setup(POLICY_PREFIX, envDataPolicyId, ds3Client);
+    }
+
+    @AfterClass
+    public static void teardown() throws IOException {
+        TempStorageUtil.teardown(POLICY_PREFIX, envStorageIds, ds3Client);
+        ds3Client.close();
+    }
+
+    @Test
+    public void putLotsaBuckets() throws IOException {
+        final ImmutableList bucketNames = generateBucketNames(NUM_BUCKETS);
+
+        try {
+            createBuckets(ds3Client, bucketNames);
+            populateBuckets(Ds3ClientHelpers.wrap(ds3Client), bucketNames);
+        } finally {
+            deleteBuckets(Ds3ClientHelpers.wrap(ds3Client), bucketNames);
+        }
+    }
+
+    private ImmutableList generateBucketNames(final int numBucketNames) {
+        final ImmutableList.Builder bucketNameBuilder = new ImmutableList.Builder<>();
+
+        for (int i = 0; i < numBucketNames; ++i) {
+            bucketNameBuilder.add(BASE_BUCKET_NAME + "_" + i);
+        }
+
+        return bucketNameBuilder.build();
+    }
+
+    private void createBuckets(final Ds3Client ds3Client, ImmutableList bucketNames) throws IOException {
+        for (final String bucketName : bucketNames) {
+            ds3Client.putBucketSpectraS3(new PutBucketSpectraS3Request(bucketName));
+        }
+    }
+
+    private void populateBuckets(final Ds3ClientHelpers ds3ClientHelpers, final ImmutableList bucketNames) throws IOException {
+        final ImmutableList ds3Objects = generateDs3Objects();
+
+        for (final String bucketName : bucketNames) {
+            final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(bucketName, ds3Objects)
+                    .withMaxParallelRequests(NUM_THREADS);
+            writeJob.transfer(new RepeatStringObjectChannelBuilder(DATA, DATA.length() * 512, DATA_BUFFER_SIZE));
+        }
+    }
+
+    private ImmutableList generateDs3Objects() {
+        final ImmutableList.Builder ds3ObjectBuilder = new ImmutableList.Builder<>();
+
+        for (int i = 0; i < NUM_OBJECTS; ++i) {
+            final Ds3Object ds3Object = new Ds3Object();
+            ds3Object.setName(BASE_FILE_NAME + i);
+            ds3Object.setSize(OBJECT_SIZE);
+            ds3ObjectBuilder.add(ds3Object);
+        }
+
+        return ds3ObjectBuilder.build();
+    }
+
+    private void deleteBuckets(final Ds3ClientHelpers ds3ClientHelpers, final ImmutableList bucketNames) {
+        for (final String bucketName : bucketNames) {
+            DeleteBucket.INSTANCE.deleteBucket(ds3ClientHelpers, bucketName);
+        }
+    }
+
+    private static class RepeatStringObjectChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
+        private final int bufferSize;
+        private final long sizeOfFiles;
+        private String inputDataHeader;
+
+        RepeatStringObjectChannelBuilder(final String inputDataHeader, final int bufferSize, final long sizeOfFile) {
+            this.bufferSize = bufferSize;
+            this.sizeOfFiles = sizeOfFile;
+            this.inputDataHeader = inputDataHeader;
+        }
+
+        @Override public SeekableByteChannel buildChannel(final String key ) {
+            return new RepeatStringByteChannel( inputDataHeader + key, this.bufferSize, this.sizeOfFiles );
+        }
+
+        private static class RepeatStringByteChannel implements SeekableByteChannel {
+            final private byte[] backingArray;
+            final private int bufferSize;
+            final private long limit;
+            private boolean isOpen;
+            private int position;
+
+            RepeatStringByteChannel( final String inputData, final int bufferSize, final long size ) {
+                this.bufferSize = bufferSize;
+                final byte[] bytes = new byte[ bufferSize ];
+                final byte[] stringBytes = inputData.getBytes();
+                int stringPosition = 0;
+                for ( int i = 0; i < bufferSize; i++ ) {
+                    bytes[ i ] = stringBytes[ stringPosition ];
+                    if ( ++stringPosition >= inputData.length() )
+                    {
+                        stringPosition = 0;
+                    }
+                }
+
+                backingArray = bytes;
+
+                this.position = 0;
+                this.limit = size;
+                this.isOpen = true;
+            }
+
+            public void close() {
+                this.isOpen = false;
+            }
+
+            public boolean isOpen() {
+                return this.isOpen;
+            }
+
+            public long position() {
+                return ( long ) this.position;
+            }
+
+            public SeekableByteChannel position( final long newPosition ) {
+                this.position = ( int ) newPosition;
+                return this;
+            }
+
+            public int read( final ByteBuffer dst ) {
+                final int amountToRead = Math.min( dst.remaining(), this.bufferSize );
+                dst.put( this.backingArray, 0, amountToRead );
+                return amountToRead;
+            }
+
+            public long size() {
+                return this.limit;
+            }
+
+            public SeekableByteChannel truncate( final long size ) {
+                return this;
+            }
+
+            public int write( final ByteBuffer src ) {
+                final int amountToWrite = Math.min( src.remaining(), this.bufferSize );
+                src.get( this.backingArray, 0, amountToWrite );
+                return amountToWrite;
+            }
+        }
+    }
+}

From be152e937254bb5c2e0c187e0c5027185c5437ba Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 10:22:57 -0700
Subject: [PATCH 129/229] Fix a thread leak when Job.startJob is
 called in a loop.

---
 .../com/spectralogic/ds3client/integration/Resources_Test.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
index 197f22d7a..a26236069 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
@@ -35,7 +35,6 @@
 
 public class Resources_Test {
     private static final String BASE_BUCKET_NAME = "putLotsaBuckets";
-    private static final String BUCKET_DATA_POLICY_NAME = "Memory Test";
     private static final int NUM_BUCKETS = 10;
     private static final int NUM_OBJECTS = 1024;
     private static final int OBJECT_SIZE = 1024 * 10;

From a05028a2f8a5592a2137983553c270974527e6bc Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 10:41:26 -0700
Subject: [PATCH 130/229] Fix a thread leak when Job.startJob is
 called in a loop.

---
 .../com/spectralogic/ds3client/integration/Resources_Test.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
index a26236069..edd09e055 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
@@ -33,6 +33,8 @@
 import java.nio.channels.SeekableByteChannel;
 import java.util.UUID;
 
+import static org.junit.Assert.assertTrue;
+
 public class Resources_Test {
     private static final String BASE_BUCKET_NAME = "putLotsaBuckets";
     private static final int NUM_BUCKETS = 10;
@@ -66,6 +68,7 @@ public void putLotsaBuckets() throws IOException {
         try {
             createBuckets(ds3Client, bucketNames);
             populateBuckets(Ds3ClientHelpers.wrap(ds3Client), bucketNames);
+            assertTrue(Thread.activeCount() < 2 * NUM_THREADS);
         } finally {
             deleteBuckets(Ds3ClientHelpers.wrap(ds3Client), bucketNames);
         }

From 21bf17a69038670443ff7f3ce1794a685d193edb Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 10:46:00 -0700
Subject: [PATCH 131/229] Fix a thread leak when Job.startJob is
 called in a loop.

---
 .../com/spectralogic/ds3client/integration/Resources_Test.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
index edd09e055..385823f1a 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
@@ -46,12 +46,11 @@ public class Resources_Test {
     private static final int NUM_THREADS = 1024;
     private static final Ds3Client ds3Client = Util.fromEnv();
     private static TempStorageIds envStorageIds;
-    private static UUID envDataPolicyId;
     private static final String POLICY_PREFIX = Resources_Test.class.getSimpleName();
 
     @BeforeClass
     public static void startup() throws IOException {
-        envDataPolicyId = TempStorageUtil.setupDataPolicy(POLICY_PREFIX, false, ChecksumType.Type.MD5, ds3Client);
+        final UUID envDataPolicyId = TempStorageUtil.setupDataPolicy(POLICY_PREFIX, false, ChecksumType.Type.MD5, ds3Client);
         envStorageIds = TempStorageUtil.setup(POLICY_PREFIX, envDataPolicyId, ds3Client);
     }
 

From dfde97f1c5f130d01d8c3b062395bf452fc61b38 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 11:08:40 -0700
Subject: [PATCH 132/229] Fix a thread leak when Job.startJob is
 called in a loop.

---
 .../ds3client/integration/Resources_Test.java  | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
index 385823f1a..1df084f1e 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
@@ -65,11 +65,11 @@ public void putLotsaBuckets() throws IOException {
         final ImmutableList bucketNames = generateBucketNames(NUM_BUCKETS);
 
         try {
-            createBuckets(ds3Client, bucketNames);
-            populateBuckets(Ds3ClientHelpers.wrap(ds3Client), bucketNames);
+            createBuckets(bucketNames);
+            populateBuckets(bucketNames);
             assertTrue(Thread.activeCount() < 2 * NUM_THREADS);
         } finally {
-            deleteBuckets(Ds3ClientHelpers.wrap(ds3Client), bucketNames);
+            deleteBuckets(bucketNames);
         }
     }
 
@@ -83,17 +83,19 @@ private ImmutableList generateBucketNames(final int numBucketNames) {
         return bucketNameBuilder.build();
     }
 
-    private void createBuckets(final Ds3Client ds3Client, ImmutableList bucketNames) throws IOException {
+    private void createBuckets(ImmutableList bucketNames) throws IOException {
         for (final String bucketName : bucketNames) {
             ds3Client.putBucketSpectraS3(new PutBucketSpectraS3Request(bucketName));
         }
     }
 
-    private void populateBuckets(final Ds3ClientHelpers ds3ClientHelpers, final ImmutableList bucketNames) throws IOException {
+    private void populateBuckets(final ImmutableList bucketNames) throws IOException {
         final ImmutableList ds3Objects = generateDs3Objects();
 
         for (final String bucketName : bucketNames) {
-            final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(bucketName, ds3Objects)
+            final Ds3ClientHelpers.Job writeJob = Ds3ClientHelpers
+                    .wrap(ds3Client)
+                    .startWriteJob(bucketName, ds3Objects)
                     .withMaxParallelRequests(NUM_THREADS);
             writeJob.transfer(new RepeatStringObjectChannelBuilder(DATA, DATA.length() * 512, DATA_BUFFER_SIZE));
         }
@@ -112,9 +114,9 @@ private ImmutableList generateDs3Objects() {
         return ds3ObjectBuilder.build();
     }
 
-    private void deleteBuckets(final Ds3ClientHelpers ds3ClientHelpers, final ImmutableList bucketNames) {
+    private void deleteBuckets(final ImmutableList bucketNames) {
         for (final String bucketName : bucketNames) {
-            DeleteBucket.INSTANCE.deleteBucket(ds3ClientHelpers, bucketName);
+            DeleteBucket.INSTANCE.deleteBucket(Ds3ClientHelpers.wrap(ds3Client), bucketName);
         }
     }
 

From 77f404c003ed5bb669a1dcf2cd1c7317cd31561f Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 11:15:29 -0700
Subject: [PATCH 133/229] Fix a thread leak when Job.startJob is
 called in a loop.

---
 .../spectralogic/ds3client/integration/Resources_Test.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
index 1df084f1e..126137db3 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
@@ -62,7 +62,7 @@ public static void teardown() throws IOException {
 
     @Test
     public void putLotsaBuckets() throws IOException {
-        final ImmutableList bucketNames = generateBucketNames(NUM_BUCKETS);
+        final ImmutableList bucketNames = generateBucketNames();
 
         try {
             createBuckets(bucketNames);
@@ -73,10 +73,10 @@ public void putLotsaBuckets() throws IOException {
         }
     }
 
-    private ImmutableList generateBucketNames(final int numBucketNames) {
+    private ImmutableList generateBucketNames() {
         final ImmutableList.Builder bucketNameBuilder = new ImmutableList.Builder<>();
 
-        for (int i = 0; i < numBucketNames; ++i) {
+        for (int i = 0; i < NUM_BUCKETS; ++i) {
             bucketNameBuilder.add(BASE_BUCKET_NAME + "_" + i);
         }
 

From 7bc2f9d773893a74283ba2902acea15885447b81 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 11:34:00 -0700
Subject: [PATCH 134/229] Fix a thread leak when Job.startJob is
 called in a loop.

---
 .../spectralogic/ds3client/integration/Resources_Test.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
index 126137db3..ade9917a9 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
@@ -37,13 +37,13 @@
 
 public class Resources_Test {
     private static final String BASE_BUCKET_NAME = "putLotsaBuckets";
-    private static final int NUM_BUCKETS = 10;
+    private static final int NUM_BUCKETS = 100;
     private static final int NUM_OBJECTS = 1024;
     private static final int OBJECT_SIZE = 1024 * 10;
     private static final String BASE_FILE_NAME = "file_";
     private static final String DATA = "Gracie";
     private static final int DATA_BUFFER_SIZE = 1024;
-    private static final int NUM_THREADS = 1024;
+    private static final int NUM_THREADS = NUM_OBJECTS;
     private static final Ds3Client ds3Client = Util.fromEnv();
     private static TempStorageIds envStorageIds;
     private static final String POLICY_PREFIX = Resources_Test.class.getSimpleName();

From ec85323b7838eb7f6833c383d2fa371698046f82 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 16:22:30 -0700
Subject: [PATCH 135/229] I hate checked exceptions.

---
 .../RepeatStringObjectChannelBuilder.java     | 102 +++++++++
 .../ds3client/integration/ResourceTest.kt     | 111 ++++++++++
 .../ds3client/integration/Resources_Test.java | 203 ------------------
 3 files changed, 213 insertions(+), 203 deletions(-)
 create mode 100644 ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/RepeatStringObjectChannelBuilder.java
 create mode 100644 ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/ResourceTest.kt
 delete mode 100644 ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/RepeatStringObjectChannelBuilder.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/RepeatStringObjectChannelBuilder.java
new file mode 100644
index 000000000..1bb32bb28
--- /dev/null
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/RepeatStringObjectChannelBuilder.java
@@ -0,0 +1,102 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers.channelbuilders;
+
+import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+
+public class RepeatStringObjectChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
+    private final int bufferSize;
+    private final long sizeOfFiles;
+    private String inputDataHeader;
+
+    public RepeatStringObjectChannelBuilder(final String inputDataHeader, final int bufferSize, final long sizeOfFile) {
+        this.bufferSize = bufferSize;
+        this.sizeOfFiles = sizeOfFile;
+        this.inputDataHeader = inputDataHeader;
+    }
+
+    @Override
+    public SeekableByteChannel buildChannel(final String key) {
+        return new RepeatStringObjectChannelBuilder.RepeatStringByteChannel(inputDataHeader + key, this.bufferSize, this.sizeOfFiles);
+    }
+
+    private static class RepeatStringByteChannel implements SeekableByteChannel {
+        final private byte[] backingArray;
+        final private int bufferSize;
+        final private long limit;
+        private boolean isOpen;
+        private int position;
+
+        RepeatStringByteChannel(final String inputData, final int bufferSize, final long size) {
+            this.bufferSize = bufferSize;
+            final byte[] bytes = new byte[bufferSize];
+            final byte[] stringBytes = inputData.getBytes();
+            int stringPosition = 0;
+            for (int i = 0; i < bufferSize; i++) {
+                bytes[i] = stringBytes[stringPosition];
+                if (++stringPosition >= inputData.length()) {
+                    stringPosition = 0;
+                }
+            }
+
+            backingArray = bytes;
+
+            this.position = 0;
+            this.limit = size;
+            this.isOpen = true;
+        }
+
+        public void close() {
+            this.isOpen = false;
+        }
+
+        public boolean isOpen() {
+            return this.isOpen;
+        }
+
+        public long position() {
+            return (long) this.position;
+        }
+
+        public SeekableByteChannel position(final long newPosition) {
+            this.position = (int) newPosition;
+            return this;
+        }
+
+        public int read(final ByteBuffer dst) {
+            final int amountToRead = Math.min(dst.remaining(), this.bufferSize);
+            dst.put(this.backingArray, 0, amountToRead);
+            return amountToRead;
+        }
+
+        public long size() {
+            return this.limit;
+        }
+
+        public SeekableByteChannel truncate(final long size) {
+            return this;
+        }
+
+        public int write(final ByteBuffer src) {
+            final int amountToWrite = Math.min(src.remaining(), this.bufferSize);
+            src.get(this.backingArray, 0, amountToWrite);
+            return amountToWrite;
+        }
+    }
+}
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/ResourceTest.kt b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/ResourceTest.kt
new file mode 100644
index 000000000..0860bdded
--- /dev/null
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/ResourceTest.kt
@@ -0,0 +1,111 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.integration
+
+import com.spectralogic.ds3client.commands.spectrads3.PutBucketSpectraS3Request
+import com.spectralogic.ds3client.helpers.DeleteBucket
+import com.spectralogic.ds3client.helpers.Ds3ClientHelpers
+import com.spectralogic.ds3client.helpers.channelbuilders.RepeatStringObjectChannelBuilder
+import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds
+import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil
+import com.spectralogic.ds3client.models.ChecksumType
+import com.spectralogic.ds3client.models.bulk.Ds3Object
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+import java.io.IOException
+
+class ResourceTest {
+    private companion object {
+        private val numBuckets = 100
+        private val numObjects = 1024
+        private val numThreads = numObjects
+        private val objectSize = 1024 * 10
+        private val baseBucketName = "putLotsaBuckets"
+        private val baseFileName = "file"
+        private val objectData = "Gracie"
+        private val dataBufferSize = 1024
+        private val ds3Client = Util.fromEnv()
+        private val policyPrefix = ResourceTest::class.java.simpleName
+        private var envStorageIds : TempStorageIds? = null
+
+        @BeforeClass
+        @JvmStatic
+        @Throws(IOException::class)
+        fun startup() {
+            val envDataPolicyId = TempStorageUtil.setupDataPolicy(policyPrefix, false, ChecksumType.Type.MD5, ds3Client)
+            envStorageIds = TempStorageUtil.setup(policyPrefix, envDataPolicyId, ds3Client)
+        }
+
+        @AfterClass
+        @JvmStatic
+        @Throws(IOException::class)
+        fun teardown() {
+            TempStorageUtil.teardown(policyPrefix, envStorageIds, ds3Client)
+            ds3Client.close()
+        }
+    }
+
+    @Test
+    fun putLotsaBuckets() {
+        val bucketNames = generateBucketNames()
+
+        try {
+            createBuckets(bucketNames)
+            populateBuckets(bucketNames)
+        } finally {
+            deleteBuckets(bucketNames)
+        }
+    }
+
+    private fun generateBucketNames() : List {
+        return (1 .. numBuckets)
+                .map { index -> baseBucketName + index }
+                .toList()
+    }
+
+    private fun createBuckets(bucketNames: List) {
+        bucketNames.forEach({ bucketName -> ds3Client.putBucketSpectraS3(PutBucketSpectraS3Request(bucketName)) })
+    }
+
+    @Throws(IOException::class)
+    private fun populateBuckets(bucketNames: List) {
+        bucketNames
+                .forEach({ bucketName ->
+                    Ds3ClientHelpers
+                            .wrap(ds3Client)
+                            .startWriteJob(bucketName, generateDs3Objects())
+                            .withMaxParallelRequests(numThreads)
+                            .transfer(RepeatStringObjectChannelBuilder(objectData, objectData.length * 512, dataBufferSize.toLong()))
+                })
+
+    }
+
+    private fun generateDs3Objects(): List {
+        return (1 .. numObjects)
+                .map { index ->
+                    val ds3Object = Ds3Object()
+                    ds3Object.name = baseFileName + index
+                    ds3Object.size = objectSize.toLong()
+                    ds3Object
+                }
+                .toList()
+    }
+
+    private fun deleteBuckets(bucketNames: List) {
+        bucketNames.forEach({ bucketName -> DeleteBucket.deleteBucket(Ds3ClientHelpers.wrap(ds3Client), bucketName) })
+    }
+}
\ No newline at end of file
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
deleted file mode 100644
index ade9917a9..000000000
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Resources_Test.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * ******************************************************************************
- *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
- *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
- *   this file except in compliance with the License. A copy of the License is located at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *   or in the "license" file accompanying this file.
- *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
- *   specific language governing permissions and limitations under the License.
- * ****************************************************************************
- */
-
-package com.spectralogic.ds3client.integration;
-
-import com.google.common.collect.ImmutableList;
-import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.spectrads3.PutBucketSpectraS3Request;
-import com.spectralogic.ds3client.helpers.DeleteBucket;
-import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
-import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
-import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
-import com.spectralogic.ds3client.models.ChecksumType;
-import com.spectralogic.ds3client.models.bulk.Ds3Object;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.SeekableByteChannel;
-import java.util.UUID;
-
-import static org.junit.Assert.assertTrue;
-
-public class Resources_Test {
-    private static final String BASE_BUCKET_NAME = "putLotsaBuckets";
-    private static final int NUM_BUCKETS = 100;
-    private static final int NUM_OBJECTS = 1024;
-    private static final int OBJECT_SIZE = 1024 * 10;
-    private static final String BASE_FILE_NAME = "file_";
-    private static final String DATA = "Gracie";
-    private static final int DATA_BUFFER_SIZE = 1024;
-    private static final int NUM_THREADS = NUM_OBJECTS;
-    private static final Ds3Client ds3Client = Util.fromEnv();
-    private static TempStorageIds envStorageIds;
-    private static final String POLICY_PREFIX = Resources_Test.class.getSimpleName();
-
-    @BeforeClass
-    public static void startup() throws IOException {
-        final UUID envDataPolicyId = TempStorageUtil.setupDataPolicy(POLICY_PREFIX, false, ChecksumType.Type.MD5, ds3Client);
-        envStorageIds = TempStorageUtil.setup(POLICY_PREFIX, envDataPolicyId, ds3Client);
-    }
-
-    @AfterClass
-    public static void teardown() throws IOException {
-        TempStorageUtil.teardown(POLICY_PREFIX, envStorageIds, ds3Client);
-        ds3Client.close();
-    }
-
-    @Test
-    public void putLotsaBuckets() throws IOException {
-        final ImmutableList bucketNames = generateBucketNames();
-
-        try {
-            createBuckets(bucketNames);
-            populateBuckets(bucketNames);
-            assertTrue(Thread.activeCount() < 2 * NUM_THREADS);
-        } finally {
-            deleteBuckets(bucketNames);
-        }
-    }
-
-    private ImmutableList generateBucketNames() {
-        final ImmutableList.Builder bucketNameBuilder = new ImmutableList.Builder<>();
-
-        for (int i = 0; i < NUM_BUCKETS; ++i) {
-            bucketNameBuilder.add(BASE_BUCKET_NAME + "_" + i);
-        }
-
-        return bucketNameBuilder.build();
-    }
-
-    private void createBuckets(ImmutableList bucketNames) throws IOException {
-        for (final String bucketName : bucketNames) {
-            ds3Client.putBucketSpectraS3(new PutBucketSpectraS3Request(bucketName));
-        }
-    }
-
-    private void populateBuckets(final ImmutableList bucketNames) throws IOException {
-        final ImmutableList ds3Objects = generateDs3Objects();
-
-        for (final String bucketName : bucketNames) {
-            final Ds3ClientHelpers.Job writeJob = Ds3ClientHelpers
-                    .wrap(ds3Client)
-                    .startWriteJob(bucketName, ds3Objects)
-                    .withMaxParallelRequests(NUM_THREADS);
-            writeJob.transfer(new RepeatStringObjectChannelBuilder(DATA, DATA.length() * 512, DATA_BUFFER_SIZE));
-        }
-    }
-
-    private ImmutableList generateDs3Objects() {
-        final ImmutableList.Builder ds3ObjectBuilder = new ImmutableList.Builder<>();
-
-        for (int i = 0; i < NUM_OBJECTS; ++i) {
-            final Ds3Object ds3Object = new Ds3Object();
-            ds3Object.setName(BASE_FILE_NAME + i);
-            ds3Object.setSize(OBJECT_SIZE);
-            ds3ObjectBuilder.add(ds3Object);
-        }
-
-        return ds3ObjectBuilder.build();
-    }
-
-    private void deleteBuckets(final ImmutableList bucketNames) {
-        for (final String bucketName : bucketNames) {
-            DeleteBucket.INSTANCE.deleteBucket(Ds3ClientHelpers.wrap(ds3Client), bucketName);
-        }
-    }
-
-    private static class RepeatStringObjectChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
-        private final int bufferSize;
-        private final long sizeOfFiles;
-        private String inputDataHeader;
-
-        RepeatStringObjectChannelBuilder(final String inputDataHeader, final int bufferSize, final long sizeOfFile) {
-            this.bufferSize = bufferSize;
-            this.sizeOfFiles = sizeOfFile;
-            this.inputDataHeader = inputDataHeader;
-        }
-
-        @Override public SeekableByteChannel buildChannel(final String key ) {
-            return new RepeatStringByteChannel( inputDataHeader + key, this.bufferSize, this.sizeOfFiles );
-        }
-
-        private static class RepeatStringByteChannel implements SeekableByteChannel {
-            final private byte[] backingArray;
-            final private int bufferSize;
-            final private long limit;
-            private boolean isOpen;
-            private int position;
-
-            RepeatStringByteChannel( final String inputData, final int bufferSize, final long size ) {
-                this.bufferSize = bufferSize;
-                final byte[] bytes = new byte[ bufferSize ];
-                final byte[] stringBytes = inputData.getBytes();
-                int stringPosition = 0;
-                for ( int i = 0; i < bufferSize; i++ ) {
-                    bytes[ i ] = stringBytes[ stringPosition ];
-                    if ( ++stringPosition >= inputData.length() )
-                    {
-                        stringPosition = 0;
-                    }
-                }
-
-                backingArray = bytes;
-
-                this.position = 0;
-                this.limit = size;
-                this.isOpen = true;
-            }
-
-            public void close() {
-                this.isOpen = false;
-            }
-
-            public boolean isOpen() {
-                return this.isOpen;
-            }
-
-            public long position() {
-                return ( long ) this.position;
-            }
-
-            public SeekableByteChannel position( final long newPosition ) {
-                this.position = ( int ) newPosition;
-                return this;
-            }
-
-            public int read( final ByteBuffer dst ) {
-                final int amountToRead = Math.min( dst.remaining(), this.bufferSize );
-                dst.put( this.backingArray, 0, amountToRead );
-                return amountToRead;
-            }
-
-            public long size() {
-                return this.limit;
-            }
-
-            public SeekableByteChannel truncate( final long size ) {
-                return this;
-            }
-
-            public int write( final ByteBuffer src ) {
-                final int amountToWrite = Math.min( src.remaining(), this.bufferSize );
-                src.get( this.backingArray, 0, amountToWrite );
-                return amountToWrite;
-            }
-        }
-    }
-}

From 2db2848f2b8f753036d3cd905f21829e4f962c49 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Wed, 14 Feb 2018 16:53:03 -0700
Subject: [PATCH 136/229] I hate checked exceptions.

---
 .../com/spectralogic/ds3client/integration/ResourceTest.kt     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/ResourceTest.kt b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/ResourceTest.kt
index 0860bdded..6f58aec9b 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/ResourceTest.kt
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/ResourceTest.kt
@@ -27,6 +27,8 @@ import org.junit.AfterClass
 import org.junit.BeforeClass
 import org.junit.Test
 import java.io.IOException
+import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.lessThan
 
 class ResourceTest {
     private companion object {
@@ -66,6 +68,7 @@ class ResourceTest {
         try {
             createBuckets(bucketNames)
             populateBuckets(bucketNames)
+            assertThat(Thread.activeCount(), lessThan(numThreads * 2))
         } finally {
             deleteBuckets(bucketNames)
         }

From 3bbdda4f4bc78c2c0689b320035b25a0cd4f7fd7 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Thu, 15 Feb 2018 12:18:55 -0700
Subject: [PATCH 137/229] We're not using the listening part of a
 ListeningExecutorService.  Client code is notified that a transfer completes
 by attaching an ObjectCompletedListener, giving them basically the same thing
 a listening executor does.  Changing the submit call to execute results is a
 smaller memory profile.

---
 .../transferstrategy/SingleThreadedTransferStrategy.java       | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java
index f64afb5c5..7991fc2b7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java
@@ -15,7 +15,6 @@
 
 package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
 
-import com.google.common.util.concurrent.MoreExecutors;
 import com.spectralogic.ds3client.helpers.JobState;
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
 import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy;
@@ -35,7 +34,7 @@ public SingleThreadedTransferStrategy(final BlobStrategy blobStrategy,
     {
         super(blobStrategy,
               jobState,
-              MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()),
+              Executors.newSingleThreadExecutor(),
               eventDispatcher,
               masterObjectList,
               failureActivity);

From 2cd717481c290a493a547650d07d061ab4c91edd Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Fri, 16 Feb 2018 10:48:08 -0700
Subject: [PATCH 138/229] We're not using the listening part of a
 ListeningExecutorService.  Client code is notified that a transfer completes
 by attaching an ObjectCompletedListener, giving them basically the same thing
 a listening executor does.  Changing the submit call to execute results is a
 smaller memory profile.

---
 .../transferstrategy/AbstractTransferStrategy.java    | 11 ++++-------
 .../MultiThreadedTransferStrategy.java                |  3 +--
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index 0b5df66d0..097a2200a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -17,7 +17,6 @@
 
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Iterables;
-import com.google.common.util.concurrent.ListeningExecutorService;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
 import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
@@ -31,8 +30,8 @@
 
 import java.io.IOException;
 import java.util.NoSuchElementException;
-import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -49,7 +48,7 @@ abstract class AbstractTransferStrategy implements TransferStrategy {
 
     private final BlobStrategy blobStrategy;
     private final JobState jobState;
-    private final ListeningExecutorService executorService;
+    private final ExecutorService executorService;
     private final EventDispatcher eventDispatcher;
     private final MasterObjectList masterObjectList;
     private final FailureEvent.FailureActivity failureActivity;
@@ -74,7 +73,7 @@ abstract class AbstractTransferStrategy implements TransferStrategy {
      */
     public AbstractTransferStrategy(final BlobStrategy blobStrategy,
                                     final JobState jobState,
-                                    final ListeningExecutorService executorService,
+                                    final ExecutorService executorService,
                                     final EventDispatcher eventDispatcher,
                                     final MasterObjectList masterObjectList,
                                     final FailureEvent.FailureActivity failureActivity)
@@ -170,7 +169,7 @@ private void transferJobParts(final Iterable jobParts,
                                   final CountDownLatch countDownLatch,
                                   final AtomicInteger numBlobsTransferred) throws IOException {
         for (final JobPart jobPart : jobParts) {
-            executorService.submit((Callable) () -> {
+            executorService.execute(() -> {
                 try {
                     transferMethod.transferJobPart(jobPart);
                 } catch (final RuntimeException e) {
@@ -184,8 +183,6 @@ private void transferJobParts(final Iterable jobParts,
                     numBlobsTransferred.decrementAndGet();
                     countDownLatch.countDown();
                 }
-
-                return null;
             });
         }
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java
index bc79af4e9..bbf0197ef 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java
@@ -15,7 +15,6 @@
 
 package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
 
-import com.google.common.util.concurrent.MoreExecutors;
 import com.spectralogic.ds3client.helpers.JobState;
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
 import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy;
@@ -36,7 +35,7 @@ public MultiThreadedTransferStrategy(final BlobStrategy blobStrategy,
     {
         super(blobStrategy,
               jobState,
-              MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numConcurrentTransferThreads)),
+              Executors.newFixedThreadPool(numConcurrentTransferThreads),
               eventDispatcher,
               masterObjectList,
               failureActivity);

From 61a341152e3897d28952eadd6dcc97ea4a117e7e Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 19 Feb 2018 13:21:35 -0700
Subject: [PATCH 139/229] 4.1 API

---
 .../GetTapeDrivesSpectraS3Request.java        | 15 ++++++++++
 ...ModifyDataPathBackendSpectraS3Request.java | 28 +++++++++++++++++++
 .../ModifyTapeDriveSpectraS3Request.java      | 15 ++++++++++
 .../ModifyTapePartitionSpectraS3Request.java  | 28 +++++++++++++++++++
 .../ds3client/models/DataPathBackend.java     | 24 ++++++++++++++++
 .../models/DetailedTapePartition.java         | 24 ++++++++++++++++
 .../models/NamedDetailedTapePartition.java    | 24 ++++++++++++++++
 .../ds3client/models/ReservedTaskType.java    | 23 +++++++++++++++
 .../ds3client/models/TapeDrive.java           | 12 ++++++++
 .../ds3client/models/TapePartition.java       | 24 ++++++++++++++++
 10 files changed, 217 insertions(+)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ReservedTaskType.java

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapeDrivesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapeDrivesSpectraS3Request.java
index 3480caaa4..984a69be7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapeDrivesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapeDrivesSpectraS3Request.java
@@ -20,6 +20,7 @@
 import com.spectralogic.ds3client.commands.interfaces.AbstractPaginationRequest;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import com.spectralogic.ds3client.models.ReservedTaskType;
 import com.spectralogic.ds3client.models.TapeDriveState;
 import com.spectralogic.ds3client.models.TapeDriveType;
 
@@ -37,6 +38,8 @@ public class GetTapeDrivesSpectraS3Request extends AbstractPaginationRequest {
 
     private String partitionId;
 
+    private ReservedTaskType reservedTaskType;
+
     private String serialNumber;
 
     private TapeDriveState state;
@@ -103,6 +106,13 @@ public GetTapeDrivesSpectraS3Request withPartitionId(final String partitionId) {
     }
 
 
+    public GetTapeDrivesSpectraS3Request withReservedTaskType(final ReservedTaskType reservedTaskType) {
+        this.reservedTaskType = reservedTaskType;
+        this.updateQueryParam("reserved_task_type", reservedTaskType);
+        return this;
+    }
+
+
     public GetTapeDrivesSpectraS3Request withSerialNumber(final String serialNumber) {
         this.serialNumber = serialNumber;
         this.updateQueryParam("serial_number", serialNumber);
@@ -160,6 +170,11 @@ public String getPartitionId() {
     }
 
 
+    public ReservedTaskType getReservedTaskType() {
+        return this.reservedTaskType;
+    }
+
+
     public String getSerialNumber() {
         return this.serialNumber;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPathBackendSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPathBackendSpectraS3Request.java
index c6fb0ff64..68851dc63 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPathBackendSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPathBackendSpectraS3Request.java
@@ -30,10 +30,14 @@ public class ModifyDataPathBackendSpectraS3Request extends AbstractRequest {
     
     private boolean activated;
 
+    private boolean allowNewJobRequests;
+
     private Integer autoActivateTimeoutInMins;
 
     private AutoInspectMode autoInspect;
 
+    private int cacheAvailableRetryAfterInSeconds;
+
     private ImportConflictResolutionMode defaultImportConflictResolutionMode;
 
     private Priority defaultVerifyDataAfterImport;
@@ -62,6 +66,13 @@ public ModifyDataPathBackendSpectraS3Request withActivated(final boolean activat
     }
 
 
+    public ModifyDataPathBackendSpectraS3Request withAllowNewJobRequests(final boolean allowNewJobRequests) {
+        this.allowNewJobRequests = allowNewJobRequests;
+        this.updateQueryParam("allow_new_job_requests", allowNewJobRequests);
+        return this;
+    }
+
+
     public ModifyDataPathBackendSpectraS3Request withAutoActivateTimeoutInMins(final Integer autoActivateTimeoutInMins) {
         this.autoActivateTimeoutInMins = autoActivateTimeoutInMins;
         this.updateQueryParam("auto_activate_timeout_in_mins", autoActivateTimeoutInMins);
@@ -76,6 +87,13 @@ public ModifyDataPathBackendSpectraS3Request withAutoInspect(final AutoInspectMo
     }
 
 
+    public ModifyDataPathBackendSpectraS3Request withCacheAvailableRetryAfterInSeconds(final int cacheAvailableRetryAfterInSeconds) {
+        this.cacheAvailableRetryAfterInSeconds = cacheAvailableRetryAfterInSeconds;
+        this.updateQueryParam("cache_available_retry_after_in_seconds", cacheAvailableRetryAfterInSeconds);
+        return this;
+    }
+
+
     public ModifyDataPathBackendSpectraS3Request withDefaultImportConflictResolutionMode(final ImportConflictResolutionMode defaultImportConflictResolutionMode) {
         this.defaultImportConflictResolutionMode = defaultImportConflictResolutionMode;
         this.updateQueryParam("default_import_conflict_resolution_mode", defaultImportConflictResolutionMode);
@@ -141,6 +159,11 @@ public boolean getActivated() {
     }
 
 
+    public boolean getAllowNewJobRequests() {
+        return this.allowNewJobRequests;
+    }
+
+
     public Integer getAutoActivateTimeoutInMins() {
         return this.autoActivateTimeoutInMins;
     }
@@ -151,6 +174,11 @@ public AutoInspectMode getAutoInspect() {
     }
 
 
+    public int getCacheAvailableRetryAfterInSeconds() {
+        return this.cacheAvailableRetryAfterInSeconds;
+    }
+
+
     public ImportConflictResolutionMode getDefaultImportConflictResolutionMode() {
         return this.defaultImportConflictResolutionMode;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Request.java
index dbb9bc0f7..d9e83cbeb 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapeDriveSpectraS3Request.java
@@ -19,6 +19,7 @@
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.spectralogic.ds3client.models.Quiesced;
+import com.spectralogic.ds3client.models.ReservedTaskType;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
@@ -30,6 +31,8 @@ public class ModifyTapeDriveSpectraS3Request extends AbstractRequest {
 
     private Quiesced quiesced;
 
+    private ReservedTaskType reservedTaskType;
+
     // Constructor
     
     
@@ -51,6 +54,13 @@ public ModifyTapeDriveSpectraS3Request withQuiesced(final Quiesced quiesced) {
     }
 
 
+    public ModifyTapeDriveSpectraS3Request withReservedTaskType(final ReservedTaskType reservedTaskType) {
+        this.reservedTaskType = reservedTaskType;
+        this.updateQueryParam("reserved_task_type", reservedTaskType);
+        return this;
+    }
+
+
 
     @Override
     public HttpVerb getVerb() {
@@ -71,4 +81,9 @@ public Quiesced getQuiesced() {
         return this.quiesced;
     }
 
+
+    public ReservedTaskType getReservedTaskType() {
+        return this.reservedTaskType;
+    }
+
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
index efc6b9149..d59f129a3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
@@ -27,6 +27,10 @@ public class ModifyTapePartitionSpectraS3Request extends AbstractRequest {
     
     private final String tapePartition;
 
+    private int minimumReadReservedDrives;
+
+    private int minimumWriteReservedDrives;
+
     private Quiesced quiesced;
 
     // Constructor
@@ -37,6 +41,20 @@ public ModifyTapePartitionSpectraS3Request(final String tapePartition) {
         
     }
 
+    public ModifyTapePartitionSpectraS3Request withMinimumReadReservedDrives(final int minimumReadReservedDrives) {
+        this.minimumReadReservedDrives = minimumReadReservedDrives;
+        this.updateQueryParam("minimum_read_reserved_drives", minimumReadReservedDrives);
+        return this;
+    }
+
+
+    public ModifyTapePartitionSpectraS3Request withMinimumWriteReservedDrives(final int minimumWriteReservedDrives) {
+        this.minimumWriteReservedDrives = minimumWriteReservedDrives;
+        this.updateQueryParam("minimum_write_reserved_drives", minimumWriteReservedDrives);
+        return this;
+    }
+
+
     public ModifyTapePartitionSpectraS3Request withQuiesced(final Quiesced quiesced) {
         this.quiesced = quiesced;
         this.updateQueryParam("quiesced", quiesced);
@@ -60,6 +78,16 @@ public String getTapePartition() {
     }
 
 
+    public int getMinimumReadReservedDrives() {
+        return this.minimumReadReservedDrives;
+    }
+
+
+    public int getMinimumWriteReservedDrives() {
+        return this.minimumWriteReservedDrives;
+    }
+
+
     public Quiesced getQuiesced() {
         return this.quiesced;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
index 04465a50a..b5e9cbba1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
@@ -28,12 +28,18 @@ public class DataPathBackend {
     @JsonProperty("Activated")
     private boolean activated;
 
+    @JsonProperty("AllowNewJobRequests")
+    private boolean allowNewJobRequests;
+
     @JsonProperty("AutoActivateTimeoutInMins")
     private Integer autoActivateTimeoutInMins;
 
     @JsonProperty("AutoInspect")
     private AutoInspectMode autoInspect;
 
+    @JsonProperty("CacheAvailableRetryAfterInSeconds")
+    private int cacheAvailableRetryAfterInSeconds;
+
     @JsonProperty("DefaultImportConflictResolutionMode")
     private ImportConflictResolutionMode defaultImportConflictResolutionMode;
 
@@ -80,6 +86,15 @@ public void setActivated(final boolean activated) {
     }
 
 
+    public boolean getAllowNewJobRequests() {
+        return this.allowNewJobRequests;
+    }
+
+    public void setAllowNewJobRequests(final boolean allowNewJobRequests) {
+        this.allowNewJobRequests = allowNewJobRequests;
+    }
+
+
     public Integer getAutoActivateTimeoutInMins() {
         return this.autoActivateTimeoutInMins;
     }
@@ -98,6 +113,15 @@ public void setAutoInspect(final AutoInspectMode autoInspect) {
     }
 
 
+    public int getCacheAvailableRetryAfterInSeconds() {
+        return this.cacheAvailableRetryAfterInSeconds;
+    }
+
+    public void setCacheAvailableRetryAfterInSeconds(final int cacheAvailableRetryAfterInSeconds) {
+        this.cacheAvailableRetryAfterInSeconds = cacheAvailableRetryAfterInSeconds;
+    }
+
+
     public ImportConflictResolutionMode getDefaultImportConflictResolutionMode() {
         return this.defaultImportConflictResolutionMode;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
index b57fc32bf..8008ca4f3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
@@ -46,6 +46,12 @@ public class DetailedTapePartition {
     @JsonProperty("LibraryId")
     private UUID libraryId;
 
+    @JsonProperty("MinimumReadReservedDrives")
+    private int minimumReadReservedDrives;
+
+    @JsonProperty("MinimumWriteReservedDrives")
+    private int minimumWriteReservedDrives;
+
     @JsonProperty("Name")
     private String name;
 
@@ -126,6 +132,24 @@ public void setLibraryId(final UUID libraryId) {
     }
 
 
+    public int getMinimumReadReservedDrives() {
+        return this.minimumReadReservedDrives;
+    }
+
+    public void setMinimumReadReservedDrives(final int minimumReadReservedDrives) {
+        this.minimumReadReservedDrives = minimumReadReservedDrives;
+    }
+
+
+    public int getMinimumWriteReservedDrives() {
+        return this.minimumWriteReservedDrives;
+    }
+
+    public void setMinimumWriteReservedDrives(final int minimumWriteReservedDrives) {
+        this.minimumWriteReservedDrives = minimumWriteReservedDrives;
+    }
+
+
     public String getName() {
         return this.name;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
index 5d79e7ce6..3a683063e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
@@ -46,6 +46,12 @@ public class NamedDetailedTapePartition {
     @JsonProperty("LibraryId")
     private UUID libraryId;
 
+    @JsonProperty("MinimumReadReservedDrives")
+    private int minimumReadReservedDrives;
+
+    @JsonProperty("MinimumWriteReservedDrives")
+    private int minimumWriteReservedDrives;
+
     @JsonProperty("Name")
     private String name;
 
@@ -126,6 +132,24 @@ public void setLibraryId(final UUID libraryId) {
     }
 
 
+    public int getMinimumReadReservedDrives() {
+        return this.minimumReadReservedDrives;
+    }
+
+    public void setMinimumReadReservedDrives(final int minimumReadReservedDrives) {
+        this.minimumReadReservedDrives = minimumReadReservedDrives;
+    }
+
+
+    public int getMinimumWriteReservedDrives() {
+        return this.minimumWriteReservedDrives;
+    }
+
+    public void setMinimumWriteReservedDrives(final int minimumWriteReservedDrives) {
+        this.minimumWriteReservedDrives = minimumWriteReservedDrives;
+    }
+
+
     public String getName() {
         return this.name;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ReservedTaskType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ReservedTaskType.java
new file mode 100644
index 000000000..7003a9bf8
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ReservedTaskType.java
@@ -0,0 +1,23 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.models;
+
+public enum ReservedTaskType {
+    ANY,
+    READ,
+    WRITE
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
index fa9f0e1f9..c82701348 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeDrive.java
@@ -49,6 +49,9 @@ public class TapeDrive {
     @JsonProperty("Quiesced")
     private Quiesced quiesced;
 
+    @JsonProperty("ReservedTaskType")
+    private ReservedTaskType reservedTaskType;
+
     @JsonProperty("SerialNumber")
     private String serialNumber;
 
@@ -140,6 +143,15 @@ public void setQuiesced(final Quiesced quiesced) {
     }
 
 
+    public ReservedTaskType getReservedTaskType() {
+        return this.reservedTaskType;
+    }
+
+    public void setReservedTaskType(final ReservedTaskType reservedTaskType) {
+        this.reservedTaskType = reservedTaskType;
+    }
+
+
     public String getSerialNumber() {
         return this.serialNumber;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
index eac412bee..52a962cd2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
@@ -39,6 +39,12 @@ public class TapePartition {
     @JsonProperty("LibraryId")
     private UUID libraryId;
 
+    @JsonProperty("MinimumReadReservedDrives")
+    private int minimumReadReservedDrives;
+
+    @JsonProperty("MinimumWriteReservedDrives")
+    private int minimumWriteReservedDrives;
+
     @JsonProperty("Name")
     private String name;
 
@@ -106,6 +112,24 @@ public void setLibraryId(final UUID libraryId) {
     }
 
 
+    public int getMinimumReadReservedDrives() {
+        return this.minimumReadReservedDrives;
+    }
+
+    public void setMinimumReadReservedDrives(final int minimumReadReservedDrives) {
+        this.minimumReadReservedDrives = minimumReadReservedDrives;
+    }
+
+
+    public int getMinimumWriteReservedDrives() {
+        return this.minimumWriteReservedDrives;
+    }
+
+    public void setMinimumWriteReservedDrives(final int minimumWriteReservedDrives) {
+        this.minimumWriteReservedDrives = minimumWriteReservedDrives;
+    }
+
+
     public String getName() {
         return this.name;
     }

From 83b91f8992dc029e31ee2c34567a478da064ce3d Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Tue, 20 Feb 2018 17:53:34 -0700
Subject: [PATCH 140/229] Added blob checksum header parsing

---
 .../commands/HeadObjectResponse.java          | 16 +++-
 .../parsers/HeadObjectResponseParser.java     | 15 ++--
 .../parsers/utils/ResponseParserUtils.java    | 90 +++++++++++++++++++
 .../ds3client/Ds3Client_Test.java             | 21 ++++-
 .../spectralogic/ds3client/MockedHeaders.java |  2 +-
 .../utils/ResponseParserUtils_Test.java       | 81 ++++++++++++++++-
 6 files changed, 214 insertions(+), 11 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectResponse.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectResponse.java
index 8097b9fc1..675b09e41 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectResponse.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectResponse.java
@@ -16,6 +16,7 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands;
 
+import com.google.common.collect.ImmutableMap;
 import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
 import com.spectralogic.ds3client.networking.Metadata;
 import com.spectralogic.ds3client.models.ChecksumType;
@@ -30,11 +31,17 @@ public enum Status { EXISTS, DOESNTEXIST, UNKNOWN }
 
     private final Status status;
 
-    public HeadObjectResponse(final Metadata metadata, final long objectSize, final Status status, final String checksum, final ChecksumType.Type checksumType) {
+    private final ChecksumType.Type blobChecksumType;
+    /** Maps blob start offset to blob checksum */
+    private final ImmutableMap blobChecksums;
+
+    public HeadObjectResponse(final Metadata metadata, final long objectSize, final Status status, final String checksum, final ChecksumType.Type checksumType, final ChecksumType.Type blobChecksumType, final ImmutableMap blobChecksums) {
         super(checksum, checksumType);
         this.metadata = metadata;
         this.objectSize = objectSize;
         this.status = status;
+        this.blobChecksumType = blobChecksumType;
+        this.blobChecksums = blobChecksums;
     }
 
     public Metadata getMetadata() {
@@ -49,4 +56,11 @@ public Status getStatus() {
         return this.status;
     }
 
+    public ChecksumType.Type getBlobChecksumType() {
+        return blobChecksumType;
+    }
+
+    public ImmutableMap getBlobChecksums() {
+        return blobChecksums;
+    }
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/HeadObjectResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/HeadObjectResponseParser.java
index 94e887093..323e936bf 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/HeadObjectResponseParser.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/HeadObjectResponseParser.java
@@ -16,15 +16,18 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands.parsers;
 
-import com.spectralogic.ds3client.commands.interfaces.MetadataImpl;
-import com.spectralogic.ds3client.networking.Metadata;
+import com.google.common.collect.ImmutableMap;
 import com.spectralogic.ds3client.commands.HeadObjectResponse;
+import com.spectralogic.ds3client.commands.interfaces.MetadataImpl;
 import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
 import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.networking.Metadata;
 import com.spectralogic.ds3client.networking.WebResponse;
+
 import java.io.IOException;
 
-import static com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils.getSizeFromHeaders;
+import static com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils.*;
 
 public class HeadObjectResponseParser extends AbstractResponseParser {
     private final int[] expectedStatusCodes = new int[]{200, 404};
@@ -37,10 +40,12 @@ public HeadObjectResponse parseXmlResponse(final WebResponse response) throws IO
             final long objectSize = getSizeFromHeaders(response.getHeaders());
             switch (statusCode) {
             case 200:
-                return new HeadObjectResponse(metadata, objectSize, HeadObjectResponse.Status.EXISTS, this.getChecksum(), this.getChecksumType());
+                final ChecksumType.Type blobChecksumType = getBlobChecksumType(response.getHeaders());
+                final ImmutableMap blobChecksumMap = getBlobChecksumMap(response.getHeaders());
+                return new HeadObjectResponse(metadata, objectSize, HeadObjectResponse.Status.EXISTS, this.getChecksum(), this.getChecksumType(), blobChecksumType, blobChecksumMap);
 
             case 404:
-                return new HeadObjectResponse(metadata, objectSize, HeadObjectResponse.Status.DOESNTEXIST, this.getChecksum(), this.getChecksumType());
+                return new HeadObjectResponse(metadata, objectSize, HeadObjectResponse.Status.DOESNTEXIST, this.getChecksum(), this.getChecksumType(), ChecksumType.Type.NONE, ImmutableMap.of());
 
             default:
                 assert false: "validateStatusCode should have made it impossible to reach this line";
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
index 58349a1cb..d7b26fae0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
@@ -17,6 +17,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.models.Error;
@@ -41,6 +42,9 @@ public final class ResponseParserUtils {
     private final static Logger LOG = LoggerFactory.getLogger(ResponseParserUtils.class);
     final public static String UTF8 = "UTF-8";
 
+    private static final String BLOB_CHECKSUM_TYPE_HEADER = "ds3-blob-checksum-type";
+    private static final String BLOB_CHECKSUM_HEADER = "ds3-blob-checksum-offset-";
+
     private ResponseParserUtils() {
         // pass
     }
@@ -145,4 +149,90 @@ private static String readResponseString(final WebResponse response) throws IOEx
             return writer.toString();
         }
     }
+
+    /**
+     * Retrieves the blob checksum type from the response headers. If there is no blob checksum
+     * type header, than NONE is returned. If there is more than one value for the blob checksum
+     * type header, than an illegal argument exception is thrown.
+     */
+    public static ChecksumType.Type getBlobChecksumType(final Headers headers) {
+        final List types = headers.get(BLOB_CHECKSUM_TYPE_HEADER);
+        switch (types.size()) {
+            case 0:
+                return ChecksumType.Type.NONE;
+            case 1:
+                return toBlobChecksumType(types.get(0));
+            default:
+                throw new IllegalArgumentException(String.format("Response has more than one header value for '%s'", BLOB_CHECKSUM_TYPE_HEADER));
+        }
+    }
+
+    /**
+     * Converts the response header value for {@code BLOB_CHECKSUM_TYPE_HEADER} into a {@code ChecksumType.Type}
+     */
+    static ChecksumType.Type toBlobChecksumType(final String checksumType) {
+        if (Guard.isStringNullOrEmpty(checksumType)) {
+            return ChecksumType.Type.NONE;
+        }
+        switch (checksumType) {
+            case "CRC_32":
+                return ChecksumType.Type.CRC_32;
+            case "CRC_32C":
+                return ChecksumType.Type.CRC_32C;
+            case "MD5":
+                return ChecksumType.Type.MD5;
+            case "SHA_256":
+                return ChecksumType.Type.SHA_256;
+            case "SHA_512":
+                return ChecksumType.Type.SHA_512;
+            case "":
+                return ChecksumType.Type.NONE;
+            default:
+                throw new IllegalArgumentException(String.format("Unknown value for header '%s': '%s'", BLOB_CHECKSUM_TYPE_HEADER, checksumType));
+        }
+    }
+
+    /**
+     * Retrieves a map of blob offset to blob checksum from response headers.
+     */
+    public static ImmutableMap getBlobChecksumMap(final Headers headers) {
+        if (headers.keys() == null) {
+            return ImmutableMap.of();
+        }
+
+        final ImmutableMap.Builder builder = ImmutableMap.builder();
+
+        for (final String key : headers.keys()) {
+            if (key.startsWith(BLOB_CHECKSUM_HEADER)) {
+                builder.put(getOffsetFromHeaderKey(key), getBlobChecksumValue(key, headers));
+            }
+        }
+
+        return builder.build();
+    }
+
+    /**
+     * Retrieves the offset from the blob checksum header key. This assumes that the
+     * provided key has the prefix {@code BLOB_CHECKSUM_HEADER} and that the postfix can be
+     * converted into a Long.
+     */
+    private static Long getOffsetFromHeaderKey(final String key) {
+        final String offset = key.substring(BLOB_CHECKSUM_HEADER.length());
+        return Long.valueOf(offset);
+    }
+
+    /**
+     * Retrieves the value from the specified header key assuming there is at most one value.
+     * If there are no values for the key, then an empty string is returned.
+     * If there is more than one value for the given key, an illegal argument exception is thrown.
+     */
+    private static String getBlobChecksumValue(final String key, final Headers headers) {
+        final List values = headers.get(key);
+        switch (values.size()) {
+            case 1:
+                return values.get(0);
+            default:
+                throw new IllegalArgumentException(String.format("Expected response to have 1 value for blob checksum header '%s', but found '%d' values", key, values.size()));
+        }
+    }
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index a9809752b..62caa3319 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -53,6 +53,7 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class Ds3Client_Test {
@@ -484,15 +485,33 @@ public void headObjectWithMetadata() throws IOException {
 
         final Map responseHeaders = new HashMap<>();
         responseHeaders.put("x-amz-meta-key", "value");
+        responseHeaders.put("ds3-blob-checksum-type", "MD5");
+        responseHeaders.put("ds3-blob-checksum-offset-0", "4nQGNX4nyz0pi8Hvap79PQ==");
+        responseHeaders.put("ds3-blob-checksum-offset-10485760", "965Aa0/n8DlO1IwXYFh4bg==");
+        responseHeaders.put("ds3-blob-checksum-offset-20971520", "iV2OqJaXJ/jmqgRSb1HmFA==");
 
 
         final HeadObjectRequest request = new HeadObjectRequest("bucket", "obj");
 
-        MockNetwork
+        final HeadObjectResponse response = MockNetwork
                 .expecting(HttpVerb.HEAD, "/bucket/obj", null, null, null)
                 .returning(200, "", responseHeaders)
                 .asClient()
                 .headObject(request);
+
+        final ImmutableMap expectedBlobChecksums = ImmutableMap.of(
+                0L, "4nQGNX4nyz0pi8Hvap79PQ==",
+                10485760L, "965Aa0/n8DlO1IwXYFh4bg==",
+                20971520L, "iV2OqJaXJ/jmqgRSb1HmFA=="
+        );
+
+        assertThat(response.getBlobChecksumType(), is(ChecksumType.Type.MD5));
+        assertThat(response.getBlobChecksums().size(), is(expectedBlobChecksums.size()));
+
+        for (final Map.Entry entry : expectedBlobChecksums.entrySet()) {
+            assertTrue(response.getBlobChecksums().containsKey(entry.getKey()));
+            assertThat(response.getBlobChecksums().get(entry.getKey()), is(entry.getValue()));
+        }
     }
 
     @Test
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockedHeaders.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockedHeaders.java
index 9563fe716..c4d6946fe 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockedHeaders.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockedHeaders.java
@@ -45,7 +45,7 @@ public List get(final String key) {
 
     @Override
     public Set keys() {
-        return null;
+        return headerValues.keySet();
     }
 
     protected Set getRealKeys() {
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
index eafa594f0..7021d785a 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
@@ -17,14 +17,24 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.spectralogic.ds3client.MockedHeaders;
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.networking.Headers;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.util.Map;
+
+import static com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils.*;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 public class ResponseParserUtils_Test {
 
+    private static final Logger LOG = LoggerFactory.getLogger(ResponseParserUtils_Test.class);
+
     @Test
     public void getRequestId_Test() {
         final String requestId = "123";
@@ -32,7 +42,7 @@ public void getRequestId_Test() {
                 "x-amz-request-id", requestId,
                 "Content-Type", "Text/xml");
 
-        final String result = ResponseParserUtils.getRequestId(new MockedHeaders(headers));
+        final String result = getRequestId(new MockedHeaders(headers));
         assertThat(result, is(requestId));
     }
 
@@ -41,13 +51,78 @@ public void getRequestId_WithoutRequestId_Test() {
         final ImmutableMap headers = ImmutableMap.of(
                 "Content-Type", "Text/xml");
 
-        final String result = ResponseParserUtils.getRequestId(new MockedHeaders(headers));
+        final String result = getRequestId(new MockedHeaders(headers));
         assertThat(result, is(nullValue()));
     }
 
     @Test
     public void getRequestId_EmptyHeaders_Test() {
-        final String result = ResponseParserUtils.getRequestId(new MockedHeaders(ImmutableMap.of()));
+        final String result = getRequestId(new MockedHeaders(ImmutableMap.of()));
         assertThat(result, is(nullValue()));
     }
+
+    @Test
+    public void getBlobChecksumMapNoHeadersTest() {
+        final ImmutableMap result = getBlobChecksumMap(new MockedHeaders(ImmutableMap.of()));
+        assertThat(result.size(), is(0));
+    }
+
+    @Test
+    public void getBlobChecksumMapNoChecksumHeadersTest() {
+        final ImmutableMap headers = ImmutableMap.of("key1", "value1", "key2", "value2");
+        final ImmutableMap result = getBlobChecksumMap(new MockedHeaders(headers));
+        assertThat(result.size(), is(0));
+    }
+
+    private static final ImmutableMap TEST_RESPONSE_HEADERS = ImmutableMap.of(
+            "ds3-blob-checksum-type", "MD5",
+            "ds3-blob-checksum-offset-0", "4nQGNX4nyz0pi8Hvap79PQ==",
+            "ds3-blob-checksum-offset-10485760", "965Aa0/n8DlO1IwXYFh4bg==",
+            "ds3-blob-checksum-offset-20971520", "iV2OqJaXJ/jmqgRSb1HmFA==",
+            "x-amz-request-id", "895767"
+    );
+
+    @Test
+    public void getBlobChecksumMapWithChecksumHeadersTest() {
+        final ImmutableMap expectedMap = ImmutableMap.of(
+                0L, "4nQGNX4nyz0pi8Hvap79PQ==",
+                10485760L, "965Aa0/n8DlO1IwXYFh4bg==",
+                20971520L, "iV2OqJaXJ/jmqgRSb1HmFA=="
+        );
+
+        final ImmutableMap result = getBlobChecksumMap(new MockedHeaders(TEST_RESPONSE_HEADERS));
+        assertThat(result.size(), is(expectedMap.size()));
+
+        for (final Map.Entry entry : expectedMap.entrySet()) {
+            assertTrue(result.containsKey(entry.getKey()));
+            assertThat(result.get(entry.getKey()), is(entry.getValue()));
+        }
+    }
+
+    @Test
+    public void toBlobChecksumTypeTest() {
+        assertThat(toBlobChecksumType("CRC_32"), is(ChecksumType.Type.CRC_32));
+        assertThat(toBlobChecksumType("CRC_32C"), is(ChecksumType.Type.CRC_32C));
+        assertThat(toBlobChecksumType("MD5"), is(ChecksumType.Type.MD5));
+        assertThat(toBlobChecksumType("SHA_256"), is(ChecksumType.Type.SHA_256));
+        assertThat(toBlobChecksumType("SHA_512"), is(ChecksumType.Type.SHA_512));
+        assertThat(toBlobChecksumType(""), is(ChecksumType.Type.NONE));
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void toBlobChecksumTypeUnknownChecksumType() {
+        assertThat(toBlobChecksumType("unexpected text"), is(ChecksumType.Type.NONE));
+    }
+
+    @Test
+    public void getBlobChecksumTypeNoHeaderTest() {
+        final ChecksumType.Type result = getBlobChecksumType(new MockedHeaders(ImmutableMap.of()));
+        assertThat(result, is(ChecksumType.Type.NONE));
+    }
+
+    @Test
+    public void getBlobChecksumTypeTest() {
+        final ChecksumType.Type result = getBlobChecksumType(new MockedHeaders(TEST_RESPONSE_HEADERS));
+        assertThat(result, is(ChecksumType.Type.MD5));
+    }
 }

From 17c6b2af6709c61ff8a135ca96d6baad5431a14f Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 21 Feb 2018 10:01:46 -0700
Subject: [PATCH 141/229] Updating to use streams in resposne parser util

---
 .../commands/parsers/utils/ResponseParserUtils.java      | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
index d7b26fae0..6514b96c7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
@@ -201,12 +201,9 @@ public static ImmutableMap getBlobChecksumMap(final Headers header
         }
 
         final ImmutableMap.Builder builder = ImmutableMap.builder();
-
-        for (final String key : headers.keys()) {
-            if (key.startsWith(BLOB_CHECKSUM_HEADER)) {
-                builder.put(getOffsetFromHeaderKey(key), getBlobChecksumValue(key, headers));
-            }
-        }
+        headers.keys().stream()
+                .filter(key -> key.startsWith(BLOB_CHECKSUM_HEADER))
+                .forEach(key -> builder.put(getOffsetFromHeaderKey(key), getBlobChecksumValue(key, headers)));
 
         return builder.build();
     }

From 8dfa11e053a3a4b87ff362c2a38d89a5f2df7587 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 21 Feb 2018 11:04:49 -0700
Subject: [PATCH 142/229] alphabetizing head object response constructor params

---
 .../commands/HeadObjectResponse.java          | 31 ++++++++++---------
 .../parsers/HeadObjectResponseParser.java     | 11 +++----
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectResponse.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectResponse.java
index 675b09e41..ed223c323 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectResponse.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectResponse.java
@@ -16,32 +16,40 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands;
 
-import com.google.common.collect.ImmutableMap;
 import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
 import com.spectralogic.ds3client.networking.Metadata;
 import com.spectralogic.ds3client.models.ChecksumType;
+import com.google.common.collect.ImmutableMap;
 
 public class HeadObjectResponse extends AbstractResponse {
 
     public enum Status { EXISTS, DOESNTEXIST, UNKNOWN }
     
+    private final ImmutableMap blobChecksums;
+
+    private final ChecksumType.Type blobChecksumType;
+
     private final Metadata metadata;
 
     private final long objectSize;
 
     private final Status status;
 
-    private final ChecksumType.Type blobChecksumType;
-    /** Maps blob start offset to blob checksum */
-    private final ImmutableMap blobChecksums;
-
-    public HeadObjectResponse(final Metadata metadata, final long objectSize, final Status status, final String checksum, final ChecksumType.Type checksumType, final ChecksumType.Type blobChecksumType, final ImmutableMap blobChecksums) {
+    public HeadObjectResponse(final ImmutableMap blobChecksums, final ChecksumType.Type blobChecksumType, final Metadata metadata, final long objectSize, final Status status, final String checksum, final ChecksumType.Type checksumType) {
         super(checksum, checksumType);
+        this.blobChecksums = blobChecksums;
+        this.blobChecksumType = blobChecksumType;
         this.metadata = metadata;
         this.objectSize = objectSize;
         this.status = status;
-        this.blobChecksumType = blobChecksumType;
-        this.blobChecksums = blobChecksums;
+    }
+
+    public ImmutableMap getBlobChecksums() {
+        return this.blobChecksums;
+    }
+
+    public ChecksumType.Type getBlobChecksumType() {
+        return this.blobChecksumType;
     }
 
     public Metadata getMetadata() {
@@ -56,11 +64,4 @@ public Status getStatus() {
         return this.status;
     }
 
-    public ChecksumType.Type getBlobChecksumType() {
-        return blobChecksumType;
-    }
-
-    public ImmutableMap getBlobChecksums() {
-        return blobChecksums;
-    }
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/HeadObjectResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/HeadObjectResponseParser.java
index 323e936bf..f0e1afca6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/HeadObjectResponseParser.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/HeadObjectResponseParser.java
@@ -16,15 +16,14 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands.parsers;
 
+import com.spectralogic.ds3client.commands.interfaces.MetadataImpl;
+import com.spectralogic.ds3client.networking.Metadata;
 import com.google.common.collect.ImmutableMap;
+import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.commands.HeadObjectResponse;
-import com.spectralogic.ds3client.commands.interfaces.MetadataImpl;
 import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
 import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
-import com.spectralogic.ds3client.models.ChecksumType;
-import com.spectralogic.ds3client.networking.Metadata;
 import com.spectralogic.ds3client.networking.WebResponse;
-
 import java.io.IOException;
 
 import static com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils.*;
@@ -42,10 +41,10 @@ public HeadObjectResponse parseXmlResponse(final WebResponse response) throws IO
             case 200:
                 final ChecksumType.Type blobChecksumType = getBlobChecksumType(response.getHeaders());
                 final ImmutableMap blobChecksumMap = getBlobChecksumMap(response.getHeaders());
-                return new HeadObjectResponse(metadata, objectSize, HeadObjectResponse.Status.EXISTS, this.getChecksum(), this.getChecksumType(), blobChecksumType, blobChecksumMap);
+                return new HeadObjectResponse(blobChecksumMap, blobChecksumType, metadata, objectSize, HeadObjectResponse.Status.EXISTS, this.getChecksum(), this.getChecksumType());
 
             case 404:
-                return new HeadObjectResponse(metadata, objectSize, HeadObjectResponse.Status.DOESNTEXIST, this.getChecksum(), this.getChecksumType(), ChecksumType.Type.NONE, ImmutableMap.of());
+                return new HeadObjectResponse(ImmutableMap.of(), ChecksumType.Type.NONE, metadata, objectSize, HeadObjectResponse.Status.DOESNTEXIST, this.getChecksum(), this.getChecksumType());
 
             default:
                 assert false: "validateStatusCode should have made it impossible to reach this line";

From 870e8a65335a50cbb7aa65396184339297d3e7cd Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 21 Feb 2018 12:26:29 -0700
Subject: [PATCH 143/229] Simplified response parser util function

---
 .../parsers/utils/ResponseParserUtils.java      | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
index 6514b96c7..e9428f036 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils.java
@@ -174,22 +174,7 @@ static ChecksumType.Type toBlobChecksumType(final String checksumType) {
         if (Guard.isStringNullOrEmpty(checksumType)) {
             return ChecksumType.Type.NONE;
         }
-        switch (checksumType) {
-            case "CRC_32":
-                return ChecksumType.Type.CRC_32;
-            case "CRC_32C":
-                return ChecksumType.Type.CRC_32C;
-            case "MD5":
-                return ChecksumType.Type.MD5;
-            case "SHA_256":
-                return ChecksumType.Type.SHA_256;
-            case "SHA_512":
-                return ChecksumType.Type.SHA_512;
-            case "":
-                return ChecksumType.Type.NONE;
-            default:
-                throw new IllegalArgumentException(String.format("Unknown value for header '%s': '%s'", BLOB_CHECKSUM_TYPE_HEADER, checksumType));
-        }
+        return ChecksumType.Type.valueOf(checksumType);
     }
 
     /**

From 467d04917f46bb5deb3df3e6ba8083ebeae8c8a7 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 14 Feb 2018 15:28:30 -0700
Subject: [PATCH 144/229] Adding VersionId to Ds3Object and updating marshaling

---
 .../ds3client/models/bulk/Ds3Object.java      | 81 ++++++++++++++++---
 .../models/bulk/PartialDs3Object.java         | 12 +++
 .../ds3client/serializer/Views.java           | 15 ++++
 .../ds3client/serializer/XmlOutput.java       | 19 ++---
 .../ds3client/serializer/XmlOutput_Test.java  | 24 +++++-
 5 files changed, 128 insertions(+), 23 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/Views.java

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
index ce860a7bf..5c1c9f7bc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
@@ -15,16 +15,24 @@
 
 package com.spectralogic.ds3client.models.bulk;
 
-import com.fasterxml.jackson.annotation.JsonFilter;
+import com.fasterxml.jackson.annotation.JsonView;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
+import com.spectralogic.ds3client.serializer.Views;
+
+import java.util.UUID;
 
-@JsonFilter("sizeFilter")
 public class Ds3Object  {
     @JacksonXmlProperty(isAttribute = true, localName = "Name")
     private String name;
+
+    @JsonView(Views.PutObject.class)
     @JacksonXmlProperty(isAttribute = true, localName = "Size")
     private long size;
 
+    @JsonView(Views.GetObject.class)
+    @JacksonXmlProperty(isAttribute = true, localName = "VersionId")
+    private String versionId;
+
     /**
      * This constructor is used for XML Serialization.
      * The preferred method is to use one of the other two constructors.
@@ -33,6 +41,30 @@ public Ds3Object() {
         //This constructor is used just for serialization.
     }
 
+    /**
+     * Use this constructor when getting a version of a file from DS3.
+     * @param name The name of the object to get from DS3
+     * @param size The size of the object to get from DS3
+     * @param versionId The version ID of the object to get from DS3
+     */
+    public Ds3Object(final String name, final long size, final UUID versionId) {
+        this.name = name;
+        this.size = size;
+        this.versionId = versionId.toString();
+    }
+
+    /**
+     * Use this constructor when getting a version of a file from DS3.
+     * @param name The name of the object to get from DS3
+     * @param size The size of the object to get from DS3
+     * @param versionId The version ID of the object to get from DS3
+     */
+    public Ds3Object(final String name, final long size, final String versionId) {
+        this.name = name;
+        this.size = size;
+        this.versionId = versionId;
+    }
+
     /**
      * Use this constructor when putting files to DS3.
      * @param name The name of the object that will be put to DS3
@@ -41,6 +73,29 @@ public Ds3Object() {
     public Ds3Object(final String name, final long size) {
         this.name = name;
         this.size = size;
+        this.versionId = null;
+    }
+
+    /**
+     * Use this constructor when getting files from DS3.
+     * @param name the name of the object to get from DS3
+     * @param versionId The version ID of the object to get from DS3
+     */
+    public Ds3Object(final String name, final UUID versionId) {
+        this.name = name;
+        this.size = 0;
+        this.versionId = versionId.toString();
+    }
+
+    /**
+     * Use this constructor when getting files from DS3.
+     * @param name the name of the object to get from DS3
+     * @param versionId The version ID of the object to get from DS3
+     */
+    public Ds3Object(final String name, final String versionId) {
+        this.name = name;
+        this.size = 0;
+        this.versionId = versionId;
     }
 
     /**
@@ -50,6 +105,7 @@ public Ds3Object(final String name, final long size) {
     public Ds3Object(final String name) {
         this.name = name;
         this.size = 0;
+        this.versionId = null;
     }
 
     public String getName() {
@@ -80,13 +136,20 @@ public int hashCode() {
 
     @Override
     public boolean equals(final Object obj) {
-        if (obj instanceof Ds3Object) {
-            final Ds3Object ds3Obj = (Ds3Object) obj;
-            if (ds3Obj.getName().equals(this.getName()) &&
-                    ds3Obj.getSize() == this.getSize()) {
-                return true;
-            }
+        if (!(obj instanceof Ds3Object)) {
+            return false;
         }
-        return false;
+        final Ds3Object ds3Obj = (Ds3Object) obj;
+        return ds3Obj.getName().equals(this.getName()) &&
+                ds3Obj.getSize() == this.getSize() &&
+                ds3Obj.getVersionId().equals(this.getVersionId());
+    }
+
+    public String getVersionId() {
+        return versionId;
+    }
+
+    public void setVersionId(String versionId) {
+        this.versionId = versionId;
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/PartialDs3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/PartialDs3Object.java
index 1ed151d08..ba6aa5666 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/PartialDs3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/PartialDs3Object.java
@@ -18,6 +18,8 @@
 import com.fasterxml.jackson.annotation.JsonUnwrapped;
 import com.spectralogic.ds3client.models.common.Range;
 
+import java.util.UUID;
+
 public class PartialDs3Object extends Ds3Object {
 
     @JsonUnwrapped
@@ -28,6 +30,16 @@ public PartialDs3Object(final String name, final Range range) {
         this.range = range;
     }
 
+    public PartialDs3Object(final String name, final Range range, final UUID versionId) {
+        super(name, versionId);
+        this.range = range;
+    }
+
+    public PartialDs3Object(final String name, final Range range, final String versionId) {
+        super(name, versionId);
+        this.range = range;
+    }
+
     public Range getRange() {
         return range;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/Views.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/Views.java
new file mode 100644
index 000000000..97f91dcfb
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/Views.java
@@ -0,0 +1,15 @@
+package com.spectralogic.ds3client.serializer;
+
+/**
+ * Used to annotate Ds3Object attributes to specify which attributes are
+ * unique to bulk get vs bulk put request payloads. Note that parameters
+ * with no annotation will be marshaled into both get and put payloads.
+ * These are used with the JsonView annotation.
+ */
+public class Views {
+    /** Denotes parameters within Ds3Object which are unique to get object request payloads */
+    public static class GetObject { }
+
+    /** Denotes parameters within Ds3Object which are unique to get object request payloads */
+    public static class PutObject { }
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java
index ab2f78f90..8e49998b8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java
@@ -17,21 +17,18 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
 import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
 import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule;
 import com.fasterxml.jackson.dataformat.xml.XmlMapper;
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
 import com.spectralogic.ds3client.models.bulk.Ds3ObjectList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Properties;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 
 public final class XmlOutput {
     private static final JacksonXmlModule module;
@@ -88,13 +85,13 @@ public static String toXml(final Object object) {
         return toXml(object, null);
     }
 
-    private static String toXml(final Object object, final FilterProvider filterProvider) {
+    private static String toXml(final Object object, final Class view) {
         try {
-            if (filterProvider == null) {
+            if (view == null) {
                 return mapper.writeValueAsString(object);
             }
             else {
-                return mapper.writer(filterProvider).writeValueAsString(object);
+                return mapper.writerWithView(view).writeValueAsString(object);
             }
         }
         catch(final JsonProcessingException e) {
@@ -104,11 +101,9 @@ private static String toXml(final Object object, final FilterProvider filterProv
 
     public static String toXml(final Ds3ObjectList objects, final boolean isBulkPut) {
         if (isBulkPut) {
-            return XmlOutput.toXml(objects);
+            return XmlOutput.toXml(objects, Views.PutObject.class);
         }
-        final FilterProvider filters = new SimpleFilterProvider().addFilter("sizeFilter",
-                SimpleBeanPropertyFilter.serializeAllExcept("Size"));
-        return XmlOutput.toXml(objects, filters);
+        return XmlOutput.toXml(objects, Views.GetObject.class);
     }
 
     public static T fromXml(final String xmlString, final Class type) throws IOException {
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
index 395570f7d..f6776882c 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
@@ -73,7 +73,7 @@ public void bucketListWithEmptyContents() throws IOException {
     }
 
     @Test
-    public void toXmlWithNoFilter() {
+    public void toXmlWithSize() {
         final String expectedString = "";
         final List objectList = ImmutableList.of(new Ds3Object("file1", 12), new Ds3Object("file2", 5022)).asList();
         final Ds3ObjectList ds3ObjectList = new Ds3ObjectList(objectList);
@@ -83,7 +83,7 @@ public void toXmlWithNoFilter() {
     }
 
     @Test
-    public void toXmlWithFilter() {
+    public void toXmlWithNoSize() {
         final String expectedString = "";
         final List objectList = ImmutableList.of(new Ds3Object("file1", 12), new Ds3Object("file2", 5022)).asList();
         final Ds3ObjectList ds3ObjectList = new Ds3ObjectList(objectList);
@@ -92,6 +92,26 @@ public void toXmlWithFilter() {
         assertThat(result, is(expectedString));
     }
 
+    @Test
+    public void toXmlWithNoVersionId() {
+        final String expectedString = "";
+        final List objectList = ImmutableList.of(new Ds3Object("file1", 12, "version1"), new Ds3Object("file2", 5022, "version2")).asList();
+        final Ds3ObjectList ds3ObjectList = new Ds3ObjectList(objectList);
+        final String result = XmlOutput.toXml(ds3ObjectList, true);
+
+        assertThat(result, is(expectedString));
+    }
+
+    @Test
+    public void toXmlWithVersionId() {
+        final String expectedString = "";
+        final List objectList = ImmutableList.of(new Ds3Object("file1", 12, "version1"), new Ds3Object("file2", 5022, "version2")).asList();
+        final Ds3ObjectList ds3ObjectList = new Ds3ObjectList(objectList);
+        final String result = XmlOutput.toXml(ds3ObjectList, false);
+
+        assertThat(result, is(expectedString));
+    }
+
     @Test
     public void toXmlWithPriority() {
         final String expectedString = "";

From 8f5df5fa31bfc91cd7ba83dc0af749f45a6a467f Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Mon, 19 Feb 2018 12:43:03 -0700
Subject: [PATCH 145/229] 5.0 API plus updating existing tests

---
 .../integration/RandomDataChannelBuilder.java | 67 ++++++++++++++
 .../ds3client/integration/Util.java           | 45 ++++++++-
 .../integration/PutJobManagement_Test.java    | 28 ++++++
 .../integration/Regression_Test.java          | 16 ++--
 .../SpectraS3PaginationLoader_Test.java       |  6 +-
 .../integration/VersionedObject_Test.java     | 92 +++++++++++++++++++
 .../test/helpers/TempStorageUtil.java         | 20 ++++
 .../com/spectralogic/ds3client/Ds3Client.java |  6 ++
 .../spectralogic/ds3client/Ds3ClientImpl.java |  4 +
 .../commands/DeleteObjectRequest.java         | 25 +++--
 .../commands/DeleteObjectsRequest.java        | 18 ----
 .../ds3client/commands/GetBucketRequest.java  | 18 ++++
 .../ds3client/commands/GetObjectRequest.java  | 21 +++++
 ...GetBlobsOnTapeSpectraS3ResponseParser.java |  4 +-
 ...tageObjectsJobSpectraS3ResponseParser.java | 40 ++++++++
 ...leteFolderRecursivelySpectraS3Request.java | 18 ----
 ...ectStorageDomainBlobsSpectraS3Request.java | 28 ++----
 .../EjectStorageDomainSpectraS3Request.java   | 23 ++---
 .../GetBlobsOnTapeSpectraS3Request.java       | 71 +++++++++++++-
 .../GetBlobsOnTapeSpectraS3Response.java      | 10 +-
 .../GetObjectsDetailsSpectraS3Request.java    | 38 +++++---
 ...bjectsWithFullDetailsSpectraS3Request.java | 38 +++++---
 ...alPlacementForObjectsSpectraS3Request.java | 20 ++--
 ...bjectsWithFullDetailsSpectraS3Request.java | 20 ++--
 .../spectrads3/GetPoolsSpectraS3Request.java  | 18 ++--
 ...bjectsWithFullDetailsSpectraS3Request.java | 20 ++--
 .../spectrads3/GetTapesSpectraS3Request.java  | 18 ++--
 .../ImportAllPoolsSpectraS3Request.java       |  9 +-
 .../ImportAllTapesSpectraS3Request.java       |  9 +-
 .../ImportAzureTargetSpectraS3Request.java    | 15 ---
 .../ImportPoolSpectraS3Request.java           |  9 +-
 .../ImportS3TargetSpectraS3Request.java       | 15 ---
 .../ImportTapeSpectraS3Request.java           |  9 +-
 ...ModifyDataPathBackendSpectraS3Request.java | 29 +++---
 .../ModifyDataPolicySpectraS3Request.java     | 14 +++
 ...fyStorageDomainMemberSpectraS3Request.java | 30 ++++++
 .../ModifyTapePartitionSpectraS3Request.java  | 14 +++
 .../ModifyUserSpectraS3Request.java           | 14 +++
 .../PutDataPolicySpectraS3Request.java        | 14 +++
 ...peStorageDomainMemberSpectraS3Request.java | 15 +++
 .../StageObjectsJobSpectraS3Request.java      | 83 +++++++++++++++++
 .../StageObjectsJobSpectraS3Response.java     | 28 ++++++
 ...alPlacementForObjectsSpectraS3Request.java | 20 ++--
 ...bjectsWithFullDetailsSpectraS3Request.java | 20 ++--
 .../ds3client/models/ActiveJob.java           | 12 +++
 .../ds3client/models/Contents.java            | 26 ++++++
 .../ds3client/models/DataPathBackend.java     | 24 ++---
 .../ds3client/models/DataPolicy.java          | 12 +++
 .../models/DetailedTapePartition.java         | 12 +++
 ...ictResolutionMode.java => JobRestore.java} |  8 +-
 .../ds3client/models/ListBucketResult.java    | 13 +++
 .../models/NamedDetailedTapePartition.java    | 12 +++
 .../spectralogic/ds3client/models/Pool.java   | 12 +--
 .../ds3client/models/RestOperationType.java   |  1 +
 .../ds3client/models/SpectraUser.java         | 12 +++
 .../ds3client/models/StorageDomainMember.java | 12 +++
 .../ds3client/models/SuspectBlobPool.java     | 12 +++
 .../ds3client/models/SuspectBlobTape.java     | 12 +++
 .../spectralogic/ds3client/models/Tape.java   | 12 +--
 .../ds3client/models/TapePartition.java       | 12 +++
 .../models/TapePartitionFailureType.java      |  3 +-
 .../ds3client/models/TapeState.java           |  1 +
 .../ds3client/models/VersioningLevel.java     |  3 +-
 .../ds3client/Ds3Client_Test.java             |  4 +-
 64 files changed, 981 insertions(+), 313 deletions(-)
 create mode 100644 ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/RandomDataChannelBuilder.java
 create mode 100644 ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
 rename ds3-sdk/src/main/java/com/spectralogic/ds3client/models/{ImportConflictResolutionMode.java => JobRestore.java} (89%)

diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/RandomDataChannelBuilder.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/RandomDataChannelBuilder.java
new file mode 100644
index 000000000..fa4222cb9
--- /dev/null
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/RandomDataChannelBuilder.java
@@ -0,0 +1,67 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2018 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.integration;
+
+import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
+import org.apache.commons.io.IOUtils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Creates a channel builder that generates random data. All object keys and sizes need to be specified.
+ */
+public class RandomDataChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
+
+    private static final int seed = 12345;
+    private final Map objectMap = new HashMap();
+
+    public RandomDataChannelBuilder() {
+    }
+
+    public RandomDataChannelBuilder(final List objects) {
+        for (final Ds3Object obj : objects) {
+            this.objectMap.put(obj.getName(), obj.getSize());
+        }
+    }
+
+    RandomDataChannelBuilder withObject(final String key, final Long size) {
+        this.objectMap.put(key, size);
+        return this;
+    }
+
+    @Override
+    public SeekableByteChannel buildChannel(final String key) throws IOException {
+        if (!this.objectMap.containsKey(key)) {
+            throw new IllegalArgumentException(String.format("Object with name '%s' was not defined for this channel builder.", key));
+        }
+
+        final Long size = this.objectMap.get(key);
+        final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(seed, size));
+        final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
+
+        final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(size.intValue());
+        channel.write(randomBuffer);
+
+        return channel;
+    }
+}
diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index 245e5ad99..0ed251ec1 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -15,15 +15,18 @@
 
 package com.spectralogic.ds3client.integration;
 
+import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.Ds3ClientBuilder;
-import com.spectralogic.ds3client.commands.DeleteBucketRequest;
-import com.spectralogic.ds3client.commands.DeleteObjectRequest;
+import com.spectralogic.ds3client.commands.spectrads3.CancelJobSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.GetJobsSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.GetJobsSpectraS3Response;
 import com.spectralogic.ds3client.commands.spectrads3.GetSystemInformationSpectraS3Request;
 import com.spectralogic.ds3client.helpers.DeleteBucket;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.channelbuilders.PrefixAdderObjectChannelBuilder;
-import com.spectralogic.ds3client.models.Contents;
+import com.spectralogic.ds3client.models.Job;
+import com.spectralogic.ds3client.models.JobStatus;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.utils.ResourceUtils;
 import org.slf4j.Logger;
@@ -31,8 +34,11 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -79,6 +85,24 @@ public static void loadBookTestData(final Ds3Client client, final String bucketN
         LOG.info("Finished loading test data...");
     }
 
+    /**
+     * Loads a single test book to a BP with the specified object name.
+     */
+    public static void loadTestBook(final Ds3Client client, final String fileName, final String objectName, final String bucketName) throws IOException, URISyntaxException {
+        final Path filePath = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + fileName);
+        final Ds3Object obj = new Ds3Object(objectName, Files.size(filePath));
+
+        final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
+
+        final Ds3ClientHelpers.Job job = helpers.startWriteJob(bucketName, ImmutableList.of(obj));
+        job.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
+            @Override
+            public SeekableByteChannel buildChannel(final String key) throws IOException {
+                return FileChannel.open(filePath, StandardOpenOption.READ);
+            }
+        });
+    }
+
     public static Ds3ClientHelpers.Job getLoadJob(final Ds3Client client, final String bucketName, final String resourceBaseName) throws IOException, URISyntaxException {
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
 
@@ -118,4 +142,19 @@ public static void deleteBucketContents(final Ds3Client client, final String buc
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
         DeleteBucket.INSTANCE.deleteBucketContents(helpers, bucketName);
     }
+
+    // Cancels all in-progress jobs associated with a bucket
+    public static void cancelAllJobsForBucket(final Ds3Client client, final String bucketName) {
+        try {
+            final GetJobsSpectraS3Response getJobs = client.getJobsSpectraS3(new GetJobsSpectraS3Request().withBucketId(bucketName));
+
+            for (final Job job : getJobs.getJobListResult().getJobs()) {
+                if (job.getStatus() == JobStatus.IN_PROGRESS) {
+                    client.cancelJobSpectraS3(new CancelJobSpectraS3Request(job.getJobId()));
+                }
+            }
+        } catch (final IOException e) {
+            LOG.debug("Could not cancel jobs for bucket '%s': %s", bucketName, e.getMessage());
+        }
+    }
 }
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
index cfdad62ab..e2d283cc7 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
@@ -75,6 +75,7 @@
 
 import static com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request.MIN_UPLOAD_SIZE_IN_BYTES;
 import static com.spectralogic.ds3client.integration.Util.RESOURCE_BASE_NAME;
+import static com.spectralogic.ds3client.integration.Util.cancelAllJobsForBucket;
 import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
 import static org.hamcrest.Matchers.*;
 import static org.junit.Assert.*;
@@ -192,6 +193,7 @@ public void getActiveJobs() throws IOException, URISyntaxException {
             }
             assertThat(activeJobsUUIDs, contains(jobID));
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -210,6 +212,7 @@ public void getJobs() throws IOException {
             assertThat(jobUUIDs, contains(jobID));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -230,6 +233,7 @@ public void modifyJobPriority() throws IOException {
             assertThat(response.getMasterObjectListResult().getPriority(), is(Priority.HIGH));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -250,6 +254,7 @@ public void modifyJobName() throws IOException {
             assertThat(response.getMasterObjectListResult().getName(), is("newName"));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -275,6 +280,7 @@ public void modifyJobCreationDate() throws IOException {
             assertThat(responseAfterModify.getMasterObjectListResult().getStartDate(), is(newDate));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -543,6 +549,7 @@ public void getJobChunksReady() throws IOException {
             assertThat(chunkNames, contains(ds3Object.getName()));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -568,6 +575,7 @@ public void getJobChunk() throws IOException {
             assertThat(getJobChunkSpectraS3Response.getObjectsResult(), is(notNullValue()));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -589,6 +597,7 @@ public void aggregateTwoJobs() throws IOException {
             assertThat(jobOneId, is(jobTwoId));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -612,6 +621,7 @@ public void testPutJobOptionsWithStreamingBehavior() throws IOException {
             assertThat(jobOneId, is(jobTwoId));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -635,6 +645,7 @@ public void testPutJobOptionsWithRandomAccessBehavior() throws IOException {
             assertThat(jobOneId, is(jobTwoId));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -652,6 +663,7 @@ public void allocateJobChunk() throws IOException {
             assertThat(allocateResponse.getStatus(), is(AllocateJobChunkSpectraS3Response.Status.ALLOCATED));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -811,6 +823,7 @@ public void initiateMultipartUpload() throws IOException {
             assertThat(e.getStatusCode(), is(400));
 
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1217,6 +1230,7 @@ public void onFailure(final FailureEvent failureEvent) {
             }
         } finally {
             FileUtils.deleteDirectory(tempDirectory.toFile());
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1290,6 +1304,7 @@ public void waiting(final int secondsToWait) {
             assertEquals(maxNumObjectTransferAttempts, numWaitingForChunksEventsFired.get());
         } finally {
             FileUtils.deleteDirectory(tempDirectory.toFile());
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1324,6 +1339,7 @@ public void onFailure(final FailureEvent failureEvent) {
             }
         } finally {
             FileUtils.deleteDirectory(tempDirectory.toFile());
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1366,6 +1382,7 @@ public void testPutJobUsingTransferStrategy() throws IOException, InterruptedExc
                 assertEquals(FILE_NAMES[0], bucketContents.getKey());
             }
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1411,6 +1428,7 @@ public void testWriteJobUsingTransferStrategy() throws IOException, InterruptedE
                 assertEquals(FILE_NAMES[0], bucketContents.getKey());
             }
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1473,6 +1491,7 @@ public void monitor() {
 
             assertEquals(1, numChunkAllocationAttempts.get());
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1575,6 +1594,7 @@ public void released() {
             assertEquals(1, numTimesChannelOpened.get());
             assertEquals(1, numTimesChannelClosed.get());
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1682,6 +1702,7 @@ public void monitor() {
 
             assertEquals(1, numTimesTransferCalled.get());
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1764,6 +1785,7 @@ public void reset() {
             assertEquals(0, numTimesInvokeCalled.get());
             assertEquals(1, numTimesResetCalled.get());
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1824,6 +1846,7 @@ public void testPutJobUsingStreamedTransferStrategy() throws IOException, URISyn
                 assertEquals(FILE_NAMES[0], bucketContents.getKey());
             }
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1857,6 +1880,7 @@ public void testPutJobUsingRandomAccessTransferStrategy() throws IOException, UR
                 assertEquals(FILE_NAMES[0], bucketContents.getKey());
             }
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -1973,6 +1997,7 @@ public void testPutting15000Files() throws IOException, URISyntaxException {
             fail("This test makes sure that we don't run out of connections when transferring lots of small files.  Oops");
         } finally {
             FileUtils.deleteDirectory(tempDirectory.toFile());
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
     }
@@ -2006,6 +2031,7 @@ public void testThatFifoIsNotProcessed() throws IOException, InterruptedExceptio
             caughtException.set(true);
         } finally {
             FileUtils.deleteDirectory(tempDirectory.toFile());
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
 
@@ -2058,6 +2084,7 @@ public void testThatNonExistentFileDoesNotStopPutJob() throws IOException {
         } finally {
             FileUtils.deleteDirectory(tempDirectoryForFilesToGet.toFile());
             FileUtils.deleteDirectory(tempDirectoryForFilesToPut.toFile());
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
         }
 
@@ -2124,6 +2151,7 @@ public void testStreamedPutJobWithBlobbedFile() throws Exception {
 
             assertTrue(FileUtils.contentEquals(new File(movedFileName), new File(originalFileName)));
         } finally {
+            cancelAllJobsForBucket(client, BUCKET_NAME);
             deleteAllContents(client, BUCKET_NAME);
 
             Files.deleteIfExists(Paths.get(originalFileName));
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Regression_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Regression_Test.java
index 43b8e1442..d0823a147 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Regression_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Regression_Test.java
@@ -89,6 +89,7 @@ public void testMarkerWithSpaces() throws IOException {
                     new Ds3Object("obj4 also has spaces.txt", 1024));
 
             final Ds3ClientHelpers.Job putJob = HELPERS.startWriteJob(bucketName, objects);
+            putJob.transfer(new RandomDataChannelBuilder(objects));
 
             final Iterable objs = HELPERS.listObjects(bucketName, null, "obj3 has spaces.txt");
             boolean foundObj4 = false;
@@ -97,7 +98,7 @@ public void testMarkerWithSpaces() throws IOException {
                 if (obj.getKey().equals("obj4 also has spaces.txt")) foundObj4 = true;
                 LOG.info("marker with spaces size: " + obj.getSize());
             }
-            assertTrue(Iterables.size(objs) == 1);
+            assertThat(Iterables.size(objs), is(1));
             assertTrue(foundObj4);
 
             final CancelJobSpectraS3Response cancelJobResponse = client
@@ -122,6 +123,7 @@ public void testPrefixWithSpaces() throws IOException {
                     new Ds3Object("has spaces obj4.txt", 1024));
 
             final Ds3ClientHelpers.Job putJob = HELPERS.startWriteJob(bucketName, objects);
+            putJob.transfer(new RandomDataChannelBuilder(objects));
 
             final Iterable objs = HELPERS.listObjects(bucketName, "has spaces");
             boolean foundObj2 = false;
@@ -132,7 +134,7 @@ public void testPrefixWithSpaces() throws IOException {
                 if (obj.getKey().equals("has spaces obj2.txt")) foundObj2 = true;
                 if (obj.getKey().equals("has spaces obj4.txt")) foundObj4 = true;
             }
-            assertTrue(Iterables.size(objs) == 2);
+            assertThat(Iterables.size(objs), is(2));
             assertTrue(foundObj2);
             assertTrue(foundObj4);
 
@@ -158,6 +160,7 @@ public void testPrefixForDirectoriesWithSpaces() throws IOException {
                     new Ds3Object("dir 2/has spaces obj4.txt", 1024));
 
             final Ds3ClientHelpers.Job putJob = HELPERS.startWriteJob(bucketName, objects);
+            putJob.transfer(new RandomDataChannelBuilder(objects));
 
             final Iterable dir1Objs = HELPERS.listObjects(bucketName, "dir1/");
             boolean foundObj1 = false;
@@ -168,7 +171,7 @@ public void testPrefixForDirectoriesWithSpaces() throws IOException {
                 if (obj.getKey().equals("dir1/obj1_no_spaces.txt")) foundObj1 = true;
                 if (obj.getKey().equals("dir1/has spaces obj2.txt")) foundObj2 = true;
             }
-            assertTrue(Iterables.size(dir1Objs) == 2);
+            assertThat(Iterables.size(dir1Objs), is(2));
             assertTrue(foundObj1);
             assertTrue(foundObj2);
 
@@ -181,7 +184,7 @@ public void testPrefixForDirectoriesWithSpaces() throws IOException {
                 if (obj.getKey().equals("dir 2/obj3_no_spaces.txt")) foundObj3 = true;
                 if (obj.getKey().equals("dir 2/has spaces obj4.txt")) foundObj4 = true;
             }
-            assertTrue(Iterables.size(objsDir2) == 2);
+            assertThat(Iterables.size(objsDir2), is(2));
             assertTrue(foundObj3);
             assertTrue(foundObj4);
 
@@ -206,6 +209,7 @@ public void testPrefixForNestedDirectories() throws IOException {
                     new Ds3Object("dir1/dir 2/has spaces obj4.txt", 1024));
 
             final Ds3ClientHelpers.Job putJob = HELPERS.startWriteJob(bucketName, objects);
+            putJob.transfer(new RandomDataChannelBuilder(objects));
 
             final Iterable dir1Objs = HELPERS.listObjects(bucketName, "dir1/");
             boolean foundObj1 = false;
@@ -216,7 +220,7 @@ public void testPrefixForNestedDirectories() throws IOException {
                 if (obj.getKey().equals("dir1/obj1_no_spaces.txt")) foundObj1 = true;
                 if (obj.getKey().equals("dir1/has spaces obj2.txt")) foundObj2 = true;
             }
-            assertTrue(Iterables.size(dir1Objs) == 4);
+            assertThat(Iterables.size(dir1Objs), is(4));
             assertTrue(foundObj1);
             assertTrue(foundObj2);
 
@@ -229,7 +233,7 @@ public void testPrefixForNestedDirectories() throws IOException {
                 if (obj.getKey().equals("dir1/dir 2/obj3_no_spaces.txt")) foundObj3 = true;
                 if (obj.getKey().equals("dir1/dir 2/has spaces obj4.txt")) foundObj4 = true;
             }
-            assertTrue(Iterables.size(objsDir2) == 2);
+            assertThat(Iterables.size(objsDir2), is(2));
             assertTrue(foundObj3);
             assertTrue(foundObj4);
 
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java
index c3e595c51..6fd9a9cfa 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/SpectraS3PaginationLoader_Test.java
@@ -17,7 +17,6 @@
 
 import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
-import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.pagination.GetObjectsFullDetailsLoaderFactory;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
@@ -76,7 +75,10 @@ public void basicPagination() throws IOException {
             // test setup
             HELPERS.ensureBucketExists(TEST_ENV_NAME);
             final int numObjects = 55;
-            CLIENT.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(TEST_ENV_NAME, createTestList(numObjects)));
+
+            final List objects = createTestList(numObjects);
+            final Ds3ClientHelpers.Job putJob = HELPERS.startWriteJob(TEST_ENV_NAME, objects);
+            putJob.transfer(new RandomDataChannelBuilder(objects));
 
             final GetObjectsFullDetailsLoaderFactory loaderFactory = new GetObjectsFullDetailsLoaderFactory(CLIENT, TEST_ENV_NAME, 10, RETRIES, false);
 
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
new file mode 100644
index 000000000..075c5127a
--- /dev/null
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
@@ -0,0 +1,92 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2018 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.integration;
+
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
+import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.models.VersioningLevel;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.UUID;
+
+import static com.spectralogic.ds3client.integration.Util.*;
+
+public class VersionedObject_Test {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SpectraS3PaginationLoader_Test.class);
+
+    private static final Ds3Client CLIENT = Util.fromEnv();
+    private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(CLIENT);
+    private static final String TEST_ENV_NAME = "java_versioned_object_test";
+    private static final int RETRIES = 5;
+
+    private static TempStorageIds envStorageIds;
+    private static UUID envDataPolicyId;
+
+    @BeforeClass
+    public static void startup() throws IOException {
+        LOG.info("Starting test Setup...");
+
+        // Create data policy with versioning
+        envDataPolicyId = TempStorageUtil.setupDataPolicy(
+                TEST_ENV_NAME,
+                false,
+                ChecksumType.Type.MD5,
+                VersioningLevel.KEEP_MULTIPLE_VERSIONS,
+                CLIENT);
+
+        envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, envDataPolicyId, CLIENT);
+        LOG.info("Finished test Setup...");
+    }
+
+    @AfterClass
+    public static void teardown() throws IOException {
+        LOG.info("Starting test teardown...");
+        TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, CLIENT);
+        CLIENT.close();
+        LOG.info("Finished test teardown...");
+    }
+
+    @Test
+    public void getObjectsWithVersioning() throws IOException, URISyntaxException {
+        try {
+            HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
+            final String objectName = "object_with_versions";
+
+            // Put different content for object twice
+            loadTestBook(CLIENT, BOOKS[0], objectName, TEST_ENV_NAME); // putting beowulf as content
+            loadTestBook(CLIENT, BOOKS[1], objectName, TEST_ENV_NAME); // putting sherlock holmes as content
+
+            // TODO Get the version of the objects
+
+            // TODO Create bulk get job with both versions of object specified
+
+
+        } finally {
+            cancelAllJobsForBucket(CLIENT, TEST_ENV_NAME);
+            deleteAllContents(CLIENT, TEST_ENV_NAME);
+        }
+    }
+}
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
index 0f7efe21d..c36e17840 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java
@@ -19,6 +19,7 @@
 import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.models.PoolType;
+import com.spectralogic.ds3client.models.VersioningLevel;
 
 import java.io.IOException;
 import java.util.UUID;
@@ -108,4 +109,23 @@ public static UUID setupDataPolicy(
                 .withDefaultDataPolicyId(dataPolicyResponse.getDataPolicyResult().getId()));
         return dataPolicyResponse.getDataPolicyResult().getId();
     }
+
+    /**
+     * Creates a Data Policy with the specified checksum type, end-to-end crc, and versioning
+     */
+    public static UUID setupDataPolicy(
+            final String testSetName,
+            final boolean withEndToEndCrcRequired,
+            final ChecksumType.Type checksumType,
+            final VersioningLevel versioningLevel,
+            final Ds3Client client) throws IOException {
+        final PutDataPolicySpectraS3Response dataPolicyResponse = client.putDataPolicySpectraS3(
+                new PutDataPolicySpectraS3Request(testSetName + DATA_POLICY_NAME)
+                        .withEndToEndCrcRequired(withEndToEndCrcRequired)
+                        .withChecksumType(checksumType).withAlwaysForcePutJobCreation(true)
+                        .withVersioning(versioningLevel));
+        client.modifyUserSpectraS3(new ModifyUserSpectraS3Request(DEFAULT_USER)
+                .withDefaultDataPolicyId(dataPolicyResponse.getDataPolicyResult().getId()));
+        return dataPolicyResponse.getDataPolicyResult().getId();
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index 559dfc55e..81aeba046 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -895,6 +895,12 @@ ModifyJobSpectraS3Response modifyJobSpectraS3(final ModifyJobSpectraS3Request re
     ReplicatePutJobSpectraS3Response replicatePutJobSpectraS3(final ReplicatePutJobSpectraS3Request request)
             throws IOException;
 
+    @Action("MODIFY")
+    @Resource("BUCKET")
+    
+    StageObjectsJobSpectraS3Response stageObjectsJobSpectraS3(final StageObjectsJobSpectraS3Request request)
+            throws IOException;
+
     @Action("DELETE")
     @Resource("ACTIVE_JOB")
     
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
index a04ac9ca7..425e4e8d0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
@@ -570,6 +570,10 @@ public ReplicatePutJobSpectraS3Response replicatePutJobSpectraS3(final Replicate
         return new ReplicatePutJobSpectraS3ResponseParser().response(this.netClient.getResponse(request));
     }
     @Override
+    public StageObjectsJobSpectraS3Response stageObjectsJobSpectraS3(final StageObjectsJobSpectraS3Request request) throws IOException {
+        return new StageObjectsJobSpectraS3ResponseParser().response(this.netClient.getResponse(request));
+    }
+    @Override
     public TruncateActiveJobSpectraS3Response truncateActiveJobSpectraS3(final TruncateActiveJobSpectraS3Request request) throws IOException {
         return new TruncateActiveJobSpectraS3ResponseParser().response(this.netClient.getResponse(request));
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/DeleteObjectRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/DeleteObjectRequest.java
index bb2da1b05..ad11d6c7b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/DeleteObjectRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/DeleteObjectRequest.java
@@ -18,6 +18,8 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import java.util.UUID;
+import com.google.common.net.UrlEscapers;
 
 public class DeleteObjectRequest extends AbstractRequest {
 
@@ -27,7 +29,7 @@ public class DeleteObjectRequest extends AbstractRequest {
 
     private final String objectName;
 
-    private boolean rollBack;
+    private String versionId;
 
     // Constructor
     
@@ -38,13 +40,16 @@ public DeleteObjectRequest(final String bucketName, final String objectName) {
         
     }
 
-    public DeleteObjectRequest withRollBack(final boolean rollBack) {
-        this.rollBack = rollBack;
-        if (this.rollBack) {
-            this.getQueryParams().put("roll_back", null);
-        } else {
-            this.getQueryParams().remove("roll_back");
-        }
+    public DeleteObjectRequest withVersionId(final UUID versionId) {
+        this.versionId = versionId.toString();
+        this.updateQueryParam("version_id", versionId);
+        return this;
+    }
+
+
+    public DeleteObjectRequest withVersionId(final String versionId) {
+        this.versionId = versionId;
+        this.updateQueryParam("version_id", versionId);
         return this;
     }
 
@@ -70,8 +75,8 @@ public String getObjectName() {
     }
 
 
-    public boolean getRollBack() {
-        return this.rollBack;
+    public String getVersionId() {
+        return this.versionId;
     }
 
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/DeleteObjectsRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/DeleteObjectsRequest.java
index 7b370467c..60729e397 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/DeleteObjectsRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/DeleteObjectsRequest.java
@@ -35,8 +35,6 @@ public class DeleteObjectsRequest extends AbstractRequest {
     private final List objects;
     
     private final String bucketName;
-
-    private boolean rollBack;
     private boolean quiet = false;
     private long size;
 
@@ -66,17 +64,6 @@ private static List contentsToString(final Iterable objs) {
         return objKeyList;
     }
 
-    public DeleteObjectsRequest withRollBack(final boolean rollBack) {
-        this.rollBack = rollBack;
-        if (this.rollBack) {
-            this.getQueryParams().put("roll_back", null);
-        } else {
-            this.getQueryParams().remove("roll_back");
-        }
-        return this;
-    }
-
-
 
     public DeleteObjectsRequest withQuiet(final boolean quiet) {
         this.quiet = quiet;
@@ -128,11 +115,6 @@ public String getBucketName() {
     }
 
 
-    public boolean getRollBack() {
-        return this.rollBack;
-    }
-
-
     @Override
     public long getSize() {
         return this.size;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetBucketRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetBucketRequest.java
index d2c38fa11..a1d2959f9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetBucketRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetBucketRequest.java
@@ -34,6 +34,8 @@ public class GetBucketRequest extends AbstractRequest {
 
     private String prefix;
 
+    private boolean versions;
+
     // Constructor
     
     
@@ -70,6 +72,17 @@ public GetBucketRequest withPrefix(final String prefix) {
     }
 
 
+    public GetBucketRequest withVersions(final boolean versions) {
+        this.versions = versions;
+        if (this.versions) {
+            this.getQueryParams().put("versions", null);
+        } else {
+            this.getQueryParams().remove("versions");
+        }
+        return this;
+    }
+
+
 
     @Override
     public HttpVerb getVerb() {
@@ -105,4 +118,9 @@ public String getPrefix() {
         return this.prefix;
     }
 
+
+    public boolean getVersions() {
+        return this.versions;
+    }
+
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
index 68c269fcc..7e6facf55 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectRequest.java
@@ -46,6 +46,8 @@ public class GetObjectRequest extends AbstractRequest {
     private String job;
 
     private long offset;
+
+    private String versionId;
     private ImmutableCollection byteRanges = null;
     private ChecksumType checksum = ChecksumType.none();
     private ChecksumType.Type checksumType = ChecksumType.Type.NONE;
@@ -149,6 +151,20 @@ public GetObjectRequest withOffset(final long offset) {
     }
 
 
+    public GetObjectRequest withVersionId(final UUID versionId) {
+        this.versionId = versionId.toString();
+        this.updateQueryParam("version_id", versionId);
+        return this;
+    }
+
+
+    public GetObjectRequest withVersionId(final String versionId) {
+        this.versionId = versionId;
+        this.updateQueryParam("version_id", versionId);
+        return this;
+    }
+
+
 
     /**
      * Set a MD5 checksum for the request.
@@ -250,6 +266,11 @@ public long getOffset() {
     }
 
 
+    public String getVersionId() {
+        return this.versionId;
+    }
+
+
     public Collection getByteRanges() {
         return this.byteRanges;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/GetBlobsOnTapeSpectraS3ResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/GetBlobsOnTapeSpectraS3ResponseParser.java
index 150e6711b..c30308283 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/GetBlobsOnTapeSpectraS3ResponseParser.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/GetBlobsOnTapeSpectraS3ResponseParser.java
@@ -31,12 +31,14 @@ public class GetBlobsOnTapeSpectraS3ResponseParser extends AbstractResponseParse
     @Override
     public GetBlobsOnTapeSpectraS3Response parseXmlResponse(final WebResponse response) throws IOException {
         final int statusCode = response.getStatusCode();
+        final Integer pagingTruncated = parseIntHeader("page-truncated");
+        final Integer pagingTotalResultCount = parseIntHeader("total-result-count");
         if (ResponseParserUtils.validateStatusCode(statusCode, expectedStatusCodes)) {
             switch (statusCode) {
             case 200:
                 try (final InputStream inputStream = response.getResponseStream()) {
                     final BulkObjectList result = XmlOutput.fromXml(inputStream, BulkObjectList.class);
-                    return new GetBlobsOnTapeSpectraS3Response(result, this.getChecksum(), this.getChecksumType());
+                    return new GetBlobsOnTapeSpectraS3Response(result, pagingTotalResultCount, pagingTruncated, this.getChecksum(), this.getChecksumType());
                 }
 
             default:
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
new file mode 100644
index 000000000..afe980dba
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
@@ -0,0 +1,40 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.parsers;
+
+import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
+import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
+import com.spectralogic.ds3client.commands.spectrads3.StageObjectsJobSpectraS3Response;
+import com.spectralogic.ds3client.networking.WebResponse;
+import java.io.IOException;
+
+public class StageObjectsJobSpectraS3ResponseParser extends AbstractResponseParser {
+    private final int[] expectedStatusCodes = new int[]{};
+
+    @Override
+    public StageObjectsJobSpectraS3Response parseXmlResponse(final WebResponse response) throws IOException {
+        final int statusCode = response.getStatusCode();
+        if (ResponseParserUtils.validateStatusCode(statusCode, expectedStatusCodes)) {
+            switch (statusCode) {
+            default:
+                assert false: "validateStatusCode should have made it impossible to reach this line";
+            }
+        }
+
+        throw ResponseParserUtils.createFailedRequest(response, expectedStatusCodes);
+    }
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteFolderRecursivelySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteFolderRecursivelySpectraS3Request.java
index d511ad6ca..46e0a48b0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteFolderRecursivelySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/DeleteFolderRecursivelySpectraS3Request.java
@@ -28,8 +28,6 @@ public class DeleteFolderRecursivelySpectraS3Request extends AbstractRequest {
 
     private final String bucketId;
 
-    private boolean rollBack;
-
     // Constructor
     
     
@@ -42,17 +40,6 @@ public DeleteFolderRecursivelySpectraS3Request(final String bucketId, final Stri
         this.getQueryParams().put("recursive", null);
     }
 
-    public DeleteFolderRecursivelySpectraS3Request withRollBack(final boolean rollBack) {
-        this.rollBack = rollBack;
-        if (this.rollBack) {
-            this.getQueryParams().put("roll_back", null);
-        } else {
-            this.getQueryParams().remove("roll_back");
-        }
-        return this;
-    }
-
-
 
     @Override
     public HttpVerb getVerb() {
@@ -73,9 +60,4 @@ public String getBucketId() {
         return this.bucketId;
     }
 
-
-    public boolean getRollBack() {
-        return this.rollBack;
-    }
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
index 5e3909fcc..dfe2b4eb1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
@@ -36,7 +36,7 @@ public class EjectStorageDomainBlobsSpectraS3Request extends AbstractRequest {
     
     private final String bucketId;
 
-    private final String storageDomainId;
+    private final String storageDomain;
 
     private final List objects;
 
@@ -48,9 +48,9 @@ public class EjectStorageDomainBlobsSpectraS3Request extends AbstractRequest {
     // Constructor
     
     
-    public EjectStorageDomainBlobsSpectraS3Request(final String bucketId, final List objects, final UUID storageDomainId) {
+    public EjectStorageDomainBlobsSpectraS3Request(final String bucketId, final List objects, final String storageDomain) {
         this.bucketId = bucketId;
-        this.storageDomainId = storageDomainId.toString();
+        this.storageDomain = storageDomain;
         this.objects = objects;
         
         this.getQueryParams().put("operation", "eject");
@@ -58,22 +58,7 @@ public EjectStorageDomainBlobsSpectraS3Request(final String bucketId, final List
         this.getQueryParams().put("blobs", null);
         this.updateQueryParam("bucket_id", bucketId);
 
-        this.updateQueryParam("storage_domain_id", storageDomainId);
-
-    }
-
-    
-    public EjectStorageDomainBlobsSpectraS3Request(final String bucketId, final List objects, final String storageDomainId) {
-        this.bucketId = bucketId;
-        this.storageDomainId = storageDomainId;
-        this.objects = objects;
-        
-        this.getQueryParams().put("operation", "eject");
-
-        this.getQueryParams().put("blobs", null);
-        this.updateQueryParam("bucket_id", bucketId);
-
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+        this.updateQueryParam("storage_domain", storageDomain);
 
     }
 
@@ -127,8 +112,8 @@ public String getBucketId() {
     }
 
 
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomain() {
+        return this.storageDomain;
     }
 
 
@@ -146,5 +131,4 @@ public String getEjectLocation() {
         return this.ejectLocation;
     }
 
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainSpectraS3Request.java
index 5acdc03ab..da025cc71 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainSpectraS3Request.java
@@ -18,14 +18,13 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
 public class EjectStorageDomainSpectraS3Request extends AbstractRequest {
 
     // Variables
     
-    private final String storageDomainId;
+    private final String storageDomain;
 
     private String bucketId;
 
@@ -36,22 +35,12 @@ public class EjectStorageDomainSpectraS3Request extends AbstractRequest {
     // Constructor
     
     
-    public EjectStorageDomainSpectraS3Request(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId.toString();
+    public EjectStorageDomainSpectraS3Request(final String storageDomain) {
+        this.storageDomain = storageDomain;
         
         this.getQueryParams().put("operation", "eject");
 
-        this.updateQueryParam("storage_domain_id", storageDomainId);
-
-    }
-
-    
-    public EjectStorageDomainSpectraS3Request(final String storageDomainId) {
-        this.storageDomainId = storageDomainId;
-        
-        this.getQueryParams().put("operation", "eject");
-
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+        this.updateQueryParam("storage_domain", storageDomain);
 
     }
 
@@ -87,8 +76,8 @@ public String getPath() {
         return "/_rest_/tape";
     }
     
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomain() {
+        return this.storageDomain;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Request.java
index 453689c31..a4574f1f7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Request.java
@@ -17,16 +17,24 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractPaginationRequest;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
-public class GetBlobsOnTapeSpectraS3Request extends AbstractRequest {
+public class GetBlobsOnTapeSpectraS3Request extends AbstractPaginationRequest {
 
     // Variables
     
     private final String tapeId;
 
+    private boolean lastPage;
+
+    private int pageLength;
+
+    private int pageOffset;
+
+    private String pageStartMarker;
+
     // Constructor
     
     
@@ -45,6 +53,45 @@ public GetBlobsOnTapeSpectraS3Request(final String tapeId) {
 
     }
 
+    public GetBlobsOnTapeSpectraS3Request withLastPage(final boolean lastPage) {
+        this.lastPage = lastPage;
+        if (this.lastPage) {
+            this.getQueryParams().put("last_page", null);
+        } else {
+            this.getQueryParams().remove("last_page");
+        }
+        return this;
+    }
+
+
+    public GetBlobsOnTapeSpectraS3Request withPageLength(final int pageLength) {
+        this.pageLength = pageLength;
+        this.updateQueryParam("page_length", pageLength);
+        return this;
+    }
+
+
+    public GetBlobsOnTapeSpectraS3Request withPageOffset(final int pageOffset) {
+        this.pageOffset = pageOffset;
+        this.updateQueryParam("page_offset", pageOffset);
+        return this;
+    }
+
+
+    public GetBlobsOnTapeSpectraS3Request withPageStartMarker(final UUID pageStartMarker) {
+        this.pageStartMarker = pageStartMarker.toString();
+        this.updateQueryParam("page_start_marker", pageStartMarker);
+        return this;
+    }
+
+
+    public GetBlobsOnTapeSpectraS3Request withPageStartMarker(final String pageStartMarker) {
+        this.pageStartMarker = pageStartMarker;
+        this.updateQueryParam("page_start_marker", pageStartMarker);
+        return this;
+    }
+
+
 
     @Override
     public HttpVerb getVerb() {
@@ -60,4 +107,24 @@ public String getTapeId() {
         return this.tapeId;
     }
 
+
+    public boolean getLastPage() {
+        return this.lastPage;
+    }
+
+
+    public int getPageLength() {
+        return this.pageLength;
+    }
+
+
+    public int getPageOffset() {
+        return this.pageOffset;
+    }
+
+
+    public String getPageStartMarker() {
+        return this.pageStartMarker;
+    }
+
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Response.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Response.java
index e20f6dcf3..7eeb7a721 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Response.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetBlobsOnTapeSpectraS3Response.java
@@ -18,14 +18,14 @@
 
 import com.spectralogic.ds3client.models.BulkObjectList;
 import com.spectralogic.ds3client.models.ChecksumType;
-import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
+import com.spectralogic.ds3client.commands.interfaces.AbstractPaginationResponse;
+
+public class GetBlobsOnTapeSpectraS3Response extends AbstractPaginationResponse {
 
-public class GetBlobsOnTapeSpectraS3Response extends AbstractResponse {
-    
     private final BulkObjectList bulkObjectListResult;
 
-    public GetBlobsOnTapeSpectraS3Response(final BulkObjectList bulkObjectListResult, final String checksum, final ChecksumType.Type checksumType) {
-        super(checksum, checksumType);
+    public GetBlobsOnTapeSpectraS3Response(final BulkObjectList bulkObjectListResult, final Integer pagingTotalResultCount, final Integer pagingTruncated, final String checksum, final ChecksumType.Type checksumType) {
+        super(pagingTotalResultCount, pagingTruncated, checksum, checksumType);
         this.bulkObjectListResult = bulkObjectListResult;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsDetailsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsDetailsSpectraS3Request.java
index d8fd50532..ba524565c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsDetailsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsDetailsSpectraS3Request.java
@@ -28,6 +28,8 @@ public class GetObjectsDetailsSpectraS3Request extends AbstractPaginationRequest
     
     private String bucketId;
 
+    private long endDate;
+
     private boolean lastPage;
 
     private boolean latest;
@@ -40,9 +42,9 @@ public class GetObjectsDetailsSpectraS3Request extends AbstractPaginationRequest
 
     private String pageStartMarker;
 
-    private S3ObjectType type;
+    private long startDate;
 
-    private long version;
+    private S3ObjectType type;
 
     // Constructor
     
@@ -58,6 +60,13 @@ public GetObjectsDetailsSpectraS3Request withBucketId(final String bucketId) {
     }
 
 
+    public GetObjectsDetailsSpectraS3Request withEndDate(final long endDate) {
+        this.endDate = endDate;
+        this.updateQueryParam("end_date", endDate);
+        return this;
+    }
+
+
     public GetObjectsDetailsSpectraS3Request withLastPage(final boolean lastPage) {
         this.lastPage = lastPage;
         if (this.lastPage) {
@@ -111,16 +120,16 @@ public GetObjectsDetailsSpectraS3Request withPageStartMarker(final String pageSt
     }
 
 
-    public GetObjectsDetailsSpectraS3Request withType(final S3ObjectType type) {
-        this.type = type;
-        this.updateQueryParam("type", type);
+    public GetObjectsDetailsSpectraS3Request withStartDate(final long startDate) {
+        this.startDate = startDate;
+        this.updateQueryParam("start_date", startDate);
         return this;
     }
 
 
-    public GetObjectsDetailsSpectraS3Request withVersion(final long version) {
-        this.version = version;
-        this.updateQueryParam("version", version);
+    public GetObjectsDetailsSpectraS3Request withType(final S3ObjectType type) {
+        this.type = type;
+        this.updateQueryParam("type", type);
         return this;
     }
 
@@ -141,6 +150,11 @@ public String getBucketId() {
     }
 
 
+    public long getEndDate() {
+        return this.endDate;
+    }
+
+
     public boolean getLastPage() {
         return this.lastPage;
     }
@@ -171,13 +185,13 @@ public String getPageStartMarker() {
     }
 
 
-    public S3ObjectType getType() {
-        return this.type;
+    public long getStartDate() {
+        return this.startDate;
     }
 
 
-    public long getVersion() {
-        return this.version;
+    public S3ObjectType getType() {
+        return this.type;
     }
 
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsWithFullDetailsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsWithFullDetailsSpectraS3Request.java
index 5f021cdad..6b6ba4fd1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsWithFullDetailsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetObjectsWithFullDetailsSpectraS3Request.java
@@ -28,6 +28,8 @@ public class GetObjectsWithFullDetailsSpectraS3Request extends AbstractPaginatio
     
     private String bucketId;
 
+    private long endDate;
+
     private boolean includePhysicalPlacement;
 
     private boolean lastPage;
@@ -42,9 +44,9 @@ public class GetObjectsWithFullDetailsSpectraS3Request extends AbstractPaginatio
 
     private String pageStartMarker;
 
-    private S3ObjectType type;
+    private long startDate;
 
-    private long version;
+    private S3ObjectType type;
 
     // Constructor
     
@@ -61,6 +63,13 @@ public GetObjectsWithFullDetailsSpectraS3Request withBucketId(final String bucke
     }
 
 
+    public GetObjectsWithFullDetailsSpectraS3Request withEndDate(final long endDate) {
+        this.endDate = endDate;
+        this.updateQueryParam("end_date", endDate);
+        return this;
+    }
+
+
     public GetObjectsWithFullDetailsSpectraS3Request withIncludePhysicalPlacement(final boolean includePhysicalPlacement) {
         this.includePhysicalPlacement = includePhysicalPlacement;
         if (this.includePhysicalPlacement) {
@@ -125,16 +134,16 @@ public GetObjectsWithFullDetailsSpectraS3Request withPageStartMarker(final Strin
     }
 
 
-    public GetObjectsWithFullDetailsSpectraS3Request withType(final S3ObjectType type) {
-        this.type = type;
-        this.updateQueryParam("type", type);
+    public GetObjectsWithFullDetailsSpectraS3Request withStartDate(final long startDate) {
+        this.startDate = startDate;
+        this.updateQueryParam("start_date", startDate);
         return this;
     }
 
 
-    public GetObjectsWithFullDetailsSpectraS3Request withVersion(final long version) {
-        this.version = version;
-        this.updateQueryParam("version", version);
+    public GetObjectsWithFullDetailsSpectraS3Request withType(final S3ObjectType type) {
+        this.type = type;
+        this.updateQueryParam("type", type);
         return this;
     }
 
@@ -155,6 +164,11 @@ public String getBucketId() {
     }
 
 
+    public long getEndDate() {
+        return this.endDate;
+    }
+
+
     public boolean getIncludePhysicalPlacement() {
         return this.includePhysicalPlacement;
     }
@@ -190,13 +204,13 @@ public String getPageStartMarker() {
     }
 
 
-    public S3ObjectType getType() {
-        return this.type;
+    public long getStartDate() {
+        return this.startDate;
     }
 
 
-    public long getVersion() {
-        return this.version;
+    public S3ObjectType getType() {
+        return this.type;
     }
 
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPhysicalPlacementForObjectsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPhysicalPlacementForObjectsSpectraS3Request.java
index 0e0ae220c..5d229218f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPhysicalPlacementForObjectsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPhysicalPlacementForObjectsSpectraS3Request.java
@@ -27,7 +27,6 @@
 import java.util.List;
 import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
 public class GetPhysicalPlacementForObjectsSpectraS3Request extends AbstractRequest {
@@ -38,7 +37,7 @@ public class GetPhysicalPlacementForObjectsSpectraS3Request extends AbstractRequ
 
     private final List objects;
 
-    private String storageDomainId;
+    private String storageDomain;
     private long size = 0;
 
     // Constructor
@@ -52,16 +51,9 @@ public GetPhysicalPlacementForObjectsSpectraS3Request(final String bucketName, f
 
     }
 
-    public GetPhysicalPlacementForObjectsSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId.toString();
-        this.updateQueryParam("storage_domain_id", storageDomainId);
-        return this;
-    }
-
-
-    public GetPhysicalPlacementForObjectsSpectraS3Request withStorageDomainId(final String storageDomainId) {
-        this.storageDomainId = storageDomainId;
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public GetPhysicalPlacementForObjectsSpectraS3Request withStorageDomain(final String storageDomain) {
+        this.storageDomain = storageDomain;
+        this.updateQueryParam("storage_domain", storageDomain);
         return this;
     }
 
@@ -107,8 +99,8 @@ public List getObjects() {
     }
 
 
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomain() {
+        return this.storageDomain;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request.java
index 891dd21f5..823687e4b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request.java
@@ -27,7 +27,6 @@
 import java.util.List;
 import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
 public class GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request extends AbstractRequest {
@@ -38,7 +37,7 @@ public class GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request exten
 
     private final List objects;
 
-    private String storageDomainId;
+    private String storageDomain;
     private long size = 0;
 
     // Constructor
@@ -53,16 +52,9 @@ public GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request(final Strin
         this.getQueryParams().put("full_details", null);
     }
 
-    public GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId.toString();
-        this.updateQueryParam("storage_domain_id", storageDomainId);
-        return this;
-    }
-
-
-    public GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request withStorageDomainId(final String storageDomainId) {
-        this.storageDomainId = storageDomainId;
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request withStorageDomain(final String storageDomain) {
+        this.storageDomain = storageDomain;
+        this.updateQueryParam("storage_domain", storageDomain);
         return this;
     }
 
@@ -108,8 +100,8 @@ public List getObjects() {
     }
 
 
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomain() {
+        return this.storageDomain;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPoolsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPoolsSpectraS3Request.java
index b783870c7..be1317a65 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPoolsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetPoolsSpectraS3Request.java
@@ -55,7 +55,7 @@ public class GetPoolsSpectraS3Request extends AbstractPaginationRequest {
 
     private PoolState state;
 
-    private String storageDomainId;
+    private String storageDomainMemberId;
 
     private PoolType type;
 
@@ -175,16 +175,16 @@ public GetPoolsSpectraS3Request withState(final PoolState state) {
     }
 
 
-    public GetPoolsSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId.toString();
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public GetPoolsSpectraS3Request withStorageDomainMemberId(final UUID storageDomainMemberId) {
+        this.storageDomainMemberId = storageDomainMemberId.toString();
+        this.updateQueryParam("storage_domain_member_id", storageDomainMemberId);
         return this;
     }
 
 
-    public GetPoolsSpectraS3Request withStorageDomainId(final String storageDomainId) {
-        this.storageDomainId = storageDomainId;
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public GetPoolsSpectraS3Request withStorageDomainMemberId(final String storageDomainMemberId) {
+        this.storageDomainMemberId = storageDomainMemberId;
+        this.updateQueryParam("storage_domain_member_id", storageDomainMemberId);
         return this;
     }
 
@@ -272,8 +272,8 @@ public PoolState getState() {
     }
 
 
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomainMemberId() {
+        return this.storageDomainMemberId;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetSuspectObjectsWithFullDetailsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetSuspectObjectsWithFullDetailsSpectraS3Request.java
index 3622a162a..6cda546d6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetSuspectObjectsWithFullDetailsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetSuspectObjectsWithFullDetailsSpectraS3Request.java
@@ -19,7 +19,6 @@
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
-import java.util.UUID;
 
 public class GetSuspectObjectsWithFullDetailsSpectraS3Request extends AbstractRequest {
 
@@ -27,7 +26,7 @@ public class GetSuspectObjectsWithFullDetailsSpectraS3Request extends AbstractRe
     
     private String bucketId;
 
-    private String storageDomainId;
+    private String storageDomain;
 
     // Constructor
     
@@ -44,16 +43,9 @@ public GetSuspectObjectsWithFullDetailsSpectraS3Request withBucketId(final Strin
     }
 
 
-    public GetSuspectObjectsWithFullDetailsSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId.toString();
-        this.updateQueryParam("storage_domain_id", storageDomainId);
-        return this;
-    }
-
-
-    public GetSuspectObjectsWithFullDetailsSpectraS3Request withStorageDomainId(final String storageDomainId) {
-        this.storageDomainId = storageDomainId;
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public GetSuspectObjectsWithFullDetailsSpectraS3Request withStorageDomain(final String storageDomain) {
+        this.storageDomain = storageDomain;
+        this.updateQueryParam("storage_domain", storageDomain);
         return this;
     }
 
@@ -74,8 +66,8 @@ public String getBucketId() {
     }
 
 
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomain() {
+        return this.storageDomain;
     }
 
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
index 5a9330fc9..1d6e8b682 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/GetTapesSpectraS3Request.java
@@ -65,7 +65,7 @@ public class GetTapesSpectraS3Request extends AbstractPaginationRequest {
 
     private TapeState state;
 
-    private String storageDomainId;
+    private String storageDomainMemberId;
 
     private String type;
 
@@ -224,16 +224,16 @@ public GetTapesSpectraS3Request withState(final TapeState state) {
     }
 
 
-    public GetTapesSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId.toString();
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public GetTapesSpectraS3Request withStorageDomainMemberId(final UUID storageDomainMemberId) {
+        this.storageDomainMemberId = storageDomainMemberId.toString();
+        this.updateQueryParam("storage_domain_member_id", storageDomainMemberId);
         return this;
     }
 
 
-    public GetTapesSpectraS3Request withStorageDomainId(final String storageDomainId) {
-        this.storageDomainId = storageDomainId;
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public GetTapesSpectraS3Request withStorageDomainMemberId(final String storageDomainMemberId) {
+        this.storageDomainMemberId = storageDomainMemberId;
+        this.updateQueryParam("storage_domain_member_id", storageDomainMemberId);
         return this;
     }
 
@@ -360,8 +360,8 @@ public TapeState getState() {
     }
 
 
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomainMemberId() {
+        return this.storageDomainMemberId;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllPoolsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllPoolsSpectraS3Request.java
index d9e13c021..235dbaf53 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllPoolsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllPoolsSpectraS3Request.java
@@ -18,16 +18,15 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.spectralogic.ds3client.models.ImportConflictResolutionMode;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import java.util.UUID;
 import com.spectralogic.ds3client.models.Priority;
 
 public class ImportAllPoolsSpectraS3Request extends AbstractRequest {
 
     // Variables
     
-    private ImportConflictResolutionMode conflictResolutionMode;
+    private String conflictResolutionMode;
 
     private String dataPolicyId;
 
@@ -50,7 +49,7 @@ public ImportAllPoolsSpectraS3Request() {
 
     }
 
-    public ImportAllPoolsSpectraS3Request withConflictResolutionMode(final ImportConflictResolutionMode conflictResolutionMode) {
+    public ImportAllPoolsSpectraS3Request withConflictResolutionMode(final String conflictResolutionMode) {
         this.conflictResolutionMode = conflictResolutionMode;
         this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
         return this;
@@ -131,7 +130,7 @@ public String getPath() {
         return "/_rest_/pool";
     }
     
-    public ImportConflictResolutionMode getConflictResolutionMode() {
+    public String getConflictResolutionMode() {
         return this.conflictResolutionMode;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllTapesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllTapesSpectraS3Request.java
index d7c174b90..a5050a9a0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllTapesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllTapesSpectraS3Request.java
@@ -18,16 +18,15 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.spectralogic.ds3client.models.ImportConflictResolutionMode;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import java.util.UUID;
 import com.spectralogic.ds3client.models.Priority;
 
 public class ImportAllTapesSpectraS3Request extends AbstractRequest {
 
     // Variables
     
-    private ImportConflictResolutionMode conflictResolutionMode;
+    private String conflictResolutionMode;
 
     private String dataPolicyId;
 
@@ -50,7 +49,7 @@ public ImportAllTapesSpectraS3Request() {
 
     }
 
-    public ImportAllTapesSpectraS3Request withConflictResolutionMode(final ImportConflictResolutionMode conflictResolutionMode) {
+    public ImportAllTapesSpectraS3Request withConflictResolutionMode(final String conflictResolutionMode) {
         this.conflictResolutionMode = conflictResolutionMode;
         this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
         return this;
@@ -131,7 +130,7 @@ public String getPath() {
         return "/_rest_/tape";
     }
     
-    public ImportConflictResolutionMode getConflictResolutionMode() {
+    public String getConflictResolutionMode() {
         return this.conflictResolutionMode;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAzureTargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAzureTargetSpectraS3Request.java
index 55c99503d..c92c57cd2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAzureTargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAzureTargetSpectraS3Request.java
@@ -19,7 +19,6 @@
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
-import com.spectralogic.ds3client.models.ImportConflictResolutionMode;
 import java.util.UUID;
 import com.spectralogic.ds3client.models.Priority;
 
@@ -31,8 +30,6 @@ public class ImportAzureTargetSpectraS3Request extends AbstractRequest {
 
     private final String cloudBucketName;
 
-    private ImportConflictResolutionMode conflictResolutionMode;
-
     private String dataPolicyId;
 
     private Priority priority;
@@ -52,13 +49,6 @@ public ImportAzureTargetSpectraS3Request(final String azureTarget, final String
 
     }
 
-    public ImportAzureTargetSpectraS3Request withConflictResolutionMode(final ImportConflictResolutionMode conflictResolutionMode) {
-        this.conflictResolutionMode = conflictResolutionMode;
-        this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
-        return this;
-    }
-
-
     public ImportAzureTargetSpectraS3Request withDataPolicyId(final UUID dataPolicyId) {
         this.dataPolicyId = dataPolicyId.toString();
         this.updateQueryParam("data_policy_id", dataPolicyId);
@@ -115,11 +105,6 @@ public String getCloudBucketName() {
     }
 
 
-    public ImportConflictResolutionMode getConflictResolutionMode() {
-        return this.conflictResolutionMode;
-    }
-
-
     public String getDataPolicyId() {
         return this.dataPolicyId;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportPoolSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportPoolSpectraS3Request.java
index 4adf8a55b..4addf085a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportPoolSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportPoolSpectraS3Request.java
@@ -18,9 +18,8 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.spectralogic.ds3client.models.ImportConflictResolutionMode;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import java.util.UUID;
 import com.spectralogic.ds3client.models.Priority;
 
 public class ImportPoolSpectraS3Request extends AbstractRequest {
@@ -29,7 +28,7 @@ public class ImportPoolSpectraS3Request extends AbstractRequest {
     
     private final String pool;
 
-    private ImportConflictResolutionMode conflictResolutionMode;
+    private String conflictResolutionMode;
 
     private String dataPolicyId;
 
@@ -53,7 +52,7 @@ public ImportPoolSpectraS3Request(final String pool) {
 
     }
 
-    public ImportPoolSpectraS3Request withConflictResolutionMode(final ImportConflictResolutionMode conflictResolutionMode) {
+    public ImportPoolSpectraS3Request withConflictResolutionMode(final String conflictResolutionMode) {
         this.conflictResolutionMode = conflictResolutionMode;
         this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
         return this;
@@ -139,7 +138,7 @@ public String getPool() {
     }
 
 
-    public ImportConflictResolutionMode getConflictResolutionMode() {
+    public String getConflictResolutionMode() {
         return this.conflictResolutionMode;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportS3TargetSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportS3TargetSpectraS3Request.java
index 8cdf9878e..5350614f2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportS3TargetSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportS3TargetSpectraS3Request.java
@@ -19,7 +19,6 @@
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
-import com.spectralogic.ds3client.models.ImportConflictResolutionMode;
 import java.util.UUID;
 import com.spectralogic.ds3client.models.Priority;
 
@@ -31,8 +30,6 @@ public class ImportS3TargetSpectraS3Request extends AbstractRequest {
 
     private final String cloudBucketName;
 
-    private ImportConflictResolutionMode conflictResolutionMode;
-
     private String dataPolicyId;
 
     private Priority priority;
@@ -52,13 +49,6 @@ public ImportS3TargetSpectraS3Request(final String cloudBucketName, final String
 
     }
 
-    public ImportS3TargetSpectraS3Request withConflictResolutionMode(final ImportConflictResolutionMode conflictResolutionMode) {
-        this.conflictResolutionMode = conflictResolutionMode;
-        this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
-        return this;
-    }
-
-
     public ImportS3TargetSpectraS3Request withDataPolicyId(final UUID dataPolicyId) {
         this.dataPolicyId = dataPolicyId.toString();
         this.updateQueryParam("data_policy_id", dataPolicyId);
@@ -115,11 +105,6 @@ public String getCloudBucketName() {
     }
 
 
-    public ImportConflictResolutionMode getConflictResolutionMode() {
-        return this.conflictResolutionMode;
-    }
-
-
     public String getDataPolicyId() {
         return this.dataPolicyId;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportTapeSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportTapeSpectraS3Request.java
index dc8bfd7f7..b126327ef 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportTapeSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportTapeSpectraS3Request.java
@@ -18,9 +18,8 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.spectralogic.ds3client.models.ImportConflictResolutionMode;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import java.util.UUID;
 import com.spectralogic.ds3client.models.Priority;
 
 public class ImportTapeSpectraS3Request extends AbstractRequest {
@@ -29,7 +28,7 @@ public class ImportTapeSpectraS3Request extends AbstractRequest {
     
     private final String tapeId;
 
-    private ImportConflictResolutionMode conflictResolutionMode;
+    private String conflictResolutionMode;
 
     private String dataPolicyId;
 
@@ -61,7 +60,7 @@ public ImportTapeSpectraS3Request(final String tapeId) {
 
     }
 
-    public ImportTapeSpectraS3Request withConflictResolutionMode(final ImportConflictResolutionMode conflictResolutionMode) {
+    public ImportTapeSpectraS3Request withConflictResolutionMode(final String conflictResolutionMode) {
         this.conflictResolutionMode = conflictResolutionMode;
         this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
         return this;
@@ -147,7 +146,7 @@ public String getTapeId() {
     }
 
 
-    public ImportConflictResolutionMode getConflictResolutionMode() {
+    public String getConflictResolutionMode() {
         return this.conflictResolutionMode;
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPathBackendSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPathBackendSpectraS3Request.java
index 68851dc63..e58104856 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPathBackendSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPathBackendSpectraS3Request.java
@@ -20,7 +20,6 @@
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import java.lang.Integer;
 import com.spectralogic.ds3client.models.AutoInspectMode;
-import com.spectralogic.ds3client.models.ImportConflictResolutionMode;
 import com.spectralogic.ds3client.models.Priority;
 import com.spectralogic.ds3client.models.UnavailableMediaUsagePolicy;
 
@@ -38,12 +37,12 @@ public class ModifyDataPathBackendSpectraS3Request extends AbstractRequest {
 
     private int cacheAvailableRetryAfterInSeconds;
 
-    private ImportConflictResolutionMode defaultImportConflictResolutionMode;
-
     private Priority defaultVerifyDataAfterImport;
 
     private boolean defaultVerifyDataPriorToImport;
 
+    private boolean iomEnabled;
+
     private Integer partiallyVerifyLastPercentOfTapes;
 
     private UnavailableMediaUsagePolicy unavailableMediaPolicy;
@@ -94,13 +93,6 @@ public ModifyDataPathBackendSpectraS3Request withCacheAvailableRetryAfterInSecon
     }
 
 
-    public ModifyDataPathBackendSpectraS3Request withDefaultImportConflictResolutionMode(final ImportConflictResolutionMode defaultImportConflictResolutionMode) {
-        this.defaultImportConflictResolutionMode = defaultImportConflictResolutionMode;
-        this.updateQueryParam("default_import_conflict_resolution_mode", defaultImportConflictResolutionMode);
-        return this;
-    }
-
-
     public ModifyDataPathBackendSpectraS3Request withDefaultVerifyDataAfterImport(final Priority defaultVerifyDataAfterImport) {
         this.defaultVerifyDataAfterImport = defaultVerifyDataAfterImport;
         this.updateQueryParam("default_verify_data_after_import", defaultVerifyDataAfterImport);
@@ -115,6 +107,13 @@ public ModifyDataPathBackendSpectraS3Request withDefaultVerifyDataPriorToImport(
     }
 
 
+    public ModifyDataPathBackendSpectraS3Request withIomEnabled(final boolean iomEnabled) {
+        this.iomEnabled = iomEnabled;
+        this.updateQueryParam("iom_enabled", iomEnabled);
+        return this;
+    }
+
+
     public ModifyDataPathBackendSpectraS3Request withPartiallyVerifyLastPercentOfTapes(final Integer partiallyVerifyLastPercentOfTapes) {
         this.partiallyVerifyLastPercentOfTapes = partiallyVerifyLastPercentOfTapes;
         this.updateQueryParam("partially_verify_last_percent_of_tapes", partiallyVerifyLastPercentOfTapes);
@@ -179,11 +178,6 @@ public int getCacheAvailableRetryAfterInSeconds() {
     }
 
 
-    public ImportConflictResolutionMode getDefaultImportConflictResolutionMode() {
-        return this.defaultImportConflictResolutionMode;
-    }
-
-
     public Priority getDefaultVerifyDataAfterImport() {
         return this.defaultVerifyDataAfterImport;
     }
@@ -194,6 +188,11 @@ public boolean getDefaultVerifyDataPriorToImport() {
     }
 
 
+    public boolean getIomEnabled() {
+        return this.iomEnabled;
+    }
+
+
     public Integer getPartiallyVerifyLastPercentOfTapes() {
         return this.partiallyVerifyLastPercentOfTapes;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPolicySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPolicySpectraS3Request.java
index 945506790..e5598676f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPolicySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyDataPolicySpectraS3Request.java
@@ -51,6 +51,8 @@ public class ModifyDataPolicySpectraS3Request extends AbstractRequest {
 
     private boolean endToEndCrcRequired;
 
+    private int maxVersionsToKeep;
+
     private String name;
 
     private Priority rebuildPriority;
@@ -141,6 +143,13 @@ public ModifyDataPolicySpectraS3Request withEndToEndCrcRequired(final boolean en
     }
 
 
+    public ModifyDataPolicySpectraS3Request withMaxVersionsToKeep(final int maxVersionsToKeep) {
+        this.maxVersionsToKeep = maxVersionsToKeep;
+        this.updateQueryParam("max_versions_to_keep", maxVersionsToKeep);
+        return this;
+    }
+
+
     public ModifyDataPolicySpectraS3Request withName(final String name) {
         this.name = name;
         this.updateQueryParam("name", name);
@@ -228,6 +237,11 @@ public boolean getEndToEndCrcRequired() {
     }
 
 
+    public int getMaxVersionsToKeep() {
+        return this.maxVersionsToKeep;
+    }
+
+
     public String getName() {
         return this.name;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyStorageDomainMemberSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyStorageDomainMemberSpectraS3Request.java
index 5fe3583a4..e4a6c6463 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyStorageDomainMemberSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyStorageDomainMemberSpectraS3Request.java
@@ -18,6 +18,8 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import java.lang.Integer;
+import com.spectralogic.ds3client.models.StorageDomainMemberState;
 import com.spectralogic.ds3client.models.WritePreferenceLevel;
 import com.google.common.net.UrlEscapers;
 
@@ -27,6 +29,10 @@ public class ModifyStorageDomainMemberSpectraS3Request extends AbstractRequest {
     
     private final String storageDomainMember;
 
+    private Integer autoCompactionThreshold;
+
+    private StorageDomainMemberState state;
+
     private WritePreferenceLevel writePreference;
 
     // Constructor
@@ -37,6 +43,20 @@ public ModifyStorageDomainMemberSpectraS3Request(final String storageDomainMembe
         
     }
 
+    public ModifyStorageDomainMemberSpectraS3Request withAutoCompactionThreshold(final Integer autoCompactionThreshold) {
+        this.autoCompactionThreshold = autoCompactionThreshold;
+        this.updateQueryParam("auto_compaction_threshold", autoCompactionThreshold);
+        return this;
+    }
+
+
+    public ModifyStorageDomainMemberSpectraS3Request withState(final StorageDomainMemberState state) {
+        this.state = state;
+        this.updateQueryParam("state", state);
+        return this;
+    }
+
+
     public ModifyStorageDomainMemberSpectraS3Request withWritePreference(final WritePreferenceLevel writePreference) {
         this.writePreference = writePreference;
         this.updateQueryParam("write_preference", writePreference);
@@ -60,6 +80,16 @@ public String getStorageDomainMember() {
     }
 
 
+    public Integer getAutoCompactionThreshold() {
+        return this.autoCompactionThreshold;
+    }
+
+
+    public StorageDomainMemberState getState() {
+        return this.state;
+    }
+
+
     public WritePreferenceLevel getWritePreference() {
         return this.writePreference;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
index d59f129a3..0dba73b94 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
@@ -27,6 +27,8 @@ public class ModifyTapePartitionSpectraS3Request extends AbstractRequest {
     
     private final String tapePartition;
 
+    private boolean autoCompactionEnabled;
+
     private int minimumReadReservedDrives;
 
     private int minimumWriteReservedDrives;
@@ -41,6 +43,13 @@ public ModifyTapePartitionSpectraS3Request(final String tapePartition) {
         
     }
 
+    public ModifyTapePartitionSpectraS3Request withAutoCompactionEnabled(final boolean autoCompactionEnabled) {
+        this.autoCompactionEnabled = autoCompactionEnabled;
+        this.updateQueryParam("auto_compaction_enabled", autoCompactionEnabled);
+        return this;
+    }
+
+
     public ModifyTapePartitionSpectraS3Request withMinimumReadReservedDrives(final int minimumReadReservedDrives) {
         this.minimumReadReservedDrives = minimumReadReservedDrives;
         this.updateQueryParam("minimum_read_reserved_drives", minimumReadReservedDrives);
@@ -78,6 +87,11 @@ public String getTapePartition() {
     }
 
 
+    public boolean getAutoCompactionEnabled() {
+        return this.autoCompactionEnabled;
+    }
+
+
     public int getMinimumReadReservedDrives() {
         return this.minimumReadReservedDrives;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyUserSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyUserSpectraS3Request.java
index a4350dd3d..2e80d47a2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyUserSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyUserSpectraS3Request.java
@@ -29,6 +29,8 @@ public class ModifyUserSpectraS3Request extends AbstractRequest {
 
     private String defaultDataPolicyId;
 
+    private int maxBuckets;
+
     private String name;
 
     private String secretKey;
@@ -61,6 +63,13 @@ public ModifyUserSpectraS3Request withDefaultDataPolicyId(final String defaultDa
     }
 
 
+    public ModifyUserSpectraS3Request withMaxBuckets(final int maxBuckets) {
+        this.maxBuckets = maxBuckets;
+        this.updateQueryParam("max_buckets", maxBuckets);
+        return this;
+    }
+
+
     public ModifyUserSpectraS3Request withName(final String name) {
         this.name = name;
         this.updateQueryParam("name", name);
@@ -96,6 +105,11 @@ public String getDefaultDataPolicyId() {
     }
 
 
+    public int getMaxBuckets() {
+        return this.maxBuckets;
+    }
+
+
     public String getName() {
         return this.name;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicySpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicySpectraS3Request.java
index 0626a9a40..634ad0b16 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicySpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutDataPolicySpectraS3Request.java
@@ -50,6 +50,8 @@ public class PutDataPolicySpectraS3Request extends AbstractRequest {
 
     private boolean endToEndCrcRequired;
 
+    private int maxVersionsToKeep;
+
     private Priority rebuildPriority;
 
     private VersioningLevel versioning;
@@ -134,6 +136,13 @@ public PutDataPolicySpectraS3Request withEndToEndCrcRequired(final boolean endTo
     }
 
 
+    public PutDataPolicySpectraS3Request withMaxVersionsToKeep(final int maxVersionsToKeep) {
+        this.maxVersionsToKeep = maxVersionsToKeep;
+        this.updateQueryParam("max_versions_to_keep", maxVersionsToKeep);
+        return this;
+    }
+
+
     public PutDataPolicySpectraS3Request withRebuildPriority(final Priority rebuildPriority) {
         this.rebuildPriority = rebuildPriority;
         this.updateQueryParam("rebuild_priority", rebuildPriority);
@@ -214,6 +223,11 @@ public boolean getEndToEndCrcRequired() {
     }
 
 
+    public int getMaxVersionsToKeep() {
+        return this.maxVersionsToKeep;
+    }
+
+
     public Priority getRebuildPriority() {
         return this.rebuildPriority;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
index c74c01fd6..3c44e3a23 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/PutTapeStorageDomainMemberSpectraS3Request.java
@@ -20,6 +20,7 @@
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import java.util.UUID;
 import com.google.common.net.UrlEscapers;
+import java.lang.Integer;
 import com.spectralogic.ds3client.models.WritePreferenceLevel;
 
 public class PutTapeStorageDomainMemberSpectraS3Request extends AbstractRequest {
@@ -32,6 +33,8 @@ public class PutTapeStorageDomainMemberSpectraS3Request extends AbstractRequest
 
     private final String tapeType;
 
+    private Integer autoCompactionThreshold;
+
     private WritePreferenceLevel writePreference;
 
     // Constructor
@@ -64,6 +67,13 @@ public PutTapeStorageDomainMemberSpectraS3Request(final String storageDomainId,
 
     }
 
+    public PutTapeStorageDomainMemberSpectraS3Request withAutoCompactionThreshold(final Integer autoCompactionThreshold) {
+        this.autoCompactionThreshold = autoCompactionThreshold;
+        this.updateQueryParam("auto_compaction_threshold", autoCompactionThreshold);
+        return this;
+    }
+
+
     public PutTapeStorageDomainMemberSpectraS3Request withWritePreference(final WritePreferenceLevel writePreference) {
         this.writePreference = writePreference;
         this.updateQueryParam("write_preference", writePreference);
@@ -97,6 +107,11 @@ public String getTapeType() {
     }
 
 
+    public Integer getAutoCompactionThreshold() {
+        return this.autoCompactionThreshold;
+    }
+
+
     public WritePreferenceLevel getWritePreference() {
         return this.writePreference;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
new file mode 100644
index 000000000..8ba4125bc
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
@@ -0,0 +1,83 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.spectrads3;
+
+import com.spectralogic.ds3client.networking.HttpVerb;
+import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.google.common.net.UrlEscapers;
+import com.spectralogic.ds3client.models.Priority;
+
+public class StageObjectsJobSpectraS3Request extends AbstractRequest {
+
+    // Variables
+    
+    private final String bucketName;
+
+    private String name;
+
+    private Priority priority;
+
+    // Constructor
+    
+    
+    public StageObjectsJobSpectraS3Request(final String bucketName) {
+        this.bucketName = bucketName;
+        
+        this.getQueryParams().put("operation", "start_bulk_stage");
+
+    }
+
+    public StageObjectsJobSpectraS3Request withName(final String name) {
+        this.name = name;
+        this.updateQueryParam("name", name);
+        return this;
+    }
+
+
+    public StageObjectsJobSpectraS3Request withPriority(final Priority priority) {
+        this.priority = priority;
+        this.updateQueryParam("priority", priority);
+        return this;
+    }
+
+
+
+    @Override
+    public HttpVerb getVerb() {
+        return HttpVerb.PUT;
+    }
+
+    @Override
+    public String getPath() {
+        return "/_rest_/bucket/" + this.bucketName;
+    }
+    
+    public String getBucketName() {
+        return this.bucketName;
+    }
+
+
+    public String getName() {
+        return this.name;
+    }
+
+
+    public Priority getPriority() {
+        return this.priority;
+    }
+
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
new file mode 100644
index 000000000..7f7143462
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
@@ -0,0 +1,28 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.spectrads3;
+
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
+
+public class StageObjectsJobSpectraS3Response extends AbstractResponse {
+    
+    public StageObjectsJobSpectraS3Response(final String checksum, final ChecksumType.Type checksumType) {
+        super(checksum, checksumType);
+    }
+
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/VerifyPhysicalPlacementForObjectsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/VerifyPhysicalPlacementForObjectsSpectraS3Request.java
index 6a1f7f72c..4057799e1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/VerifyPhysicalPlacementForObjectsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/VerifyPhysicalPlacementForObjectsSpectraS3Request.java
@@ -27,7 +27,6 @@
 import java.util.List;
 import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
 public class VerifyPhysicalPlacementForObjectsSpectraS3Request extends AbstractRequest {
@@ -38,7 +37,7 @@ public class VerifyPhysicalPlacementForObjectsSpectraS3Request extends AbstractR
 
     private final List objects;
 
-    private String storageDomainId;
+    private String storageDomain;
     private long size = 0;
 
     // Constructor
@@ -52,16 +51,9 @@ public VerifyPhysicalPlacementForObjectsSpectraS3Request(final String bucketName
 
     }
 
-    public VerifyPhysicalPlacementForObjectsSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId.toString();
-        this.updateQueryParam("storage_domain_id", storageDomainId);
-        return this;
-    }
-
-
-    public VerifyPhysicalPlacementForObjectsSpectraS3Request withStorageDomainId(final String storageDomainId) {
-        this.storageDomainId = storageDomainId;
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public VerifyPhysicalPlacementForObjectsSpectraS3Request withStorageDomain(final String storageDomain) {
+        this.storageDomain = storageDomain;
+        this.updateQueryParam("storage_domain", storageDomain);
         return this;
     }
 
@@ -107,8 +99,8 @@ public List getObjects() {
     }
 
 
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomain() {
+        return this.storageDomain;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request.java
index 98b0597fb..0f4ee9d73 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request.java
@@ -27,7 +27,6 @@
 import java.util.List;
 import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import java.util.UUID;
 import com.google.common.net.UrlEscapers;
 
 public class VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request extends AbstractRequest {
@@ -38,7 +37,7 @@ public class VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request ex
 
     private final List objects;
 
-    private String storageDomainId;
+    private String storageDomain;
     private long size = 0;
 
     // Constructor
@@ -53,16 +52,9 @@ public VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request(final St
         this.getQueryParams().put("full_details", null);
     }
 
-    public VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request withStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId.toString();
-        this.updateQueryParam("storage_domain_id", storageDomainId);
-        return this;
-    }
-
-
-    public VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request withStorageDomainId(final String storageDomainId) {
-        this.storageDomainId = storageDomainId;
-        this.updateQueryParam("storage_domain_id", storageDomainId);
+    public VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request withStorageDomain(final String storageDomain) {
+        this.storageDomain = storageDomain;
+        this.updateQueryParam("storage_domain", storageDomain);
         return this;
     }
 
@@ -108,8 +100,8 @@ public List getObjects() {
     }
 
 
-    public String getStorageDomainId() {
-        return this.storageDomainId;
+    public String getStorageDomain() {
+        return this.storageDomain;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
index de63306c6..9179c432f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ActiveJob.java
@@ -79,6 +79,9 @@ public class ActiveJob {
     @JsonProperty("RequestType")
     private JobRequestType requestType;
 
+    @JsonProperty("Restore")
+    private JobRestore restore;
+
     @JsonProperty("Truncated")
     private boolean truncated;
 
@@ -260,6 +263,15 @@ public void setRequestType(final JobRequestType requestType) {
     }
 
 
+    public JobRestore getRestore() {
+        return this.restore;
+    }
+
+    public void setRestore(final JobRestore restore) {
+        this.restore = restore;
+    }
+
+
     public boolean getTruncated() {
         return this.truncated;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Contents.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Contents.java
index 9f5f97dd4..217d6dffb 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Contents.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Contents.java
@@ -18,7 +18,9 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
+import java.lang.Boolean;
 import java.util.Date;
+import java.util.UUID;
 
 @JacksonXmlRootElement(namespace = "Data")
 public class Contents {
@@ -27,6 +29,9 @@ public class Contents {
     @JsonProperty("ETag")
     private String eTag;
 
+    @JsonProperty("IsLatest")
+    private Boolean isLatest;
+
     @JsonProperty("Key")
     private String key;
 
@@ -42,6 +47,9 @@ public class Contents {
     @JsonProperty("StorageClass")
     private String storageClass;
 
+    @JsonProperty("VersionId")
+    private UUID versionId;
+
     // Constructor
     public Contents() {
         //pass
@@ -58,6 +66,15 @@ public void setETag(final String eTag) {
     }
 
 
+    public Boolean getIsLatest() {
+        return this.isLatest;
+    }
+
+    public void setIsLatest(final Boolean isLatest) {
+        this.isLatest = isLatest;
+    }
+
+
     public String getKey() {
         return this.key;
     }
@@ -102,4 +119,13 @@ public void setStorageClass(final String storageClass) {
         this.storageClass = storageClass;
     }
 
+
+    public UUID getVersionId() {
+        return this.versionId;
+    }
+
+    public void setVersionId(final UUID versionId) {
+        this.versionId = versionId;
+    }
+
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
index b5e9cbba1..94a5ba2f3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPathBackend.java
@@ -40,9 +40,6 @@ public class DataPathBackend {
     @JsonProperty("CacheAvailableRetryAfterInSeconds")
     private int cacheAvailableRetryAfterInSeconds;
 
-    @JsonProperty("DefaultImportConflictResolutionMode")
-    private ImportConflictResolutionMode defaultImportConflictResolutionMode;
-
     @JsonProperty("DefaultVerifyDataAfterImport")
     private Priority defaultVerifyDataAfterImport;
 
@@ -55,6 +52,9 @@ public class DataPathBackend {
     @JsonProperty("InstanceId")
     private UUID instanceId;
 
+    @JsonProperty("IomEnabled")
+    private boolean iomEnabled;
+
     @JsonProperty("LastHeartbeat")
     private Date lastHeartbeat;
 
@@ -122,15 +122,6 @@ public void setCacheAvailableRetryAfterInSeconds(final int cacheAvailableRetryAf
     }
 
 
-    public ImportConflictResolutionMode getDefaultImportConflictResolutionMode() {
-        return this.defaultImportConflictResolutionMode;
-    }
-
-    public void setDefaultImportConflictResolutionMode(final ImportConflictResolutionMode defaultImportConflictResolutionMode) {
-        this.defaultImportConflictResolutionMode = defaultImportConflictResolutionMode;
-    }
-
-
     public Priority getDefaultVerifyDataAfterImport() {
         return this.defaultVerifyDataAfterImport;
     }
@@ -167,6 +158,15 @@ public void setInstanceId(final UUID instanceId) {
     }
 
 
+    public boolean getIomEnabled() {
+        return this.iomEnabled;
+    }
+
+    public void setIomEnabled(final boolean iomEnabled) {
+        this.iomEnabled = iomEnabled;
+    }
+
+
     public Date getLastHeartbeat() {
         return this.lastHeartbeat;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
index 3e7ad9e22..255436d8d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
@@ -67,6 +67,9 @@ public class DataPolicy {
     @JsonProperty("LtfsObjectNamingAllowed")
     private boolean ltfsObjectNamingAllowed;
 
+    @JsonProperty("MaxVersionsToKeep")
+    private int maxVersionsToKeep;
+
     @JsonProperty("Name")
     private String name;
 
@@ -209,6 +212,15 @@ public void setLtfsObjectNamingAllowed(final boolean ltfsObjectNamingAllowed) {
     }
 
 
+    public int getMaxVersionsToKeep() {
+        return this.maxVersionsToKeep;
+    }
+
+    public void setMaxVersionsToKeep(final int maxVersionsToKeep) {
+        this.maxVersionsToKeep = maxVersionsToKeep;
+    }
+
+
     public String getName() {
         return this.name;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
index 8008ca4f3..b54a5e473 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
@@ -27,6 +27,9 @@
 public class DetailedTapePartition {
 
     // Variables
+    @JsonProperty("AutoCompactionEnabled")
+    private boolean autoCompactionEnabled;
+
     @JsonProperty("DriveType")
     private TapeDriveType driveType;
 
@@ -78,6 +81,15 @@ public DetailedTapePartition() {
 
     // Getters and Setters
     
+    public boolean getAutoCompactionEnabled() {
+        return this.autoCompactionEnabled;
+    }
+
+    public void setAutoCompactionEnabled(final boolean autoCompactionEnabled) {
+        this.autoCompactionEnabled = autoCompactionEnabled;
+    }
+
+
     public TapeDriveType getDriveType() {
         return this.driveType;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ImportConflictResolutionMode.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobRestore.java
similarity index 89%
rename from ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ImportConflictResolutionMode.java
rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobRestore.java
index bcd4bc4db..e15314c85 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ImportConflictResolutionMode.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/JobRestore.java
@@ -16,8 +16,8 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.models;
 
-public enum ImportConflictResolutionMode {
-    CANCEL,
-    ACCEPT_MOST_RECENT,
-    ACCEPT_EXISTING
+public enum JobRestore {
+    NO,
+    YES,
+    PERMANENT_ONLY
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListBucketResult.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListBucketResult.java
index 96b1bf0d6..ae82147f2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListBucketResult.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/ListBucketResult.java
@@ -60,6 +60,10 @@ public class ListBucketResult {
     @JsonProperty("IsTruncated")
     private boolean truncated;
 
+    @JsonProperty("Version")
+    @JacksonXmlElementWrapper(useWrapping = false)
+    private List versionedObjects = new ArrayList<>();
+
     // Constructor
     public ListBucketResult() {
         //pass
@@ -156,4 +160,13 @@ public void setTruncated(final boolean truncated) {
         this.truncated = truncated;
     }
 
+
+    public List getVersionedObjects() {
+        return this.versionedObjects;
+    }
+
+    public void setVersionedObjects(final List versionedObjects) {
+        this.versionedObjects = versionedObjects;
+    }
+
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
index 3a683063e..757d401f0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
@@ -27,6 +27,9 @@
 public class NamedDetailedTapePartition {
 
     // Variables
+    @JsonProperty("AutoCompactionEnabled")
+    private boolean autoCompactionEnabled;
+
     @JsonProperty("DriveType")
     private TapeDriveType driveType;
 
@@ -78,6 +81,15 @@ public NamedDetailedTapePartition() {
 
     // Getters and Setters
     
+    public boolean getAutoCompactionEnabled() {
+        return this.autoCompactionEnabled;
+    }
+
+    public void setAutoCompactionEnabled(final boolean autoCompactionEnabled) {
+        this.autoCompactionEnabled = autoCompactionEnabled;
+    }
+
+
     public TapeDriveType getDriveType() {
         return this.driveType;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Pool.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Pool.java
index 00581f1b8..86e19b17c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Pool.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Pool.java
@@ -73,8 +73,8 @@ public class Pool {
     @JsonProperty("State")
     private PoolState state;
 
-    @JsonProperty("StorageDomainId")
-    private UUID storageDomainId;
+    @JsonProperty("StorageDomainMemberId")
+    private UUID storageDomainMemberId;
 
     @JsonProperty("TotalCapacity")
     private long totalCapacity;
@@ -236,12 +236,12 @@ public void setState(final PoolState state) {
     }
 
 
-    public UUID getStorageDomainId() {
-        return this.storageDomainId;
+    public UUID getStorageDomainMemberId() {
+        return this.storageDomainMemberId;
     }
 
-    public void setStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId;
+    public void setStorageDomainMemberId(final UUID storageDomainMemberId) {
+        this.storageDomainMemberId = storageDomainMemberId;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/RestOperationType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/RestOperationType.java
index 9f951979b..0babc6ea6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/RestOperationType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/RestOperationType.java
@@ -36,6 +36,7 @@ public enum RestOperationType {
     REGENERATE_SECRET_KEY,
     START_BULK_GET,
     START_BULK_PUT,
+    START_BULK_STAGE,
     START_BULK_VERIFY,
     VERIFY,
     VERIFY_SAFE_TO_START_BULK_PUT,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SpectraUser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SpectraUser.java
index 9dd159b0c..5b47f2592 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SpectraUser.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SpectraUser.java
@@ -33,6 +33,9 @@ public class SpectraUser {
     @JsonProperty("Id")
     private UUID id;
 
+    @JsonProperty("MaxBuckets")
+    private int maxBuckets;
+
     @JsonProperty("Name")
     private String name;
 
@@ -73,6 +76,15 @@ public void setId(final UUID id) {
     }
 
 
+    public int getMaxBuckets() {
+        return this.maxBuckets;
+    }
+
+    public void setMaxBuckets(final int maxBuckets) {
+        this.maxBuckets = maxBuckets;
+    }
+
+
     public String getName() {
         return this.name;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
index 971f592c7..5722fdca6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/StorageDomainMember.java
@@ -24,6 +24,9 @@
 public class StorageDomainMember {
 
     // Variables
+    @JsonProperty("AutoCompactionThreshold")
+    private Integer autoCompactionThreshold;
+
     @JsonProperty("Id")
     private UUID id;
 
@@ -52,6 +55,15 @@ public StorageDomainMember() {
 
     // Getters and Setters
     
+    public Integer getAutoCompactionThreshold() {
+        return this.autoCompactionThreshold;
+    }
+
+    public void setAutoCompactionThreshold(final Integer autoCompactionThreshold) {
+        this.autoCompactionThreshold = autoCompactionThreshold;
+    }
+
+
     public UUID getId() {
         return this.id;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobPool.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobPool.java
index 4a304e644..6e9c5c18b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobPool.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobPool.java
@@ -40,6 +40,9 @@ public class SuspectBlobPool {
     @JsonProperty("LastAccessed")
     private Date lastAccessed;
 
+    @JsonProperty("ObsoletionId")
+    private UUID obsoletionId;
+
     @JsonProperty("PoolId")
     private UUID poolId;
 
@@ -95,6 +98,15 @@ public void setLastAccessed(final Date lastAccessed) {
     }
 
 
+    public UUID getObsoletionId() {
+        return this.obsoletionId;
+    }
+
+    public void setObsoletionId(final UUID obsoletionId) {
+        this.obsoletionId = obsoletionId;
+    }
+
+
     public UUID getPoolId() {
         return this.poolId;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobTape.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobTape.java
index 1f628d2ba..8d162caec 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobTape.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobTape.java
@@ -30,6 +30,9 @@ public class SuspectBlobTape {
     @JsonProperty("Id")
     private UUID id;
 
+    @JsonProperty("ObsoletionId")
+    private UUID obsoletionId;
+
     @JsonProperty("OrderIndex")
     private int orderIndex;
 
@@ -61,6 +64,15 @@ public void setId(final UUID id) {
     }
 
 
+    public UUID getObsoletionId() {
+        return this.obsoletionId;
+    }
+
+    public void setObsoletionId(final UUID obsoletionId) {
+        this.obsoletionId = obsoletionId;
+    }
+
+
     public int getOrderIndex() {
         return this.orderIndex;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
index 4b6a50990..938b4ad5c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/Tape.java
@@ -85,8 +85,8 @@ public class Tape {
     @JsonProperty("State")
     private TapeState state;
 
-    @JsonProperty("StorageDomainId")
-    private UUID storageDomainId;
+    @JsonProperty("StorageDomainMemberId")
+    private UUID storageDomainMemberId;
 
     @JsonProperty("TakeOwnershipPending")
     private boolean takeOwnershipPending;
@@ -290,12 +290,12 @@ public void setState(final TapeState state) {
     }
 
 
-    public UUID getStorageDomainId() {
-        return this.storageDomainId;
+    public UUID getStorageDomainMemberId() {
+        return this.storageDomainMemberId;
     }
 
-    public void setStorageDomainId(final UUID storageDomainId) {
-        this.storageDomainId = storageDomainId;
+    public void setStorageDomainMemberId(final UUID storageDomainMemberId) {
+        this.storageDomainMemberId = storageDomainMemberId;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
index 52a962cd2..a90f131a4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
@@ -24,6 +24,9 @@
 public class TapePartition {
 
     // Variables
+    @JsonProperty("AutoCompactionEnabled")
+    private boolean autoCompactionEnabled;
+
     @JsonProperty("DriveType")
     private TapeDriveType driveType;
 
@@ -67,6 +70,15 @@ public TapePartition() {
 
     // Getters and Setters
     
+    public boolean getAutoCompactionEnabled() {
+        return this.autoCompactionEnabled;
+    }
+
+    public void setAutoCompactionEnabled(final boolean autoCompactionEnabled) {
+        this.autoCompactionEnabled = autoCompactionEnabled;
+    }
+
+
     public TapeDriveType getDriveType() {
         return this.driveType;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureType.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureType.java
index be8cdb2dd..5e1203ab7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureType.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartitionFailureType.java
@@ -31,5 +31,6 @@ public enum TapePartitionFailureType {
     TAPE_DRIVE_TYPE_MISMATCH,
     TAPE_EJECTION_BY_OPERATOR_REQUIRED,
     TAPE_MEDIA_TYPE_INCOMPATIBLE,
-    TAPE_REMOVAL_UNEXPECTED
+    TAPE_REMOVAL_UNEXPECTED,
+    TAPE_IN_INVALID_PARTITION
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeState.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeState.java
index d943bd7ab..a6a06f17b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeState.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapeState.java
@@ -38,6 +38,7 @@ public enum TapeState {
     CANNOT_FORMAT_DUE_TO_WRITE_PROTECTION,
     SERIAL_NUMBER_MISMATCH,
     BAR_CODE_MISSING,
+    AUTO_COMPACTION_IN_PROGRESS,
     FORMAT_PENDING,
     FORMAT_IN_PROGRESS,
     EJECT_TO_EE_IN_PROGRESS,
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/VersioningLevel.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/VersioningLevel.java
index 46ee8e863..05fa2be69 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/VersioningLevel.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/VersioningLevel.java
@@ -18,5 +18,6 @@
 
 public enum VersioningLevel {
     NONE,
-    KEEP_LATEST
+    KEEP_LATEST,
+    KEEP_MULTIPLE_VERSIONS
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 62caa3319..476cc2774 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -1425,7 +1425,7 @@ public void getPhysicalPlacementForObjectsWithFullDetailsTest() throws IOExcepti
 
     @Test
     public void verifyPhysicalPlacementForObjectsTest() throws IOException {
-        final String responsePayload = "false10000t1false48d30ecb-84f1-4721-9832-7aa165a1dd7776343269-c32a-4cb0-aec4-57a9dccce6eaPENDING_INSPECTIONfalse20000LTO5false";
+        final String responsePayload = "false10000t1false48d30ecb-84f1-4721-9832-7aa165a1dd7776343269-c32a-4cb0-aec4-57a9dccce6eaINSPECTION_PENDINGfalse20000LTO5false";
         final String bucketName = "BucketName";
 
         final Map queryParams = new HashMap<>();
@@ -1445,7 +1445,7 @@ public void verifyPhysicalPlacementForObjectsTest() throws IOException {
 
     @Test
     public void verifyPhysicalPlacementForObjectsWithFullDetailsTest() throws IOException {
-        final String responsePayload = "false10000t1false5a7bb215-4aff-4806-b217-5fe01ade6a2c2e5b25fc-546e-45b0-951e-8f3d80bb7823PENDING_INSPECTIONfalse20000LTO5false";
+        final String responsePayload = "false10000t1false5a7bb215-4aff-4806-b217-5fe01ade6a2c2e5b25fc-546e-45b0-951e-8f3d80bb7823INSPECTION_PENDINGfalse20000LTO5false";
         final String bucketName = "BucketName";
 
         final Map queryParams = new HashMap<>();

From 2f43352a36e0aeb42b5ecadd2fca31fc9bc1b247 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 21 Feb 2018 14:30:31 -0700
Subject: [PATCH 146/229] Added versions to canonicalized query params and
 updated unit tests

---
 .../integration/VersionedObject_Test.java     | 23 ++++++++--
 .../ds3client/models/bulk/Ds3Object.java      | 42 ++++---------------
 .../ds3client/utils/Signature.java            |  8 +++-
 .../ds3client/Ds3Client_Test.java             | 17 +++++---
 .../ds3client/serializer/XmlOutput_Test.java  | 12 +-----
 5 files changed, 48 insertions(+), 54 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
index 075c5127a..93fed68cb 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
@@ -16,11 +16,15 @@
 package com.spectralogic.ds3client.integration;
 
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.GetBucketRequest;
+import com.spectralogic.ds3client.commands.GetBucketResponse;
+import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.models.VersioningLevel;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -29,9 +33,13 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.List;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 import static com.spectralogic.ds3client.integration.Util.*;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
 
 public class VersionedObject_Test {
 
@@ -40,7 +48,6 @@ public class VersionedObject_Test {
     private static final Ds3Client CLIENT = Util.fromEnv();
     private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(CLIENT);
     private static final String TEST_ENV_NAME = "java_versioned_object_test";
-    private static final int RETRIES = 5;
 
     private static TempStorageIds envStorageIds;
     private static UUID envDataPolicyId;
@@ -79,10 +86,20 @@ public void getObjectsWithVersioning() throws IOException, URISyntaxException {
             loadTestBook(CLIENT, BOOKS[0], objectName, TEST_ENV_NAME); // putting beowulf as content
             loadTestBook(CLIENT, BOOKS[1], objectName, TEST_ENV_NAME); // putting sherlock holmes as content
 
-            // TODO Get the version of the objects
+            // Get the version of the objects
+            final GetBucketRequest getBucketRequest = new GetBucketRequest(TEST_ENV_NAME).withVersions(true);
+            final GetBucketResponse getBucketResponse = CLIENT.getBucket(getBucketRequest);
 
-            // TODO Create bulk get job with both versions of object specified
+            assertThat(getBucketResponse.getListBucketResult().getObjects().size(), is(0));
+            assertThat(getBucketResponse.getListBucketResult().getVersionedObjects().size(), is(2));
 
+            // Create bulk get job with both versions of object specified
+            final List objects = getBucketResponse.getListBucketResult().getVersionedObjects().stream()
+                    .map(obj -> new Ds3Object(obj.getKey(), obj.getVersionId()))
+                    .collect(Collectors.toList());
+
+            final GetBulkJobSpectraS3Request getBulkRequest = new GetBulkJobSpectraS3Request(TEST_ENV_NAME, objects);
+            CLIENT.getBulkJobSpectraS3(getBulkRequest);
 
         } finally {
             cancelAllJobsForBucket(CLIENT, TEST_ENV_NAME);
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
index 5c1c9f7bc..38e7012e1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
@@ -41,30 +41,6 @@ public Ds3Object() {
         //This constructor is used just for serialization.
     }
 
-    /**
-     * Use this constructor when getting a version of a file from DS3.
-     * @param name The name of the object to get from DS3
-     * @param size The size of the object to get from DS3
-     * @param versionId The version ID of the object to get from DS3
-     */
-    public Ds3Object(final String name, final long size, final UUID versionId) {
-        this.name = name;
-        this.size = size;
-        this.versionId = versionId.toString();
-    }
-
-    /**
-     * Use this constructor when getting a version of a file from DS3.
-     * @param name The name of the object to get from DS3
-     * @param size The size of the object to get from DS3
-     * @param versionId The version ID of the object to get from DS3
-     */
-    public Ds3Object(final String name, final long size, final String versionId) {
-        this.name = name;
-        this.size = size;
-        this.versionId = versionId;
-    }
-
     /**
      * Use this constructor when putting files to DS3.
      * @param name The name of the object that will be put to DS3
@@ -124,6 +100,14 @@ public void setSize(final long size) {
         this.size = size;
     }
 
+    public String getVersionId() {
+        return versionId;
+    }
+
+    public void setVersionId(String versionId) {
+        this.versionId = versionId;
+    }
+
     @Override
     public String toString() {
         return this.name + ":" + this.size;
@@ -131,7 +115,7 @@ public String toString() {
 
     @Override
     public int hashCode() {
-        return java.util.Objects.hash(this.name, this.size);
+        return java.util.Objects.hash(this.name, this.size, this.versionId);
     }
 
     @Override
@@ -144,12 +128,4 @@ public boolean equals(final Object obj) {
                 ds3Obj.getSize() == this.getSize() &&
                 ds3Obj.getVersionId().equals(this.getVersionId());
     }
-
-    public String getVersionId() {
-        return versionId;
-    }
-
-    public void setVersionId(String versionId) {
-        this.versionId = versionId;
-    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/Signature.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/Signature.java
index 89b24e39c..cf5f84a9d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/Signature.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/Signature.java
@@ -91,14 +91,18 @@ public static String canonicalizeResource(final String path, final Map queryParams = new HashMap<>();
@@ -1445,7 +1445,7 @@ public void verifyPhysicalPlacementForObjectsTest() throws IOException {
 
     @Test
     public void verifyPhysicalPlacementForObjectsWithFullDetailsTest() throws IOException {
-        final String responsePayload = "false10000t1false5a7bb215-4aff-4806-b217-5fe01ade6a2c2e5b25fc-546e-45b0-951e-8f3d80bb7823INSPECTION_PENDINGfalse20000LTO5false";
+        final String responsePayload = "false10000t1false5a7bb215-4aff-4806-b217-5fe01ade6a2c2e5b25fc-546e-45b0-951e-8f3d80bb7823PENDING_INSPECTIONfalse20000LTO5false";
         final String bucketName = "BucketName";
 
         final Map queryParams = new HashMap<>();
@@ -1470,7 +1470,7 @@ public void ejectStorageDomainBlobsTest() throws IOException {
         queryParams.put("operation", "eject");
         queryParams.put("blobs", null);
         queryParams.put("bucket_id", bucketId);
-        queryParams.put("storage_domain_id", storageDomainId);
+        queryParams.put("storage_domain", storageDomainId);
 
         MockNetwork
                 .expecting(HttpVerb.PUT, "/_rest_/tape", queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD)
@@ -1535,11 +1535,18 @@ public void getBlobsOnTapeTest() throws IOException {
         final Map queryParams = new HashMap<>();
         queryParams.put("operation", "get_physical_placement");
 
-        MockNetwork
+        final Map responseHeaders = new HashMap<>();
+        responseHeaders.put("page-truncated", "0");
+        responseHeaders.put("total-result-count", "2");
+
+        final GetBlobsOnTapeSpectraS3Response result = MockNetwork
                 .expecting(HttpVerb.GET, "/_rest_/tape/" + target, queryParams, null)
-                .returning(200, SIMPLE_BULK_OBJECT_LIST_RESPONSE)
+                .returning(200, SIMPLE_BULK_OBJECT_LIST_RESPONSE, responseHeaders)
                 .asClient()
                 .getBlobsOnTapeSpectraS3(new GetBlobsOnTapeSpectraS3Request(target));
+
+        assertThat(result.getPagingTruncated(), is(0));
+        assertThat(result.getPagingTotalResultCount(), is(2));
     }
 
     @Test
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
index f6776882c..5e360d3df 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
@@ -92,20 +92,10 @@ public void toXmlWithNoSize() {
         assertThat(result, is(expectedString));
     }
 
-    @Test
-    public void toXmlWithNoVersionId() {
-        final String expectedString = "";
-        final List objectList = ImmutableList.of(new Ds3Object("file1", 12, "version1"), new Ds3Object("file2", 5022, "version2")).asList();
-        final Ds3ObjectList ds3ObjectList = new Ds3ObjectList(objectList);
-        final String result = XmlOutput.toXml(ds3ObjectList, true);
-
-        assertThat(result, is(expectedString));
-    }
-
     @Test
     public void toXmlWithVersionId() {
         final String expectedString = "";
-        final List objectList = ImmutableList.of(new Ds3Object("file1", 12, "version1"), new Ds3Object("file2", 5022, "version2")).asList();
+        final List objectList = ImmutableList.of(new Ds3Object("file1", "version1"), new Ds3Object("file2", "version2")).asList();
         final Ds3ObjectList ds3ObjectList = new Ds3ObjectList(objectList);
         final String result = XmlOutput.toXml(ds3ObjectList, false);
 

From a01640296bb4112a9d44294744136709e26da49f Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 21 Feb 2018 16:47:40 -0700
Subject: [PATCH 147/229] Fixed stage objects job command

---
 .../integration/VersionedObject_Test.java     | 32 +++++++++++++++++++
 ...tageObjectsJobSpectraS3ResponseParser.java | 15 ++++++++-
 .../StageObjectsJobSpectraS3Request.java      | 23 ++++++-------
 .../StageObjectsJobSpectraS3Response.java     |  8 +++--
 4 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
index 93fed68cb..e15c4ac85 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
@@ -18,7 +18,9 @@
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.commands.GetBucketRequest;
 import com.spectralogic.ds3client.commands.GetBucketResponse;
+import com.spectralogic.ds3client.commands.spectrads3.DeleteBucketSpectraS3Request;
 import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.StageObjectsJobSpectraS3Request;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
@@ -106,4 +108,34 @@ public void getObjectsWithVersioning() throws IOException, URISyntaxException {
             deleteAllContents(CLIENT, TEST_ENV_NAME);
         }
     }
+
+    @Test
+    public void stageObjectsWithVersioning() throws IOException, URISyntaxException {
+        try {
+            HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
+            final String objectName = "object_with_versions";
+
+            // Put different content for object twice
+            loadTestBook(CLIENT, BOOKS[0], objectName, TEST_ENV_NAME); // putting beowulf as content
+            loadTestBook(CLIENT, BOOKS[1], objectName, TEST_ENV_NAME); // putting sherlock holmes as content
+
+            // Get the version of the objects
+            final GetBucketRequest getBucketRequest = new GetBucketRequest(TEST_ENV_NAME).withVersions(true);
+            final GetBucketResponse getBucketResponse = CLIENT.getBucket(getBucketRequest);
+
+            assertThat(getBucketResponse.getListBucketResult().getObjects().size(), is(0));
+            assertThat(getBucketResponse.getListBucketResult().getVersionedObjects().size(), is(2));
+
+            // Create bulk get job with both versions of object specified
+            final List objects = getBucketResponse.getListBucketResult().getVersionedObjects().stream()
+                    .map(obj -> new Ds3Object(obj.getKey(), obj.getVersionId()))
+                    .collect(Collectors.toList());
+
+            final StageObjectsJobSpectraS3Request stageRequest = new StageObjectsJobSpectraS3Request(TEST_ENV_NAME, objects);
+            CLIENT.stageObjectsJobSpectraS3(stageRequest);
+
+        } finally {
+            CLIENT.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(TEST_ENV_NAME).withForce(true));
+        }
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
index afe980dba..609ad9de3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
@@ -18,18 +18,31 @@
 
 import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
 import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
+import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Response;
 import com.spectralogic.ds3client.commands.spectrads3.StageObjectsJobSpectraS3Response;
+import com.spectralogic.ds3client.models.MasterObjectList;
 import com.spectralogic.ds3client.networking.WebResponse;
+import com.spectralogic.ds3client.serializer.XmlOutput;
+
 import java.io.IOException;
+import java.io.InputStream;
 
 public class StageObjectsJobSpectraS3ResponseParser extends AbstractResponseParser {
-    private final int[] expectedStatusCodes = new int[]{};
+    private final int[] expectedStatusCodes = new int[]{200};
 
     @Override
     public StageObjectsJobSpectraS3Response parseXmlResponse(final WebResponse response) throws IOException {
         final int statusCode = response.getStatusCode();
         if (ResponseParserUtils.validateStatusCode(statusCode, expectedStatusCodes)) {
             switch (statusCode) {
+            case 200:
+                if (ResponseParserUtils.getSizeFromHeaders(response.getHeaders()) == 0) {
+                   return new StageObjectsJobSpectraS3Response(null, this.getChecksum(), this.getChecksumType());
+                }
+                try (final InputStream inputStream = response.getResponseStream()) {
+                    final MasterObjectList result = XmlOutput.fromXml(inputStream, MasterObjectList.class);
+                    return new StageObjectsJobSpectraS3Response(result, this.getChecksum(), this.getChecksumType());
+                }
             default:
                 assert false: "validateStatusCode should have made it impossible to reach this line";
             }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
index 8ba4125bc..62d5f5adc 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
@@ -16,16 +16,15 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands.spectrads3;
 
-import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.google.common.net.UrlEscapers;
+import com.spectralogic.ds3client.BulkCommand;
+import com.spectralogic.ds3client.commands.interfaces.BulkRequest;
 import com.spectralogic.ds3client.models.Priority;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.networking.HttpVerb;
 
-public class StageObjectsJobSpectraS3Request extends AbstractRequest {
+public class StageObjectsJobSpectraS3Request extends BulkRequest {
 
     // Variables
-    
-    private final String bucketName;
 
     private String name;
 
@@ -34,8 +33,8 @@ public class StageObjectsJobSpectraS3Request extends AbstractRequest {
     // Constructor
     
     
-    public StageObjectsJobSpectraS3Request(final String bucketName) {
-        this.bucketName = bucketName;
+    public StageObjectsJobSpectraS3Request(final String bucketName, final Iterable objects) {
+        super(bucketName, objects);
         
         this.getQueryParams().put("operation", "start_bulk_stage");
 
@@ -62,12 +61,8 @@ public HttpVerb getVerb() {
     }
 
     @Override
-    public String getPath() {
-        return "/_rest_/bucket/" + this.bucketName;
-    }
-    
-    public String getBucketName() {
-        return this.bucketName;
+    public BulkCommand getCommand() {
+        return BulkCommand.GET;
     }
 
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
index 7f7143462..1eb1b3e47 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
@@ -16,13 +16,15 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands.spectrads3;
 
+import com.spectralogic.ds3client.commands.interfaces.BulkResponse;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
+import com.spectralogic.ds3client.models.MasterObjectList;
 
-public class StageObjectsJobSpectraS3Response extends AbstractResponse {
+public class StageObjectsJobSpectraS3Response extends BulkResponse {
     
-    public StageObjectsJobSpectraS3Response(final String checksum, final ChecksumType.Type checksumType) {
-        super(checksum, checksumType);
+    public StageObjectsJobSpectraS3Response(final MasterObjectList masterObjectListResult, final String checksum, final ChecksumType.Type checksumType) {
+        super(masterObjectListResult, checksum, checksumType);
     }
 
 }
\ No newline at end of file

From 6131765da4a35589adbd57d53471ba93de01120d Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Wed, 21 Feb 2018 16:51:08 -0700
Subject: [PATCH 148/229] code cleanup

---
 .../java/com/spectralogic/ds3client/integration/Util.java  | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index 0ed251ec1..d7c22b50a 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -95,12 +95,7 @@ public static void loadTestBook(final Ds3Client client, final String fileName, f
         final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
 
         final Ds3ClientHelpers.Job job = helpers.startWriteJob(bucketName, ImmutableList.of(obj));
-        job.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
-            @Override
-            public SeekableByteChannel buildChannel(final String key) throws IOException {
-                return FileChannel.open(filePath, StandardOpenOption.READ);
-            }
-        });
+        job.transfer(key -> FileChannel.open(filePath, StandardOpenOption.READ));
     }
 
     public static Ds3ClientHelpers.Job getLoadJob(final Ds3Client client, final String bucketName, final String resourceBaseName) throws IOException, URISyntaxException {

From e51da4fd38560aa11907cea5751f0175fc68b4b2 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Mon, 26 Feb 2018 14:47:02 -0700
Subject: [PATCH 149/229] Updating to the latest version of gradle, and
 updating the version to 4.1.0

---
 build.gradle                             |   4 ++--
 gradle/wrapper/gradle-wrapper.jar        | Bin 54783 -> 54708 bytes
 gradle/wrapper/gradle-wrapper.properties |   3 +--
 gradlew                                  |   6 +++---
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/build.gradle b/build.gradle
index 1b9e50da4..ca3acc82b 100644
--- a/build.gradle
+++ b/build.gradle
@@ -35,7 +35,7 @@ plugins {
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '4.0.1'
+    version = '4.1.0'
 }
 
 subprojects {
@@ -62,7 +62,7 @@ subprojects {
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '4.1'
+    gradleVersion = '4.5.1'
 }
 
 project(':ds3-sdk') {
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index 1454129915e967f2239442034b31dae2c3787c80..7a3265ee94c0ab25cf079ac8ccdf87f41d455d42 100644
GIT binary patch
delta 17061
zcmY(qV{qn8u)rJJwrv}mjcwbu`QK!dCr&oD&5ezXH|ECJ*tYN9w@%%2&s0rS&vZ}M
zeCV2)uKD$66=YKt-)xD?l$Tt$@3XqEIHjn6>X@JeT;%ue%*@4SvlL32Fs9kwru
zqPBCln?X%_j#y2u=-IXq;
zEbYA$)}yEFVB|V3K+Sk4-Z<)oa(AL70K1}A-8chEa
zF=TJ_R?>JcYnJHDH8dm;A(}WzI(`rh(-SJnepU9wx$@?VEE%XfVvP4B|CR#!dGurt
z3Fu$*iv0!%z=;9-zn52!%}=mhPJc=#zzPshJdAJQYd9&bNE&IHThX&wR5#(ldqp+CAAR%J8@`6wVd;0P$F
zxD1an%M;W=EOwrzc0Oq|CrGNS1Rk@$0mC5z#Zd0q;^^iMn?;FemPLF#+*42V*&&$r
zODQ`Fr@0{-Lx;to$&QEg?pc(1|DLyGb4H*pd$%`TFxqh6eX$gl{b5qe5z;WxT
zKpfH>br?n&RRgjn8=+)913#g*MEfE?|}8*`z{lI-6_*8=Xg(HxN%%Hf5K>kA@WYbDCXB>O?x2G+9e4o
z;lyR8M<*(^ddsS6BL;j00((<-vWo1ot?2RIW3BO&d^oB>X2an=MFt@vzyV9V^+3cZ
zX~|(!>99oJ@W*qs
z=RD`&SJ1Qdf2;+!mfL<^TtsNAz^aMmY{@gf0XYJTCHp+dGrkWVQF9&c_Qkx8k_P#I
z+{B%IS@=YJ;}qsb+%9T&H^1$dBqmn2hgQO6L2Tz@^#8b_6g-s!_%tpE)e!e|2YhnI
zikY@ZSTna%$2N6>tSot?E!hHd1&us}NvOLV1fdmzn`^MW+4j~$e?Y9^6y@SP{;
z)LdEiO5NcbhZWP59{kh?8`0f4H+T<_HcPL0wCbJYLc1iMR%iY6sw~oR@3x
zibUK)?CexC8b}+vlI8Fc2sjZ&O>?st!z>lJD|;*9itH3|_GSxMw1eq1W~F@e!;0+W
zAO%-Bf(@py4EoJbTt)P>N!wZbD9#4&UWciM?H<(hC;YeSFDfq#j#02f;t*-d_i?dE
zaYS)(W$;zNQ9=@}bPn5qLsuVIWY@{jlN_a+xo7iRwsvEvkms6^8+yQ3JU5<5B6B1uadk4$j
zZF&aj{;j&W6JIM=QTuVN!7A6XK96ICLW`?^hrhqTYKL&Q_Iqnlu|0yZg-nB-l9^FO
zoj_Ytmu+)sOHx)EjHN^0b&WiAE!|Kb#3?gCQ>phgG4jXj6XMB@j^?+HZ~9RjV{*=v
zj<2xiwv0(T-Szc*BzjV14{Y+cl)K7yroP3av4n_eQ);o;nSyyTezuw`IW#?iX(eLt
z`)az)rJk0KxVAM0drZfPht|oncuIh5Y<+>kU;{>F$qfR!ta;1|sq9o+zi#(1no%t?+pRk-$U1r1)7v2rZMPCtHZ2kez*hvu`z=)I%4O
zmIl_YJ60oK3Y*3fKPTs^Q5sPgq=;QgZ@iE0rdc^(JkoRSAENU^Je)~>T{p&dbuY3#;$(B?KXDyZ6Tv0tH;HPoY)q*kXvk+}GM_b9A
z8Q`LVp>>-x`dpEmmCB7yjF*+!Y-u#REV@(rZs42@&$`uEZ0z}r+4KUMa7t=O+#LzIXEC
zl2A9-Azfix3~{u#??9Zh5NF{wz6;XnqcH_)(!HZ<{N-;9Df39fj*qO^Hpex6mM{@zGKN1nJ?sr&yt4oC
zK*3UP_}7L=4C<-I)*hEQCmM~l+>t~zRMVkohW^8U*?t^-$NxI;#Ji1vTBbY~OS8mJ
zO{W>-+Qw)Q(;%UD-O)*3LR0@$Q=`?>NHU)pqbl^Y0xEZl)D!yi9VC9fymyMr~mnfyo9=rr`lM25_!Qm
z933~hyc@<{HqCEnON`K?5Pn0hC1ZDd9LERAncQjL+Z
zLCeEJ^FpPwqx`wvx`6iWruEF`gRDlle&e#MoZ_m#9Y1DAaNS!IFJYSgiAY!rngFGr
zr@JD~`805v6BYh&8UyA0B_68r=%g#SWs0dIZ)7nSG22_|1~Ua5Uh?m`~ry?w+kB3~@)Hd0Sa;}Zf&K8|ZZj{DXx{Jz$2
za690p{c0BgFL%WVqYrG*HvAmtgvMas?lXvA{vO1dqmPIM?Rt=U&fh&
z71LPXg&N{jJ3{wTPuqkyQ-_rL4p2TeMtcq4I-`M3-1_q$IjsDk2gh{`$5-;#*L2z@ruc-R&5XRsc*ScX);NnbIemTy6a&?`$$CVNF0vI=mX@Xrng^6P1%=X3bujmbr(m`rOW4H1=R
zDKCo{F0B(kz`nm8V;EmDF*K?dBs_AiUIXqYFC6$^wUd8V?Xjx*HuwA%rcRs!U6y?n
z-)@v^hb?y7hFy7p>PH<_W^))-rW0BzXM9Af3)T_(rObv?mS%pf5-QA$5jAxGpwq>^
z&?j{oV>qyo!m%tO-ke+Vm2|)PoqaQ48oh<|koXDSGntY<>_!@X8>=zCsY*5;P-7>i
zo`~qKjKk;?v-oq4OMQmBg6NjF(Kc)pk({TNC>pD)Nugnyxq=eg@z!f3)Bhxzw1fMB
z92CKZZUh4cwvPw~#`!Og#7U|iAp=sh^xZKJ@q{1zA$uB
zN%d)vxi`4(hJIa~3-9)6uBz4gCa)|jzgq0*C$|c6o6kC+n9s1HK}VYoHE8DSb}H-5
zPisF`wDwbd3FvG{KyAaEeiG~nKMdSD^w$R7FQrK#`9jwOa{Jp+gd*X&6#zTFpOkU*
z$j0*tgZ$0DVl|~}*M*tJYP7fuSDxGk8UxhGtgQoTaC=BT=N}+aqe<}=J6rPW%-;c%I(bnKgGXO@TZlz3H
zTCWGDLg|gR)cSXCD`(ZtDa0Aw2l{_1?;1yH)3W_0dl*I>q~lWM8UQWARs(yj7pTmp
ze#qci53KNW%anDOrYhK$BxYZyii_7d)PLH`7xuXnrp@;5wvil
z*K)TIf)rwDUh?j#r*dUD0w!3u45|&-#F&j3VNny6vlADe-=*^npQB?OjPyjDUnLIc
zWrTb7Q@iD`0IC+p6=11d)0?Y|dzPis2vbaCRi(w*QWmFD-g2dt{Ul*@benQvNtZ$w
z*})B@kx^%D$XNcmKuVf1QKNX5Fs)JfTNXswD9rd>mcw=3i)bjV#RHVGYEX}3WwCi}
zcjK^;8a&`}FMNk?Bq2%5DkvwfA3U;{V9JGQw86@K+WtCw1`w)^!GK%_egzS+u|+^a
zNZH%K^n^Jh3iZ5%5!b9K>At}9e7_`kA~7OY24&B61#AE5L+f$=3)^PhsOJ%SPl!yPsA4ERGkKYV4-|0}P6Ete=fFWV4lsajj
zO3b!7^LiuCUc_q;V+2QIZTFNC+f>$7j3WPDN@0`(&xsSZo-|b*G(sS$q>F9jBx}8B4`j&hZ!u$P`kmkF#nM1844`%$
z$(USru$qUF7bipIJu`QU~giuC*U0nQSr`?lP6d`
zqd!WHc#d!dd9h4tA%FrruW7^hrp2>7X6lS=O8fL)iOGb9G28@^f|+v}T%NJbDda5J
znz}}Qpenj${km0{fN0>Vgt8{nAlS{U0OvPC)q!4ZS^4xwV`(&7MBQY59_eY>6TOo-
zf~^G^+b=^T0YA;@za84m$WyMH$mHYKWZv`1w(5CT!GdUzYoZbkCL`fC{-PQpSd~Nk
z5AN&VQO<^F&Iaa%*eSUKNQ@(bAVZxb+SKCA00RKsxDab&a7-p25p9}@LZrJlc#tA-
z-aFrQW%G;ljoDM2r@6ksYPA3mer5GjtX&$7V
ztZwiL@*K&`kfdZ@^8(e82GW(gtazH%fvfthu0H7cWw8FQ7k{q}C{k?xMjCLWIFMl@
z03uKR48FK)YVasrvtG9>ExoUQF)hvN{@kOyXs{40uXIMAzB8H-wS{j$@Cy$?a&pON
z6evOj9FdiL(l535C>*|L&r;Z|TYr2;DogUZFEjI0;aj3(X1MekPfj#dF4xq~
z9A~}PhvD2tP%4dH+Y>^$H%YP@kZSEH6HP*aZ+eFFAtod}TFCJM63RWCo;BA1cvHDY
z%6+?JFQ>Pw$N|K+L+LWv`?3MS{CAd@Rukp@_;Dg-(BYx@gcQ~MU|E&2iuxzhq0W=W
zM@((7|1#|r%g)fFFHxBZ?uOLC%MUnXEh5(IHwtH#tCg;GvmU}S7d^>o33@c^PgoUs
zNGL2YFgQ3cFfcJNs0zOg0|>Cj3etQCCz~Q?C8wHXW
z7^_2ALg;enLOn8`1)+nvnao#|=Q>HMoCKv9D^Dt7*{l{1nhj1_T3P*-*rXp*O~*=)
zn~WR&@t>W7;|(6)fOlxojn~cArB|cex9Nd+@MSWdie*rzro*88-i=y&mqMQ{HZ_Jp
zNE)M080pWuT}`(~JM5o*fN;?I>4-gcmy%@4U}gX5{x+dT7rL|Bl@&HO$`dm~%Wh5L
z;o9DfS9_QEP+*&JA0)!&@kl4t(FQ~6t}w!8l^OPCnHl~RosY^8!RWq-Q1RdQpnxxu
z#3JPs(eTgQ^l7N0`=+9$Z$1)Xr14|#<_J@jK18Hi*XRKnLnBwN0CwIlOazgNzv>Jv
ze`T@rhOOV8s*9hqN%OR?pr4;5eV>wr-V!bDl1YtIV|sQ*w6OyeSKV3-dM-wYZypJQ
z0@R0EuN4
zxQhG&WR!>YHlbgt>vi-@3{7sT*LZKQWs`V#WR6DBZO=#u!(+w@t#0`6Wkci<;bD|Ym?`UNX&K_hM!4{O=?>bk
z&EJG1R^6m-fTM^WGQ(e~EN*AxI`uw2Ff8XDKIcUZf|F3^OtD!-MQSV2U4>}%sIuDo
zUptftW@9WdfLPp*j}@J{b3@vq;U@k+FTZ69P@%LIr{jjta*(P|B3!wQRKIPi;EjjX
z@;r0kEZ`PA?<&Yv=&%qcg0MH2wkA2n_{XimVU?(vfp?23ykj}-i604aYX)|umRQL<
zuCRR-u$74%b5=Ci+^h40?02ER3bUOyMz3S`zsXh-PYv!+zDq4c#Om{NWd-)o5T3GT
z(?}i|bJG^y?uQDPWNj~u?-YhOCO~i-M4xiS>}Vmk{
z0s)9bz}x{6d)7!g7C}&?G5u3BXzOq5AU}#euDBRAk%rifydiioSkBgpZ1CbHAC4Av0Dl2Ts%2GmjLTkwBZviGGULhI)Rwk{
zYy3KoH68Lnp;EmdzZrw@2Fp@rxt3e(?Rkkvx<8@)5eTB1n3iohj(d6_!Dgv^p*)H5
zLW4P52VD2#>!|V>iIQ@)K?p@AEO1E;`$8!uwLvq*g(zx|RbRWrkKEC;4~4lX>X`Kc
zK!dV8q9n6T=HyiHD<^lo0u%1?VgYSZDK-K7Q5+N|IaciH@g1Kqp@3QG*V+#9O`-xy
zdQSAHn?%9tvRIXkRS(1QYOZs^fQ1OXOdWD9xkz%w>cpK8)6T>!btU@gn^`+ocJVcP
zkVEp1XpGDY3=Q%mqJm_CpQ*Kx+-5a2z+$98@c8e6bMAuT4TrDCu9O6V7@SgH>-YGd
zTImXnFp=Bh&R2@M+1!Y&Kpu_4T<4hDM$ud?s_2TDI;@$xaUl)^nQc(5jAq}fqZW*#XcU+RYic5Qp1kq{w{&f{wa0`BTJeb>>u
zSWO{1x(gyomwO_XC?S@!7)NAie|`zeYOj@hi2JZMF0cDAX?TWly>(-sM!^j`_gNXX
zvK5OKOXx@a`lashsj#pH_|A_mAf}ro?0ERnSDZRm_viK#AF6g$p^#
zP%WP*XczW*%nigUG85G{m7Tbi=$=!bmY`0>3wcOmap)G?q&N+Ak;^>F;d_i@I_B#u
z+tUa$qILlP+~3%pG+n#~zxKsrqHc?1xITuYtQTiLNl@EHe*m1KZVyvn*C$Mx4aIN9
z2ih$Gw#HE03qsdisqG~i0Nuc_cDN&*h*;a`t6loCl%z~IZK?;{{kgtv181(
z(X)mR-yR;FeZksvk~GZj+dk(BocHaBWnVd|cnR}4yy*9IAK1H%L2>^)3|pe>G+z|t
zG0Wt&8paQQfkYP935#8Z`4vtEs>>U=;$EK=$A28~^-p<)^nFDDE>I878D+`$AVH`1
zp`qlk_W8xG5IbUo`)EsQgl!PKE`Nxij`$|FI3(D=GGRo|V`(cb?dh_-b>EWf##v7d@oA0OQ+G{D<_8SDAv2S6+X7QF*6Oi@Tgh$#lh~
z&H-V(9Y#yRY#-{Yx(+wdowCV26iaSDfty~|PzQdeH~BA5=8dkn)WumtUr3j`941)aR94$#ELfOGnQ=zNM892myOK!=CZ$xU(
zDK#2#4Y}sC9Y;Uf<~4S9N-x^uj_>un+rv5m5klT$+N48m=%0pU-hAOisVFx4LR&d2
zndZC@cO
zuUma+?4D1JHno!o??7jf|BWJ;1o14d~o<4d~*C2Dqvhz}0(gHyr
zlttq*q&sGV++sLyTbiJcb(C#m)+RR^^z*NUrlk(pa&X#KEyUG1p8b&%u
zJTixf8Ld`X4r*>0$TZw)~lf>Bc?rhBbLsZ1PWn
z5;c&-x*iu!0T9?NR)%o`YZ-%r6p9PB*6R6!(T2jOW$a$^G56gFK%n(#Aoc2z-SGv=D{fnq<;=jP;loe!6)8WQH-;i)YzOFOg^Cn~F>Zx38b5vo
zz_{Ftt?JdZEG}Bx2cB02{&YU}|LIJB30OHfIVFny+Dmh`>e;XUbnm$h
z^u2dh7M7Pld@e$}cl|;jnVHY!3(SSy%M+RlYy=aLlC3yB|0YxrpWunzY@u}8?mC_EP89W1*MI;KxO;0iko
zw?TpbblHS^+;j8gSH1`U%Wp+7
zBSzv$a)IzA*+!AU!&{gz{jW`+5m?_MAzoQ^{GG`z$!$a|rxaOz@us-eu<{I64Ilpq
zy-;UPW2`;m8`pZS6ogZ9z;2%>Y)!l=b}fVmz5#s?C83owzH=F?h6Vmc$yZ7;J-z=L
z>2H4aW2OWSX2geKyjplGBgj)JXa;Z~Ar2LZ^(Zzec(l+u7%=4^7EK$Re~Fa~#}KT~(-
zT3uyUwpjH)s6+VL=4(9L(rFZQ!hjUjXvdB0kWc
zbb1Cc2V)lLr&42oK&*%KJ!oJZOk(&rjmj{T7|rgUa`T7(eo=hWarPpR)z|Whm`CIkBG}!aC-guT2C0JZAsj2>P=fuYS~MeCT10$|
zTRlsK*gUCCGI^vbHK!P6(pPMko3DRc4R8p)G&nYsCxC%)V&#lh4=uC|8^^
zHfqP;(C0%0FgLLA;rNtHm`D@7_>4Rvnr59hDff<0TS0B<$Xh>U5OIPB2*r!_t`V+Wt{fY1~@v
z3zz6vbbb&|;EJ@DZgZnIG1cZ&#$;ZNC^dh_HriBd%6kW)(W2wpZ8|hzq4q6Cg+S4<
zu};AXU<(PtC6u!GULd`mo!&$lnVbV&IN;~mA9dVvu5cxtFz}_U6Zx5My!^B#)M81K
zp7TVlKUXSgMCk{*In>mWWr(vR4AfNAFrSWIg%h5rzua#{D0>X{8T`Yb1!^;P;W#?=
zdJ5B;NuNIoPVBqOY5C1Ri#-^z6sWR1bu)0;fkM!p^7w#SdO4N)R1moDHMYZLh*>JQ
zjq>IJ{u4TRa#obF)z@qO-1_A62m?wd{Xl$UKCj_9yS-u<`~)g!X$!Y+P*Z3Lp1SXe
zhoQ;zS%IoJ^&lSxD+`R~Y4mYYc8gXO7DBlIN%QtHa)AQj*^RW6@9C9tLhebe8Z3(@
z0CV1V8er(z8p<0D6B>d(bJ;AL$q1i?tZwa?z%JvTG>sV
zjYXCb3pYuNS~#tI$Lo?^nYr53^BR7+fRV~EA{C{$$DgYsn6!k`0fV{!Y!F)ZX$N&o1e0*3pqP&7NFmm1o8P$d*HM
zMsDWE
zl*_+@33^@jFkzQ!&slHGDVGP`3|j+_a_)I!kI^uwJmyj~`kaOV
zwM9Qt4e4xdqmyo%W%_08u0v}I-kLl)Ymcu*6dLPRXsEI@ZIO?2v>P$?U>n-3?#zP#oGCJ!D+DI%}qeZBCM2-^-T{fgq{apwF=33lerxdj#VKQ=?7J5MTl
z3&ljq9Nzz&$`1#r$Lh|Xg?DWh6#SLpfuW!|oW+o1JlNRKhYL^Ymy17gQ|_tz5i
zIVqdulenXnWYHrRQ-X+M$*Z*FG33$WgpAR4>WNlNdl0KUf;@rn
zHO?>BlKBfaVCRcZWX`?^e^Ig1+1tds+)%yeeI`j0rnp-SsqUsYJ?9GAX^X1}k_Vsd
z3jbhCL62j#ztBDXU82GTb<%+)TLCHyp?OS9C**L6Oy8)g255OSVO3bRbJln*RY5|%
zh5VY!E$K|$5=``^PSPUpb4fgNyYb8-{$IMG-&5!Y{Is~sj0-o6-yN3?^mSvU{N`0F
zEH`4%wwE>fQV@bnafHryyRf_E9Xup)q)|44>6+}Os$Ee*O~MPN)B|riIc{qO%%OQPpZ8@a^TBe05>fQzRjgGscv4
zR_U(?M%xhuZi|2LA4tjGSRV9CHH^G?RE&A7!e>|vp-ruWB`z@YYGmngd~W|H(AnPm
zRw{vtL5mP;O0wcE9_-z7t`2ZYOBTf!gus;FzL*)yzL*tm*}Mtk;hR5amu4yFh)U7ueQKF5
z%i!&26&zUnLxy)P)nY}@F3UnP4SW@9;qw6lJ~kP3z488F7bi*IeK72Y`Gv#sqghqA
z)UP8PDz)FLQ@FlYl08F6PjR!f#$+=d15zFE82fYhcgP+18ef}2oq_`cb@ZiL=ZI$<
zVD7q-F8Zt6xsrM!TfqVmtKM~dc^Ch7otzBB9wD0Ut%lf!(|39OC~Zweor$s9;`l6%
zyz{7|X`RVyOtaDgxxd;TZ~z6L7ezoV+~EsC!CrNsv}ol~*$VfJo6;$mEq9bNpj*d=
zha%)p%A77CJrLagcZU&mo}SxxyDKR3v^V@_U~CZj-Qy&+j>7P^IZl8+ak%!%PehLp3{H9=mBeN5#q>dxC4yYMU
zu%>kLeiS+%cC9T$%Kcg2rsRJ+d!?t7v!3u{76zjvH>*1l^Mg}U$Tw_?e;BBSPA+yf
z42@_{P38n_jR+1Fx_Fzu)pBmFB(*!IvH;FkPFZA1g36fH$~^um&t4
z5@a`jk!>R+lGi3mKLx)|JBXxcMC=m&t}+h0_z=yn!SkdnxJhaH7es$^!Hr}kV-;Th
z2?0=&AJmuAg!fISt;8f@C&FlR?nHWICv|asEOqhAGHd0+o`{q@l(PJLxb2F|#FS^Y
z8#JDgV}mFYAqWax%pAy$&HpNi~pOty;vdtjawR1Kb%?W*p&8fi?208V{a90)#7!
z{C99#nKBiAVR`AgAOe?~MBF-PtjZ>$&1b&%x%aq&t_s
zetn8%Zz64b1)t_*Cro=$-@j`t=o!_yeVZ~GpZxBjWd9ex((xXm=g0lVN?1`A26LGO;4F~WJmLvLw
zkV?JDq;Hr&a-y}OndXTCJ1iw+l4W6_FEwbj0oqp*qaS+m)uwXXi_z$IV+k<}ymQXs
zU`?7ctuG|{$_=dpm!j~IGXD3i>mE9;1FkCis=vv*ziItzJ#Z}a4Ig#ZOd=5Z;OEzg
zR`Py-v_oymo}bbRmy&}*5p>I_0;mI<@jLIXIxlEu=4dWL5M({5f8SB+Ft)rPvxOsu
zagV8W-t8PjS?3x5wuM{GR(&%=TW9i%H40C&x1oKH*!-48I=oIguRP=Z+t+_S_+bQp
z8c-1R6cWwY><2Cl<3P@HYbZ_bdmN12u1-T~zk>Yw;s!jf&5Iw^_0@gF{6!_*Nb(In
z!MB1HjZr6KPrjolM3pC{?`ErbnxZ3C`9F_7*!bh<-Ke6VT!k?GGo
zsou10(6Mj}Qok9pTXjqKhA!`-lO)ar>bMh3d@@S!?c=*Mml+Elj3rck+|@3@C}BYH
zBCCa;dFQY+bWjOrCFUx28T7LaE8$+u3YN=oRs4v@^EfW!Q+Zh=qXxUaSVT96^f(67
zr332^ItJuGkdPLL>*}vI*@$l%I{XcnN0a>?X&x{+tu9x;#cZ003DftfJlA|dKTu0p
zI#9T0J&Rg?d$YfgD28sD+K8YEXnh%PJQBBeT15S=f7b338$BrU;B(n
z7Z?)oq8um5`-|kZZX-yhH`uY&JnNlV3-(JZtO@Xe*AiC4fzuuNLEas<4wVV7mEq~)LtW(p9t6}CdNQNQ`;UofO3Wj}fPpT9E@@o>o5e+p~UifIAi0{$@
zI+fSSy6uUDU9wddP-$E^pR^vtoqR54(VqyA?a}sg$7Fx8C^oA$Tu?2V5Zd&Dr#@#s
z{bsL8`D6t?r2+XYU*T#>Sz0*GB8~^*JGtV$RMtME5;D5>S3fi_ug7~bV&e9n%1Hf6
zXU9`S&OCV+Q=JX#@YOCTuU#>Hn@#biNMoj|%;E0Gw@NTmV*T!b({4pM#?vQwV~|Wc
zDZ;kzFq;At7}y~~l9-JIpiQWW^`XRJYLf02S&UtP{bM1tco4CqS#~yAjSv}E5!%-N
zj0(Sh#)4)_f$a@`%W&M=cr8uUKwdlVo%jRAGf!3qlV&cNS~uJMAvf^(Vbl6DL-h0G
z92M-chPHNYvz#4KW1ZJVnss^n_IQoCeov?L9xWfIYUa3+flJuE7neP
zR(}1gvVasrojVB$*tNWuII?N9vIkAEp+{PwCW}+6s@}O#SZ8G#(ag%m9~u79YKqb`
zOTW0DV93-P-%**Jd2Djqp!&W2+qRB)rUh%B!axWelKXLj?a|IRkw>p~D$$98(zbqC
z0epZMm0-!DLY#3$L26@>A=0xXH72*P!j1cT)GrQ=;$q}+YtC|P6hg==e8@Pw%UT(g%-qEF9Y~9(|7tS@R
z8d-3XLZv4MO%1inPM(0zdBDRlkDpTA<15Ubof8hqLSasfpDt3#=ShQ6cZvY65%0gN
z)CuR0KfzB`pt@JdXs^YhD^eCzKkxMqjK#-qOh>PAIAaTljvsOAB1RlJr-dWT)W(UfD!gi
zknterEmJb-)?Inx^*Y8e(PfuCBNQuiqeZ(qZNWbPJdOIKJYt_h;c%Y?6tfOc7XRT^
zPowsEpHpWEWM6H22@u~|Kfja1
z&as2=d1{Hpt)9^BJ=%UegnyS>f6vFac$_l6zrZF>Hd8R!qsVqk{4(lp4(%TrOo|L
z-SFE{4zd0TQGpLmSg@!?ZTwa~t0(mjmQ&~JSnvOfsw8bq5HvV?VIl0Lag_)_fPpPS
zC4CE41ys?5y`Ob-aHE?fYbt86nT@^Vjbnn*QU)zB%@l23f`Sn@tyUbp(uR9?fWJd&
znNKDG57PLDEQB%8LxX4+6FI`}R#RtZkEF&u5H^1Yzr`g6#|WtT6vmYaplG&L7gZLO
z7ES$dlGO$|f+X7Arhc6Hi1*darqucf=G=t=!0;=IB!Z#yuN?OadC&OoqgUNXcn<)r>m#~9Iw#CTkwmp|_M;4b4#iey1rS(E{T)fLG
zdv=4!-sT9#XugX?I4umDw)^+}Y}lO#7XEH$H!hWTEguUrT*-0w_A)h=
zv5titewz!V5xi>M{X1&ylqnYdznC$l>lBCv%zkebbJRyN6LXEFeZF!4=6Q%_%X%dd
z+a~W#cs#p*_-@bYPoKa3v0gT+r?I1N01&wbJ_P(SK}w|9mb2EWp``~Byh%p37b_*vEBhAi0{<;$(faQ^G&}?;Deg#>(ieNSG=B}VbVc1^KM(9^t(+vdn%T5$b2aJx)m0Kl^@Pfx_n>=3
ziJS1t_m*FX3RQ$@x~{jRKkj1gPm=__8O|WRb60KSiA68fy!;H+3%0ZeqPp>oqnX{9
zm)29YdAD@Co_7tmaog;n@sa+KUCi)5@fRyw@McsPR--$$V7CJ!q
zp<*_~;fQu4cncF>dhb~Za6$D`Pc4f4VHX%&9~uu|o6~w8e;wOBT5Rg%+jMqN+d)U)
z;Sd%h6BK&q!v)6^x$5q43D~y($OeSNuVUfADeum9zj}qK7N($mV_%0g7PnKicuUEq
zJ~qe^6yJf^q4kQ}^S(3ux|(x|;TVA{M@QxL6)PHp8y*Ga4DWnWC*D>FT1Xl9c8VXdO%+%94!fRwu3
z8?a}QqeNbbdTN+=jpmYLi!ptbeJ8{k$NI%Fudz+~kn0aqEQRiV1dCcVmTc5Q1W}+{
zluz)wbE9&~1r!3_yW<}4FS0eElUPD&|AjS7l1C;vaIBYqQ)JQ{p;J)uTkQ)RJ$WJx
zjUoYi-;X0lx+D&ZPI<4^K>%85Q)oFFSB8r{_q`~_xdl6=kekG7GyZwgf2+&4(YQD8
z4Jkj=6A^wwzwXYA!r81ZBv`Urh^M}|Rd$soRWTV4rX{(m-w~wKQ#Wm*(Ir00rlwvS
zuxUpE$b6RYx%}qTyFKY&!R?vzTCxq5W-yVt%ncoLfBb6P9vihJgx&)uoJX$fX?Uzn
zgbtPHlEppkyo_v2BrsPiI7Z{3-WJ}mjJORuH@%TvF=4Vu867osXg{)}u^i_@=mo2L
zagSArwJW(Iml=rq*UmpI6OG6>NQUp>CA{7ZsF}IwtFI2X5SOhs
zn;gUhl?C!^Nsv`Jk!^elJJEPR>3bJ}GjyeigfGTdTk_>}csrbUORb(~0$pcy46&^N
zpENcsM8`b+MMotwE|YiBeH_Bv(FZ#?!U~WEys6cu4GxK191*MrAplvJdFUee`5t|-
zjItM4TdeERcHx7^_bp1PAd@nm;Ph*p(rXkc2mfnr`;FhZeVJ@i=!fTL?OM}lysLR8
zU1n(!xbVU=Jq$vM0K%0j(k;8iGv5L51a%XDK`j!dnHvdL;~C`zZ|jJmjy_7fP9BTd
zu4&{I3GP@z3BBIQ;)tbDy+0YyxqN
znC4G(yf#)O%P5eJ|kwNs(7#OS(b^uu#
z8Ix_#NKTH+0M>j!H@rL}#dJ1j^2W6{dkbQM(CmWjU>%uK44
z0m;fBnsW>cZYVAbZUigbeLjHcV>6hkf5DwexChLfb-|5Mbn?Rs0Wv5xIEn!rlfX*z
zFFG=cP2PAhl}UIyRDgZ5<0TmxLn8|yRm~6CmKORs@CYnq!%h{c-RQ`Pkaza09g=5I33mo
z>6Mg@*3@W2ga77o^ua8TJnD%+=AuGo<6r&D5OipT%VK)#h2^e{X6e{Q`HW_V^z5^E
ziqGv0(TXJrH<;5Y-mi<#@8|Dp{@Yha-*Z+FcR+X~BucBX5fA
z27qWWC%!bM2IochL!bVOpa%Qpr;q$dROYKB_1T5X@deJ8G|v4@{VV3$M}^&&JjXXf
zhU4h5V4?7L*+CCn<{;qWr8wq$?uH)h{Wc-S@t(GID+sAxQ=Bl|224_12vb-zD5w}#
zAC=u@d{(@QK@Q$=K`e{-T>K9M4z+sa96(G9RlZ%TBGN*>@zf+Mz`4!<6ItIb&V_Rx
z#W|9JoHpkK!P(&kJuV7>cx01*0f
zm>>c^8+&&#Mj}9IJS(k^+Wa8yV#;;KLxaTpx8YsotLMfAflA~%I9rTlC3gk|TviK%
z&KC7V+F8u2!F<|YecuuEhAgGNWUlk}FH1h&ECQKg-;MZwURr3uLjxC;>Le_B(vr?u
zUunuSLZnk${h`XFGMSoq^2
zimkr`V~JH{HX0PPmueBWiC0hJ?1d++VY|_JEl>6*h{4
zqVYn6_$mQlbJsRDFQP1@7Yi#tJ+g{hbLQIZPX`Ga
z;?danhzE$4^}}*^)@~u$xm8qbfWW~*a5KB))eo2>i)F@!2+#dhRTs*_QW=-7wo$>;
z#Clt6+jWBH1kj8LzK(`XZ@{LX3TjnjFu~u3`LVJ)tE)<^%p{I6CT!IqW-vN^u2Z=V
zIs~A``gVhNZ<@+WcUKdr1C6=uLDbbrYNzo=j@wxqJh^K&e^!3;-%Xq|d0k{otESCa
zT?1GJxok>cR9$Ardl09QXIWvP4MQGHf3Z+&OAu4x{^)c&3WEB32w)M^bD7DogoS#d
zWhBy}waSGp`C7YeN*P@)r({sJdZa2SC0w-^O>09(Igt2-!j6*I*1_H(uExVQIXU7N
zF=rNSD!W6wYyCP{@@
z{O^Wt!5wzS3;#wvQ~L#&!+P;EwN;u^$
z1uEg-Lh&m7UyE02Gw~c5ZC>Sc{mJrq80B^{^~?Av1NlwU2ygM&aKb)T3WMHA*C4hY
zRZj!dcRA;rRmTXheLqz^w6641R`DLL!tfueco>p$&T2SM4gk;IB?@!=80d+#ACC0*a0}|5G3uLB4S&S&6rJiIvLF+TK5IA;zZ{ib0TBW3DjO=mJn8k?1Z28AV~61u5iq9ugPCe4USpE?k5>BJ`|m7ci88}V(|l+6Ciev%(`Z1K
z6jc&q7+txBY^iOAa(TDMvWUWPY}H8^4H?t2#oDkGb3haIbR?|jVPd@_t;^zx%oH`J
zrK*9tb>E!WINGm3UZ|KHo=s{0HWNQ={eY%25p);SKSSLyN%LodQU$K?5w?qluc%i#4;^+Nt?2N&v+i&^obPP059dEqoBX2@b0lW
z8$dwa=2SR?s(pXGlY}TgGc%`)HR#M$>=Sx=1+dD;g+xDCVM(6`SiWhFMswIfrVa*;+6g>^|@g)QS@grMadu|C`g)nflPm
zYR=g4^gv_tnw`gfwKHTIw2ZEhj0I~<0gy+Zj@ql2pNn)*#~QljFZvmK7rySi9Oe-SbKrObJC;#ENB@tRG3P~PyH#dIQ_7WKK-a%!?n
zcm3l6ba_0j@0&1VQ#7o)S>>!A>X@;cMV!}g1)dL3KQ3El$hUeZu#`u|-_`K(1?XT*
zAt3*4oVH6l8nJSC90B{#0cU40_4GZk8fb!D(ZBR<1d-i9>mgHA=^*2}%+sRlzde7vYd%4+@b
zn6AeT7M)Ms7}k@+Z?e{g&0@?#<
z&A7+10TDBdEatzH-t0a%*9LVPh;m;_QfY!4tnj!qV4xd=v=e!)+7nWt91(Obt*%~9
znWRpO9($zDH-`S2(J^HoS!!e36LQ`>Z41>X4!VM5chMSxgxN^mm62Ds1(31dpmO9r
zcjjF-`AK3mL~*Isd^s&mw_e|7HDS>-MYD9DNMBD`1>=;=Sk$Q=nv~SLUh9X?&Z!Il
zlhC8ff8v=W*F8rE{Pdj-r*Ng+vWQ`<(cN|K3}587)S!I}iJ?2MFBRNc-mVJ^adG~w
zvZ;T0t+AiaoxHDsa|6Cz2~cC(h4J&x6dd*QXEnVP)K=1P%ha+e{NmX@^a{LMnM)l?
z&z^-3I^di|E7@;~v?2%(s<%3WZfueSjntA8#k3XLNvvhi!cK%FF1!`Cv30kLU>7
z+xCf{RAs|x6`j97wefKNLLr(tbNLj$EpVj|?Mg|$08;S?5uNaA9Qu`_8u$Yggah~J
zB)kISg~&#i_B`CYU&Js9$gn&iC~FefDYO-n>Lft(Q9KoQA%_q4pwqTW42$D=Zn=hg
zm^Io*)W{r4gAqB^HOm8t%o`55(3`2UA?zMMb6y_t`f6zeHr*Hy)z1&Z{>OjEr85p5!L<2!+e
z7dN_G345#Peqa|jPF&OOg$EQ4<{FNZl9^|IwRMz{tXXYghU5a%493;s_>kA;V%w6c
z+cWbYKZ+j~$m_P%*(_2-NEOW+4^9^I%vp%50p(hnV>^OJQ*R{ZGGtk>f}Cc>O4$K|3oii
zcf^!cnAiG-Sn$jK5$GRyE8x4%CbbhB23{Z7&`SP-JZeGdw_e6O(*z=a2d&GbVtdQm
z(~t*>j+V1B%B(WhV-{xew@w>`CFZE53kGYuF_-qU+2Y~2y#n;5`dI&U(C3~%e*
zhQ@i4c~bycFg_buxX7pD-VXe=n?~y{I4ZfMwH5QW;HYjtl#x!WJ!4dzz|IWo@Z3`I
zn)G?O7~-TbVbZJvJhm&8>PoCS?(7P9Xg?F!FQkrUXmzI>k2Y*}c%pvZb5xl{W(%fd
z-B(4BHfZ`*XaN154x8#1N;fnQTdzjzT(p-jP?G_+Q_?v?Kn)KfrsSCY+OEn3+;I2k
zUiW(Q8G241!**c~NA8$At^{rL$F{YJdORxAJDsT~OzeGY5tq)pz}~-7vqf8Xok#8S
zWuXXILxD1hdeJfA+L0pS7E9I^JCeJ|R+?WVq#Ma8Gphpeo-iskj&0~QI75WtS9-T@
zep>+iDQo&h+G-E+1czv^Ml~F61>Y?4gr;5!OQ|Gk@9Cz%5XH~%
zITZ)f{8&Jxinsy`UfLBmcH-zD^oBE<;rcwbhK=r$PG*wlNX3LDZZjNSam9B)>-`gn_;{+^UeK>Lm?-79g!=+;Poj_s6xX8{3YXu&!-xy4ki_WMr
zW~+DQHlnxSB4;-|sccRg^U`gHgpH!4JREJ$Nv|{#bZ4=Da;hNWX|2XxHTYZDX$#IrNDPpA!n<#1r$OPMftK
z2k>1reB`WNix#`VMlQu6EpQ)rj@{)rYaI6@?n;FxUfKwdtX&r#NfrFwY$3>a+(M}bTy4S&@L^KA3nk7`{snl+7
zv7%)(@zCu#=mkz!?8!j0#R`}M@6}s*%)Dr?X@lF0HMq+>%<(H32bhF65BdGXm6;KD
zJ3Y>LHa$dYMXw=U$d%i*XjK!=WqCs4@fglAqw?><%3JY5MOxR6h2X0C7E8dyUlCe6
z-|4sfDjyVg0X)2cn)g`4fkWG1!OkuTCM*fEPBd=8sqA=EG
zL`w(>r#=OVYg^OpKtrSn0RA?bY&Kck2I=K8rCB
zyGV!B``#diZPmtDldQy_zt9dZYWF-H)8py($E4+Te{FcMrAU#dd(Z(=eFd$|6*Q$=7+L&F2boMj#gh}`!YS0|4B{C{I_g4vdlYV9*
z?#X=;D#?O@qk%NU5Y~Xgw}cyAO=KHg7{6!0$HA(&3dK5zBP;deE<5p4*>axL1n&Dc=1&Yf7q($b-y>Sj#D
zJrqZG#BF?+BZ-8ur8fu4Z2-AGZuHoL0UCk8*zy-FfI#&=s)tA)CAyEy5C>n`{=!QX
zVNMcT&i%nXOgE8W4;5lgLa1RvRQBD$M8cfr$4hx&jkNCHLoS5a&_+RcX_KgtvHoYAf6WwLUSmYHRsX50evU%Q
z0+(x0-(@W}^4jhRxtI`1;G5JWRF2&F!GQz|pmSoIz{!2fi%qz|$N39}u8ddBs-9k7
zF($z#BhQwkY99ue2_8$68v4oDan~|0#RCt38+mq_J72W)UPi)-<
zAmMTvW{@^+DvQbeR8(a}#|M_;&eI85cIm*Xz#D0Ld|ImZL{dZLx)xek;Y5sI5+$|H
z65g{6``v3tclH#cifp=`N#koqw(}FT;@m>g^hp6y!)=fYr)E;LW+wKCL^DGVJIM{g
zDzH<7t2f9v_~$#%yCNmsr<#WG{)?#`K=(CE9vf4=eIA|Fj7*h7;;}KRv%;fa*y%5r
zx-Rvf^u8t?
zb54&t?`gw=>)BkoFW5c-lwB|Kf-P!a1HNU~0)E7awPG%qAtA(Y*Sei%Co2jE@U6~M
zh`TVj$u(R!dn5ZH(^td(0og-7?HrJQa8@duWB)2xog1>V_l)!9FG%wO3=uqFlq*kz
zZPHbYcDp#h`HJYVe&PMfBM#teYd}qXL2WcW0)ods(qH{3wd86Or
z(15D*FH6oKU|Y$vOQ()ZxDpY6VN)iHV9#U9?^j$BgLbgQZcE-%s79xnB1bN#N^-@S
z81k{{BpGK@W}rb+E(;eY$ql2CSBo5q(8>?TsY_JIIOv|=VSSxXQP7$Lw8rx%+JK|i
z$)RFO`}#^+7gHohqJ#3TKZx}=ijYEXAzP9JkudWo;gnd4N=})Ls&fubBlD!Q9pph}
zY$EfU2idF?*<@H0SzpJSN({~q%GAloz*(C#p<%*paC(}~w~aUWG)m}A&OW6Q(PHa5s3U?+|v{W6z!=uTXW{UXtO~b$FMu{6D33TU5jgQ=iiWcxIoRpL{
zxg~bbJ=r4g7%HRaA3eRf#fqvlv>!p;UR&ARr-CJ+X(l>6hQ1bZ
zuLB_wlV5$^IaSFiMsjG5cY`f^`{C+;X)4}rb2BrJ{WWi3t0d4z2-TU#iam3ok}meB
zt#U9he!$tVIgymPr?tqnuXpC+nCFSC@<-3za^0r4Eq2reMAXSjR4wfJ;6>wZQVHB8!X`RkY*qOi2s!NVn)pYNkQW#bEb!C&54ZX=Pn_ymlhG8_W?O|Qr06Cm+
zEdAQw&;^_*T!7XEm`&F15Deco5)|}coFwqlHx_*nA$uGh@EmvqOEMvN&U!$&)cvgT
z4ZQ@fAI@|FJfg!LJsP;4U@!Vk&o(zs-|NZHW>I&QydLrkUayU7dfu3lkua-rgod>J
za&;%V>(CWOw{%+FD(1~9*}wu`WtSU<{cgsr&LO18F>pNm4W1rJ
zsk~GP07n;zEK!nA9Kxm>%cU3v7EOKS0uIOmT^cIe%m&C9Y&8DZ44@?={**1Q4YV~n
zBGZs3Vhdgx5dH>VmfPK@t|TvU!>-J!^F{>KTbY}>u(qb(Qjf)9GlIaWL&vSfm0wjX
z(f=d=Fol5?l-+OM3cum>GyRstZUkE|{FhY|z?bvyOQ?tB)h>U(>L5
z5@DRFmJ~PQuEbzi6CuUE)a(i8O{_G-(4iC5NPLipb`_q$$Zknvl}~2T*cIF(PXI7#
zgjzR7h_mO_>(>AtaSg;rRS{*067MEbX@<}z+37Y8=3**Xy0a8FI@r2$FhaXoiTxr9
zAQaspAEmY@W=*2(j{Fj)b8)IdrQX){D|>39dOGuxrs)&3zLk6;e`%XO;oZPMTRj@W9T*L7L{FgagPpXUb{^q@5Xg%i`
zmUJ+BY`Y#zS#l<)qHE<#EQ8fq!SD(Sz)jSzYL+V0*0)OfiiGaA%_TE(bCAy1g7xny(|9IM+0_Icie38HyMhlhFiS;&pY+Y^EWn8t-&4Qa~!LQTwlilRH}x@
zGpEGWvBj;WB`JA#PDmcs)rBr&9MhV?)!Bi9>S)W1BlvTex9O{<88441@0s{_08Qg;
z-~5>U6QdJ4uqR{3`_gbEd+LGmMO|cSLJPzuRjML2!ZvL>`oTWcpgU06(Rlr@asOZQ
zdKp|?s=WB*ZQ%{`Xq%z?8=rnncq3F^(W^{yEO{kJ+|rG&n2Z>1|T
zTF!UYQ=mYO*tgta)@36VakRM{&377{mbD@7*jB$j!?QiZ^Bpx@oHaJNoDq{?ti$-8
z%3l}cP8*bqp>MDTe0n%2cu0G~bbM40kV|S15TgI8SUGlNfZ9=$ASG)WZ<-zvxCquV
za<~~HFAfsCblfUZTrC#TEyXXOUK;s!n<&bK2BC(|mBg0)5WbBydJ=eag#A`ivqnW#
zQ&aQuQ`1vdchU7f>-=u}ZNVbV07}v0^StfIees=n?PF;0`VO2yngjh}68gLZO2iK{
z%4f3$@Ua12=S%On8HCLF4ECT2_FJ;ymHc7N?DE-ye`F5=i>P64&=?SAtN7H@bNd%S
z+0sfTKCoeC9_}kaW{%_fOCYRpW}t7*_%yv_3b{r1Jb$E59f0YLoBt?3aCxnaD^^J_
z_o+U{2|juE4Cse7AI=AvIvASo8Jv+lYLgq}R~-gS!zeMYym_|qX&&CLb+YuH+_e4Z
zT%zuz5t`_QHGQ0ClUkrc(|`@W|LBzPN=Fkg+?#o6Tbr>hDj*j+x#O{3{`msBo$F-l
zzkK5%=$LpyVOFo!e6fr%y^#H8
zwG%+nzuCDXXTWdybCdec{m7XUH-D+fZ<~513<_193nN!S3q_tU$NPoFS}qv%$NE_G
zycP>{*$qRmH`pTSuY*lW*f;UH39t|CZV`c+xE9(WN=+&sbX4Xt`MOkxWywY{5k)2l
zTpZ06dzZkqw0*+4fRVhTKBNfrX}gN?9wCq}rfaj~`o
zY6gzVL4T2CL;D4hNh3c=;z=vLRu2Eq$l^0!{CKne)uI{>5-)wp&}^y7C6jb|-Dm)A
z@bCcx2=@*Qsf8A88Uw>8c$a`S5H(W2Ic-dr5b;6|H9b0YJ0Yt1;#dOy&whYMD8^c4
zT8K{xPnSl^kC`c_Bf9QzF(TjGfkk#%MnI^1S_F%McbqQ0cj-4-^5&R$+3jY2}~
zF{DyBxJLN*{i#Fo5M|@tCUZrvHu5xlNE)(@mgSN~_tI&+<0=Z*oVS^_$lgpJJ=Ud^
zgwZx#z)50Ep~^7^f3`zcY&iS}dmk{1<292?#H5&&Kkql0{J?pQHY^=z7(75CCA-$l
zyo9TZYM_968Q3VKxss1%p4zvPeR+vidSa&=Y-M{L`phnCIu52P30hocTBu`Q0~(F3
z->!}c>$ogm&>*;>fxGW)Wc5Y$Ib8nl(>1i#Z9j{OUp#lvI*H36SE{;O+3QhQyjJ^|
z$%1nC$#1
zJR#qgZV+v`&8&HTJjG8YXqNmy{;8rwe~1ujr{0xrJuMz*hilL2C>Co>G$I6Cv2h(@
zqqROa)n`gCj+8E@aIL6A%&i}cxNu{&AwvN*3!1mG$p|(l`Gfqxy9$^cz;&g+JRA%{
zkD;uBL$Y74IIvJ_pz{vo48()v;lCOk(U#D6947Cmmcc;|MR{UNXLApXIwf`LWmWDG(rtwqX`ai$Hh&Wj|aYVG@Zv&<$S2Uu3SE
zXT&#`KQ`W3OE57@>saoCd9C{D
z&@Gwnt=_H3)@pe4^VI$lbRd$G%rqH!iFGagEaBcRSY$0&#L7V0#DEHK1vO7=6DIGY
zk&P5%-sCgY!lWTu7PJC^z1V#_4js{rBr^0tz-~VNYc_DQ^UG!tYv5z0$uPnpiSY2#
zq!x?@0Em2Mwim2MqTYtC4GYcBQ2dOIDL
z0hfh?hbB-(rh#G$CZT^K*7T*+(kzSEbm2v3W<5u+Rx8>SvZSw%(g5qiic{at4jWC7
zexlX?ViD=vpmB9k#WYs5-$Q3}cY8#TnM(xhK?MHL6V
zv<9)B`Xh-DgH0g+j@VCaZ$!;c4-b&~vsN9kWQu2D@s0d<(jPzLQ4Wh`rz!AJSCOy$@I9u=Bs~jOqinSQO#0BiG
ztQr+YGTc4~kL4U*Gylir#3tS47hutRMyevK(dF-Bs-kR``MDk<1xI}yraM{ut*M4Q
ze-ID~-+5%uu3E|IjZ)8Sw-@fw{?i1177#Ge*LexfNetAR$DBtdFz!2SRyUWQrCLK>
zrlSVT?)5HWDunOS(*to6!bw?yzOt!~Lo0BG$!JL&ZTnN92Qn#3_9Fo|02JPdGZ!!G
z&(+to_oVTIi*t@G$Dsw6C_?RRQEJjTtgAXFfd{vB>otMFsJw3bN^iYKY8Rl~Uvkvp
zBc{Bc47jFIQ60W-=2Cs(%4+kMhf-IWg;}RHwu*zK(y}QYIBn8T1uXuA{HrAOGv61)
zQ#ab~39_7%bEJmIZJP7%0Jx2Tn2}ral5W1X6l9@|6#{cp2EDszG0}FTKTtYY(|n(U
z&-zBnAX(>4T6`fY4Et2tf&nz
zlc##WUh{dgDpq%E0k|WZi;HF(MuYT=w>K*+8b>h`v9rFmK)9sXT5obp@n6k3`X|XeV=|
z7~vDVQiqc?@hFNq@0ec4D-af5L^(onXB9{Z*SWQ@^T%9y0A{JO4W;0Ymb1}V)W5Cf
zQ%)zuo-#%UpE9J9Onbuh2p;@`yCVqVkD(x>`hJ-Yfyiq`7UvdyIOD{ogg0&H{`QBl
z{dL4#wjBE9EWCpx??8+A7TA`AuKq`jO-MDnXkSudOdAq^Lbwg3USrO+Jmm1+o_mFB
zlf|XFux8Fc3XmvkIk$2~p#(i^(vJP~Xk^d5CDJ0=a5j67(2m-0JPY>*x$W~HU-@Km
zq_6eozPtdDGmp_f{sN|iAa8e_IDJ`zS!zL-Pqqsc_LLf6?-0*vNjOysbBr(>RkTfV
ztv~Xmf<5_&COnq*9_y>cE!1F>cXZh-46{U&UGzl)0L(s5cCi6S@L}H~J|e4mL`s&9
zA?_iqVxAqw5osXqf1WUkhf
z6<@K%GKA#0&!#OR{0chbo3{pYn4#yo5<4Y+rK$lPFq51P^gRsm|5H&!|{sr~%0+bPOU_pAxN-SXDV%hcJ2a3DYy1?_x
z#0G31oT}T?O3fHQm|grU5~#DXLm+}|O_yNR3=O!Y*qgR#gg-xct&l~QnO5Xk7SrS)9;XDfynW~Vh3qRUwUCX
z0FcWas}X-hUmBBJ8K{I6)V|EiqHLvstp#e%i7~i5!Uc2rEwKBCjvsGXlJE{{BT48#x)Vgg
zP7U7E6kW!yywZim%P|H`kuLIbi}0>4;1n+`3E@U1_tTG8ic%GVUsSTrkH+b%GP?cj
z=Ir?LdjGkf6lwcCE3aQB+qYg@YHgP`>=@0*n9-0m!TUx6~3DH8i5BNS^+yBz|N$h
zy7vn9fbGmi0Jg{{Wc$2uPHbb1o9|}N0{A4-XOhAtQ*AWq8`cq@Y$>Z+{su+=nbbdp
zu=wX|`Nw)agr_)4e=z;Kb;N)>t-*cPN~8`*N4?$gh9qlBUw`DS3sgHMaXunZ?C3r7
zA@WZq}$qG@oqmbOS=pdgz6&DB(
zV&8V`?kBU>8Xq5|9rwt
zIjl%H`1p(%v@n}l<*bH7*pyaz&_B(Y}zVuE%2g5leP*ICUW0LNKfj>cby
zEVNC|4nrxuemZ+Gq^@HHh44z=zr>A4K!Sh$Ez!Fcp6Qc2MPKKO0TrB?N6WRRQ6j~W
zbEYcvs9?+9t{b3{beK@}A@ENOoH9pu=Tke#t9h)<)DD;5ZFIpkNMgkVHT-j!1b>Jy
z{)o4E6|dE|2r(eSb^A>M2Y5iy>WVV*fS~Y3r@mw-`$(}SM#Id*_@fVP(IwmCzj}a7
zkTA=y^$T+HTUPhb9Qzl2f#KTmczngzV)bugE@`Ao{CtJA>vlXdH(4(ZGJaf-wnXEo
zEQC$c7S%c{6>37|Qx+lS+=nURe2xx{RPRhO>taN8#@|(OD~A4yFZ#^W;xBB0?E`SALuP3rYL_Fr>Q`1O-zc9q#WXTD%#Ev`54fxDyv=Jn
zCQXGeUD&*TAu6+?O#zIhFqbfK834p`g3;QacwbO!3vEsC2LRBYlr7u-0s&HEVvIHs
z$q_V*DE6{lzvo}ZX*v>4w*Z$jAfkl%T-@zp@0g9>2i-O-PSDZUAcIw6%G!p|Z$1@=onj
zOowuL0C{G$(;XEKWukn$;+R`XdZaS)2Pk
zeL~aw>uM-BYe%8)Hk6~j*i}9;8oBp^4?Pa9oj(=hr@!ZRR4oh;FQm$E2PR8H`9t-n
zxy2&|MaK|jrUf1ThGu1LZ7PmP{5^*7TMgIWN;{Z725xy;Ci4cMPaDT(Ft27kIY$X|
z)0H1NG59V|cW_Jnnx3iRgJ}BkE!f~I)kX;8L9wr91P%g%0}B+g5(E6kQAhhifRkfM
zI}Xfar>K&KLkESgYPP0llUZ0a8WS%Ix2ARXhfYJYaIy#!`~GuXedKL;0c)=(tC{nO
z|ApX|6F_8ETcq8;ulKZOdF=>z&-8scs4)cmbb=6+;tJg$NhZ%6!r!ixYK>GRuOyi&
z$+?~5hsLgy)*2Z`;sD&fB6SNTb3XsPDFR1qhOFfea`qXb1<>`QJ)Ax&h8mUx$JV&k)o0`rt@!K^ofWkGGg
zl4PPCjXAEvm(*gXO1+vHp2r#$
zN}A?-Bov;4-i0=^UR<0a<{Y1^p=zs*M~C%Rtx=u^&W$Dj-3+!~
zY@!nsiMm(AV;Zuyash0fbNToGNx?FY0V@xK<(wR>?`ga>$PkeH$?{mq`;Zw0{g4cJJy
z<)FZlP+U*CWhgeEG}0L~#l99rj!#h*QXZx=RIa1VAJlvjvtoI=6%*5zfM|yjl|bJa
zXX>lzrN`#5fs1qYzcyQR9?iJMo^~CgB@OiC2`Hh_MSH?!x?27_E0b2FBffO?6mF5~
zr{9$4f(!6FomME!>A#0L7+KQXR;@JE5GTAtG?)uyBuk@HQX2&
z`eG}qovqkcQ{5*<`dbNtJ+-gt(5lg*t%yov68|yDQrp4*^
z5cf?i)UYFe)AcN!0{@o$%W=-1PZ2Ogin2%Eexl;u>#73(`&JMrP)2wK!G&{6Peq<<
z<7)FpnRSD6MhV}ZzJ`M=$KH%B5YQ$dxNAiB2{K76
zULU}9K?gN}6=p6rej7?Xm+8u)(<2V!`HRx?Sv4opr3W64g9zU^%=mL2*(@ZdR^$&W
z9OGUawH#Ey9;5_@ukg+cR^*)P1C*(Oo!}7JRHv{ZSO7HfDJlWyjfU;nHD;aVVc$h=
zw@_l>Diks*>|lTBplHO+6;i!qP^4t7SeQ8mQjufSY_Jo0iWP$q+%pwKO4f_Z)r-r6
z<2_jDBc67L%&ordNq{Vdv`f}7sIuhA3^O
zLE~rjq1!cyZ+_7pib!VcD0(v?ik2}EA+r%#c>V#IX|ZPSo-*9C57-?_Nq;f&`Inck
z>2T#>AyGwBy-%jwPNw;PKb_P#?)+f8;R6FxWRW@}58&@ggjaKIwaInZEer4sF_Yz{
z;W7zJqNRkpZRSmc!__|;{ksJV3T~rr@nn^LR
zhX1GS!3yGSA2w+7rkM^lhqbhuik~`?DiY=XtDUP?lJ~9+N*9Az){uu2jim)Rfq8%b
zD#HVxItsHm$w6R1m9%^-H10JKl4@ywl~A3@e_#{}aaD}QN0w_ssp`>p!=c)EBQ>;b
zdne5wf7mSJjB`vRa4;_0>{eij?_)>(X_d+aPMy{vV{U?~3z^N_`LV-nXp+M@Sld)i
z8nX3h!)zxJd%SuTg}(x97Z?CMMf@!MnQoAa9dU6gjq*o;b37lB38C68KTDc7k`A6Q
z{9N;f@zBZ4TK=3!ai8b)7|$R6FfRvM8jCy@3ZF8gW{DArcl|YLa9ZyDb}f*)2AMCI
zOp$bem8lK8UgIPUx#Fo=8H)DE8OW#bo5>lr1$m+?KmB)px+8J{VO*8;vLBIh$u~qo
zLlP$AbU-AE_U2Gmtr!OH;Vqt2xQWaAbkPdn|X+57I=DcP!3+
z0j@Cv(U`w}Idm>%I}Nvg$Pu9NY%%Jl{!OwCAYp6Y-dwZWO7G$`pD6#Xv?YK7Ril%|Ke~;KmJF92ytJfYsF=
zD|yczz(VHo>I+$CMtBbAi#{;_J0luw>$MU;wGg{>m3?Y+1@F(bXwj)JyI#oTcAe#_
z#+U=1ZOri+!XK4px-IIo^o#Q#t@2d`{VLw|ys`@R1tsW>*-BA@W8iUXJF6$KV3cU-
z10Uiot|OS}hHjYj&V0Yox2=~6`Fb{xab6!60Qrnlif2ukZeydNU*#2Jyc8J_O{Ch!
zaj@MNZ*U)xh2saax6$ubK08bg0j-Ac;##Loi{`mRaSdvAT8-l-d!No%ko1k;bcS90
zRs)B7tG!NU(`Lh&mELW7(>&@PlykBO!*Z&51F4JBjID9KiXxo6kwJ@Vzx$zzvQ0w;
z*u!@_ni65Q8w#Pbt~O}a+j4=Q;hN($jU6IFi`wNM@X%Z4a!AZ`x^8QAugi6T6ICuM
zz=(B(rP0@a4kThL{cQ8m#Z`0LPH5RAH!{X@h;>YNE%s*+{v&KXqn>{`2d_CP89mfA
z2Xj)b3S=}5=yvR4rC|MBPH@4k$CFmXZ>h`%u(
zc#aT_^65v`XhLK4WG1omk6Tz?$vet+2I8w}_tNDd*us}?ZwE2HKR)Xb%aP!i%v7^k
z;C->}cub+nZ4XefpTNppXz9&+?rf7NpORCP1G4Zmd!DU^8&|?HkxnVa8NGZ-J0NJclf6jC@K`o;b-AW<{vqT|sXWQ*>2}H^5i#l^
z0rY31Mj6D(If6#B;|^OUSY=WpUQ2v{YMpIdn!kj
zwg#kNK*{%3jeV&7mrHBY$%v_)WG4hA#3M|yf_UAUBQqQ(g
zlfT|ZxQsU+=k|F*(_4%?iMVJ53n}^H{dFjzJ!tbA1Uux(d;nQkHN5_jv3yEs{g{xv
zr9F7HSY^0-94Vc8Nt!GbSR
z`9T3VMVy6{{>fCW^MG}`N+mr2W59#435(#V2Ra>V!v&|$_$9;&$KoxkkA+|4kHkI(
zzc#!cv}}j*dYYNF4w7Lj__rLw2vijq&8VtWe2x{E6Yhc1FY!CEP(fk|XQL9QfYhfS
z?N7)$cHXzwTkXmGf0*;KNT+qzejQ~KS2uNwcuz1OGa`BAE?N8Je2Mk~B$+2i;*Jhd
zk5EVNrO{r|G(0KD8kZe#kX5$HibWE5TSW;dT`ER*8RjNPLcI^I%kck+RKt1-<)Z9d
z@^0|oNIn^uR?lA_S?!HVv=gWlLZ=cEGc=gk*_HR?>-iFZkIAfCkghPpWkxY~%zOy9
z4XLp91g2)JXoMm^3J&>Ui6R$?wv2u##=Sx`%w`f%g5&#P%K{$;IN_FAAwZO5e?l#L
z&N~7*2NnNQXP2D?mV4LH_qKkM)23
zfH2rZKR-2cFdg#Up?B#^)!LNws!2om1^LyG+$&wppUB>$f?CNOmh2}n!-
zzax>;{|742BLYMJb%W0Ezd=NX|3EH=|3JL|hq3VgbV17gpQ9W*{C`m*rT%vmKdJw~
zYAHCN;}{Jnvh2T05}<-C)_=pHl1GC5pIEpF9FrCVxEbB8}+w0X>Z{6Ufq>9B?Ls=~2$)g=I36
zubpvZs>lN~|6Gxu9CB8Q=|(=7+jG{8X
zD&EOEfLxVYup+-&X^?+qP+EN`7H2eq6;5hYot%3~VDhr_QcPkkU?GDRnaO8?LV_2h
zm@In0LID?CnXLLJ8}{3SWTfHMJaGGsIf|*kIJ*eUx>JElH78HFXwBp^9n6fF&I5K2
zywC);LTyoW>dyizw4NmfbdMI3^jxsT{Bs3?Hfk{y%?ER3E_*O-STwnDk;de{%UVp)
zTPO2f0Y=zXHgM#A+k+gr#(RO15lof`CUYIIncR0qVDgqLQcS#uCSSOc%Jksy
Date: Mon, 26 Feb 2018 21:33:26 -0700
Subject: [PATCH 150/229] Removed auto publish from gradle bintray upload
 settings

---
 gradle/scripts/publish.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle/scripts/publish.gradle b/gradle/scripts/publish.gradle
index 833862f54..484adbee5 100644
--- a/gradle/scripts/publish.gradle
+++ b/gradle/scripts/publish.gradle
@@ -7,7 +7,7 @@ bintray {
     configurations = ['archives']
 
     //dryRun = true
-    publish = true
+    //publish = true
 
     pkg {
         name = "$project.name"

From 77df8847c0cee43476eaf3256d3ec7f5de19aeff Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Thu, 8 Mar 2018 14:47:43 -0700
Subject: [PATCH 151/229] Updated 5.0 API with old version removed and
 versionId added

---
 contract/4_0_0_contract.xml                   | 17222 ---------------
 contract/5_0_x_contract.xml                   | 17648 ++++++++++++++++
 .../ds3client/integration/Util.java           |     1 -
 .../com/spectralogic/ds3client/Ds3Client.java |     1 +
 .../commands/interfaces/BulkRequest.java      |    15 +-
 ...tageObjectsJobSpectraS3ResponseParser.java |     6 +-
 ...ectStorageDomainBlobsSpectraS3Request.java |     2 +-
 .../ImportAllPoolsSpectraS3Request.java       |    16 +-
 .../ImportAllTapesSpectraS3Request.java       |    16 +-
 .../ImportPoolSpectraS3Request.java           |    16 +-
 .../ImportTapeSpectraS3Request.java           |    16 +-
 .../StageObjectsJobSpectraS3Request.java      |    31 +-
 .../StageObjectsJobSpectraS3Response.java     |    14 +-
 .../ds3client/helpers/JobState.java           |     8 +-
 .../ds3client/models/BulkObject.java          |    16 +-
 .../ds3client/models/DataPolicy.java          |    24 -
 .../ds3client/models/DetailedS3Object.java    |    12 -
 .../ds3client/models/S3Object.java            |    17 +-
 .../ds3client/models/bulk/Ds3ObjectList.java  |    33 -
 .../ds3client/Ds3Client_Test.java             |    12 +-
 .../ds3client/helpers/Metadata_Test.java      |     4 +-
 .../ds3client/serializer/XmlOutput_Test.java  |    11 -
 22 files changed, 17696 insertions(+), 17445 deletions(-)
 delete mode 100644 contract/4_0_0_contract.xml
 create mode 100644 contract/5_0_x_contract.xml

diff --git a/contract/4_0_0_contract.xml b/contract/4_0_0_contract.xml
deleted file mode 100644
index 8fd0958f5..000000000
--- a/contract/4_0_0_contract.xml
+++ /dev/null
@@ -1,17222 +0,0 @@
-
-    
-        
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.AzureTargetFailureNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.Ds3TargetFailureNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.GenericDaoNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.JobCompletedNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.JobCreatedNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.JobCreationFailedNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.PoolFailureNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.S3ObjectsCachedNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.S3ObjectsLostNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.S3ObjectsPersistedNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.S3TargetFailureNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.StorageDomainFailureNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.SystemFailureNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.TapeFailureNotificationPayload
-            
-            
-                com.spectralogic.s3.common.platform.notification.domain.payload.TapePartitionFailureNotificationPayload
-            
-        
-        
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.9D854BF00D3B516D4045D9DBEB4B174A
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.A4F544C672FC8C11593427E560385533
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.B17329949B488995473FD573BF5A0507
-            
-            
-                
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.013CF864155A896718908128F7C31E1E
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                    
-                        410
-                        
-                            
-                        
-                    
-                    
-                        411
-                        
-                            
-                        
-                    
-                    
-                        503
-                        
-                            
-                        
-                    
-                
-                1.5B99BA64EAFD43AEC9359E716A93B234
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.416E43D944A06C824AD8BF5005FA1DC3
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.46B3A42F88F14FF8E433CEF3CED4C70E
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.773E50A74C3DC9EB87D6CD924CA1B34A
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.3A8DB55ED9BC0A138E7E63E96E6EF55F
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.8D18E6B210A254D12ACD074CF265700F
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        206
-                        
-                            
-                        
-                    
-                    
-                        307
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        503
-                        
-                            
-                        
-                    
-                
-                1.F50E6F8DEFB864FE89D9385A71A6C25A
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.5AB983AB4BABA77BCD64173290A741B4
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.CF182CD57551902A475553F26582BC78
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.871E6F61ED03FE40EECFD0AAD53AAA34
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.12220CAF96D6A1B3E1BC31330E95EA51
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.51D7750881DA0A4621487FFE61533D85
-            
-            
-                
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.E368236F01A4B820CB381ABB9DC6A481
-            
-            
-                
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.8926D009A5E919B034A8C12CC7BC6AA4
-            
-            
-                
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.AB6BB38915AC1B434AA79A746AEC1CA9
-            
-            
-                
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.090A514BE0B269138CD466668F79C773
-            
-            
-                
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.1AF4C0F11AB88FCD30536164F07B3944
-            
-            
-                
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.878704403C5B1CFAD98804ABD56AB840
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.09B5FAE5CF69BF5E945E124D60852125
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.D65ECD1D08D1272AD5E969F80235DFAC
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.0500995E51CDDBCFFC70327F12F2428F
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.C6B44D720B8BC5FEC46CEDA4EA04CA5F
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.CE89262FB9070D85EAB1C361CB74D99D
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.D561CC0BB1BDF0FCD73E2AC1509393F7
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.14A107B85D096EB8554EDB0A4A8589B9
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.8A4F25C489C1E3C970CAD13B9AD4754E
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.4751D97F3546F963C451CE9F9AE16C76
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.7E0B95E1E222BB82689E2AA8CD3FE344
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.4FAD1C67C3247080EFA33C224B87544D
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.8C885A22F5F8EA044A104A020AE563D5
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.0EB0673D4EFC79596617FFE680876C1F
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.E697E63D8C13C0C06626E57BB95305CD
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.90CDE2CBD14856E5FDA2474704819E6B
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.581382F387E3FC77A09A11F14FEB312D
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.4E2A47F1477F1A5E5D4BA9166FD13700
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.8D11714D318A13D76782B5A71471D887
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.00FF753915ABC8F954677D57253B3230
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.002A078CDCBEB6844EA00B0BBAC7CAC8
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.B27EA3CC583291FFD676193DBAA4551C
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.8E0A2BF92DD79E7C634778C97619CDEF
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.1789113D83DC2E48D1B5D1C221DC1872
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.C554C793C965F9B60414FF9092DD6907
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.15D63B0FD0C4257041795BA2235277C5
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.A1CF121219F77829C8480AC814898D87
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.74C5BC1D5C856DAC66A4AC9893FC4BB9
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.B6E603A64C60859EEA83B2E687FB55AB
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.CC4C9ED25B1509ABD0F30B375ABA4D78
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.377DC47D5802F8BBB59CC5D011268847
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.E3B3238A4639A19B8507203907EA2D52
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.5C597451975C08FCCE2F0647EDD16872
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.D032BD8264B23E052A900CD171273D84
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.779978550E2AC7519DBEE1B6418D7FA1
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.BD89C8ADCBAC5C96569ABC8DE7D841D9
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.760344CC17CCA3AAC0D1AEAAB22664BE
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.7FE0018AE76D9D24DADF93BC83A73EF2
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.9A7A0BABDD6E7CC2D134B62D158E6C57
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.8C6D05D23CDCAFBC0344FE0C91513C32
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.40C486BA0D58B76E2C253E81EB297D0E
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.18B2E9524D43E524BBB9C54ECA35E5F3
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.87B1B5789ECD05E15FC0C8BFED48E73E
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.B23CBBE34EDFFE1E8FBF444E4E0EA0ED
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.F85D09C772C76D466ECDE210ABD0F6E2
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.D90C5C1EF5D3968AE776DED4DD4B41CA
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.4C6316BC92D7D1285A6565E212E6999E
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        500
-                        
-                            
-                        
-                    
-                
-                1.BA76BEF97888F20DAAB5ABAF89492E6F
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.B024C098E863E2B1C9275B0EBFBA3BF6
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.E7A5FD996704C81CD95CC171BA42549E
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.F51C99474F629A2CFD5CDB6BC5CB7C6D
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.FB9358D923F09B224C1272BD9B459121
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.B030FA7F19336B7B8C64BAF6E6F6346B
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.FCF1BFADC9D4DE1CF114901B69B22280
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.3970B4F7B6EEDF3D16C14B1FE8D74EA5
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.FCDDCC0EE93BE92A31D4BD84C9DD57C7
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.39020D24E8F55C01FF5ECC4524C78189
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.823E96C86A899E615759E3AADB7355E0
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.C8788BC1989E0E16F93945C2F157893B
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.0D4C916F64F122E94EEB6674D7B9F4D1
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.620EF529E4CD2E0912AE9F42E2B802C0
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.2F15899DE5E70055E0490AD29530C842
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.DE22EC0929BAB9F743BFC38EAB10D680
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.905EEF6FC284DC0AC05F8F1B347829C0
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.BEECB5756E585359144D71D4410D4846
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.B8E477B5613F3E518F79082D5552E7AC
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.1480092553E000D0103C80E1BC65B2C5
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.B32DA5E8C42BDE53830D288C86582A9B
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.8FB599E4270D6BDE67B81DDEAACACAC8
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.1B32364BA03FED4EBBD9BEF15A49EFB1
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.F1B9C4424263D106712B36B7809F8EFC
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.782935F97E84EF7CAABC357C5CEE2271
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.F4A8A78DF6670ECDAB83A83A18D1547D
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.2C0E9739F7F0BF5B9C16B620D3C195EB
-            
-            
-                
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.8611905E16872C3EEFC979239C47CFFE
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.D6C48865C0991834DDDE193AB40E04EA
-            
-            
-                
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.5A1DF310850382BEEFC771A9C1DECC3D
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.33DEA650BE9ABCDCDF53E462D402DAFE
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.E434ED8B2A34F104A95442717B1A4BA7
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.25BA53F39517497CBFEC02F6F1A1281D
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.02D9D53D22A0F5B6893643EE40ADB324
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.ABED6B8929079AC01A596C9E4C08CBF8
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.CA6D598467AAD4083472C11E72F28A8A
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.03E8ABF4920FE0C59B424EEDB8001FAB
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.3289F9FD0E9ACCD50787DB1A0055712D
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.4DB23DDC56235578526ECD74A1E1D151
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.278BB8E0110D66EDACF595107C09427A
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.5787086FAAD0AEFD6965BE9007AEF32F
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.F66A4CBE1684496D395E5546FEB8C8B8
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.EDC1DDE6748591464113649924079D39
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.5A6D3D3A60E77AC626275AF623C4C766
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.C079F9483A04E55B04DBB93A82435427
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.1212DC6708EBC50394B744582FAA65B4
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                    
-                        500
-                        
-                            
-                        
-                    
-                
-                12.7853EE12A10929010211D370E4E9FCB8
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                13.33017D3F34393D367E5E63FB7F1DEB3E
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.1AA08A21610BD902ED2A0BFDBB145B9C
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.5F29580D0FA9D13D4F31F6FA1479253C
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.BE9B123D699A33A084C657EDA216FD46
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.366B50C862A74986AFC2A1FD9AFA0DC8
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.2CB8F6FD0FD4AAB2DBFDA3B66906F811
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.82806C1E451AD1325DD45123292E6360
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.AF1AD1BB3EE186EBC9E967F37EF47B63
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.81935DED0B11C2F6579420B1D503F846
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.96AA26481EDB1492BF1A4696FB4A5E90
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                    
-                        410
-                        
-                            
-                        
-                    
-                
-                12.E85F3447509D8658DC4747BDBC1124AE
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                12.D2E1E5F53C787AAE87CD6C64E57DF392
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.5441D6DB77E32465283AFFCD70BF0BD9
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.A91ACBDE9427D4617DBA938B01FD4A65
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.46852EDC36DDDB68792432241EBAB228
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.8DD6A5CAEBA22CCE74B380322FB5EDD6
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.6F544A6CED02468AF923384632138561
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.A1093E63522E0D37459A3DD2ED4BA382
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.381372DCEE1323BA3EC6135F28BD40E7
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        411
-                        
-                            
-                        
-                    
-                
-                1.B66BD19B24CA38C77915186CAC7278FB
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.61F4F2C6BB6C6B6682D36487830BB9DC
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        402
-                        
-                            
-                        
-                    
-                
-                1.5A3E7F31249938D514C56E364FC6D02F
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.6DAEDCFBF2E71F76371CDB2DE03FB216
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.4EB20E46D4C9A9D1A9C832A10416B269
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.D6574E9F317A632E897CC1497A014EE7
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.2963DE5A15B7A94AE018A555B04855E3
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.BA2D594EAC2171E3CD66A1E4B261C157
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.F6F99FB1D96FAAB52761A4A6F9C007F4
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.4493BC861FFB54D0474875CDA923917F
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.BCDE69D118A54DB3850A7A1DCB9044A5
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.B9B14FD95B7365854A4873C9E03CF714
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.CB59926ABCCE409B8DA620D7098051EF
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.8A412C738BCEA2E768EEE5AFA91A02E3
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.F58DEB2156F89FEFF73AA296FB996076
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.6D9C05CCD6FC973CEC7EC06B2852C004
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.D2F89CEFD1D076C722E383CD47F54B0E
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.C3E90840E58C0ED58DE5285F1C629007
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.697EEA5D819D1CCED8356F3707A80CC0
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.F8EE6F2C578B71C30A5818323B8C63DD
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.CEC966DCE333C324697D8D2C014D5844
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.F75FB6B03E5778971F18B3E156F1C368
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.FB37D5236F8B9FAA3C23E1E1ED75EF7B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.BF2D067D7C365D5CD381837F5376FBCC
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.1DCB7A5A19D0073AB4C529DE68EFF590
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.AE62860962CCA944FDE6BA7A1FA29BE8
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.74228C20E21299810A61BDC1D72981CB
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.5A5F6DB8F16E8BA2850BBC9786BF933A
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.9B0188827522EB7EEF626D5957F16EF7
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.057E27FF0C7515839363635ABD87011B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.815B2098111720A02816432FD0B11263
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.F17A1AE113B6FD07741496A307004005
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.3FD065CF09A9346481021BC207729FC8
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.67E8651E0A9719E5AA68ED2783076E40
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.E7E2388B6F0182AA8B191836FDC2EF46
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.0B3D7131A8DE08F23DD8F92E4FB445CE
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.E016A3B10B9E2BCFBDC052B607BEF218
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.405BE0983E37E87E48E0D549DFFD22CD
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.D55B0B0605B4BF3AE32149CAD90110CE
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.89699929828B8EC5AE4F1F41003A1293
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.02BC732C912E22F368101E336487D192
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.58AA7409654D5D50B6B8F55539C0774B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.6B1CA505E2302A04F63AA27099F0FCFF
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.96BE833135E4A5C8F7DE52803D85CCA1
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.3FB901D98106C97ED02FBD2FCB72A368
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.06D8300D0336BED673A20EF70C44CCCF
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.C9293362A29ABB99EA2382C6B71D0649
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.351B871DFA1717273590632B38A4EBC7
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.6DFABCE265B2455CBE17C12CF01CDDD2
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.C4D2CFB128128AFB2C46656E3EE693BB
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.9222B04244CF97705F3488EE7906FC41
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.AA0AD23B2F046B29AA8947581FDFB860
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.D74722063AC27B6ECB815A78C0E790C5
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.E5ACD5AF9B1F02DDDF00504737297CFD
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.8E55D6D6105C140A17D3438C4F4D8D54
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.86F306440C4799F14DD5F4C0719E507C
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.301D82C48DB6624F74A9A6BD309A9E40
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.098207A6680442B3FA9643BB23CB370E
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.0EC16E3818F17C785ABB022A34A7A083
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.F1ED654859180EBA33AD4697444A2636
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.84A39C5D428089EF64F5DB0558FA2C45
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.4C47E15164EB0479D97A05D6AF22162F
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.BEEFBBCACF4A979C567F1D144C04067C
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.51440FCFF3ADF3834C5500FF554FFBE2
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.70D55191E13B0C77A5FF57DA126E0849
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.6B1C8916BB856B9EEF24DE4CB561989B
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.1F6F3DD14D68AD328398FA0C8C1BA523
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.8677D16DE53C2800647C7DB609C6EE44
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.67D1B7BB3E64E9C05C62F59054DF73DE
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.867574E65E9156E31D51517D17DDE283
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.833E8B67365C5B36D6B3E2C048660EEB
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.E7BF64005D511F7DF092A1FD63CAFDB2
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.6DA60D3A441DEB83245CF9390DB652C2
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.FCB7A1CCECDBFE60B9F9268A4DA13E64
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.5EDB52181C6563D67BB04B9F6F9FFCB3
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.602C9239FB1B18A48D8E6AB920F9865E
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.613F01620D33F44F7F08DAD27477860D
-            
-            
-                
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.B3F196534F6B3239F608E60F46CC5054
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.9395090644FD6A9EEDE620F2EDB2659C
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.51360490B8BC0FEA841584D3A7941DA9
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.21D484F75EB67BB6E52299CFE9026E8C
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.B470C7165F3CC8ED799ED4F8B5BAEA45
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.C56316687F31CD493F7064FD951EC99C
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.79EEE017A84D0C323D68CA23A8CB1076
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.A4AFD3D5DA908492647C739E941CC97C
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.EF17F568146C914F8023380045758F9C
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.3D539F4D333D3240C489C26EC0A011EF
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.B0E4F8E2EF4C7507621D37BF97395F31
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.A426A6A86571D4DB76768FBF9E3540DF
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.BB0DDF53B3466065E0CC36A0EDFF2834
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.033F928F2F6A0DE70086DFE3C999B9B6
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.C21A692F9E87799AB2806A96B545C0C6
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.6B887FD15AF647C5F0ED993C439A0EE0
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.28359B909678D42A4868FB82F28934BB
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.486CB0412BAD44C82290FBD64DEC1F6D
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.C2B1E38A255F67B820C6DB2C5A021FC5
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.AA7061BA6F294ACDB2FA2282E0742BCC
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.A5E2220D36BE184AE56CCD3CFC42D0AE
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.F6BC5F562CC7CDDF11632321B21D97DF
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.97CAA19C72182CEF847821F9E6FB3AAA
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.3B8F9F9CA8F1879985BF29BF5CBD21E6
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.019396D4D4686224FF7C3CB6D1E0C0FF
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.56E9BDA78930F90CD41C280B94F3EC92
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.A7B51CF0108CAE7FF2ED2ECFEEF990D9
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.02F4797EB106FFD0157EC4EAB8FBB3BC
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.6D17F999F7BD299E10604D67FB269746
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.FC523C8224CB7FBABCF213D5D951CB18
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.0647B93684A5AF9B4EF9AAFC7148E0C7
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.04BA526681D926E6F55834C3FF14C800
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.C7CE8CF71088F3DB585CD3B02AD6C01E
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.C784129AA41D66504348A3879B18409C
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.B3C891ADBE07985D8EB07F47AD5B9DA9
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.574C8D1B7983E679131E37FD5C3DF1B3
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.75E4BA966FFC501D192F53E91B7DA97C
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.CB9A35E0D20D8F23F42BE689119E06FA
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.D7B16FCF4CB54A4DDD0B7B96965B0C89
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.6C78CB68979C1B30B65792A51C4909B4
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        503
-                        
-                            
-                        
-                    
-                
-                1.1425C7EF3EB1C555EEDB3A7B83959C18
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.6C71667DFCE4EFE9D2D594675F5D0548
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.3FA058A0E73CD716458E4AD473D01D8E
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.7A7A1F423942BCD065A53F5E2C08CA10
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.686D026F4E37237A3FB0CFDDADBC9F8A
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.285B3B8B3249390AC0EA62095620648D
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.483EDC95D9050419855BB2C9B7D9395A
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.57134A2893313DE704198EED54D5B6E6
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.9193C0F29388C5B9B6583A6BF6C2B347
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.424FC52414BFE0A49B6746D5DB1256AB
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.13C8280BFD2D49B31E703A6C942A14BB
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.2ED8A733A34F7268A165FCB3C4DE0232
-            
-            
-                
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.BCD510DF25513B8FAF0352336F094053
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.D24D24F09DCA78579C08ADE1D5528A18
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.388A14B11DC4161D76E7BA3524CED336
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.E8ECE9BA39F0B03F8394B4AB5065B60D
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.67C819CC3E37319CB3D533A2D31F00D7
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.7A9D46F2B10E01B201BD25BF3DA3E3D6
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.466E11BA31409FC6549BB27A0C1400CC
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.5458586BDBC05CE6DC7B96FB305E279E
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.D07A22B79C602A5E8C570C5E049FD7AB
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.0AE60BB682E75CC3CB3CE074995267EF
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.EF0CC59153D7673ED206E0F310880A6E
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.D946E977B16DE185BC7065F3C36BBA94
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.6758EF1C334FA94EC51F836A229DFFCA
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.D96F1DC7CD390C447BBA6A55F52476AC
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.74729E8767FC62D9F49C309DA2993DE6
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.D9690323F9BD00A1D7AC14D449C5C5F5
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.2BDEFEC679C5FBAF0D8D2934FE55F708
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.2ADB64C9DC3241D0FFE93910210E1FF9
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.12337E3CF98F2E0E44AEB002441C4918
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.F0DCE0BD19F895084F3C461358ABB31B
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.362B98FD68A372E063F679FC32F89B31
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.CB7932D0E8A306C5A2E35C8916C0E4B4
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.E62DED290400027E754EB45DBAC5062D
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.DBBD22390D6F477450A6966FDA2CA298
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.FED65C1D6BB97C21B4F9DA0B7A997D35
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.44F69227AF5D5C013E4E8870692550C7
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.6B2469AA8D0FC2655F6EB394348EB437
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.A0A9BFEABE8E3C638E51660CF67D0B3C
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        500
-                        
-                            
-                        
-                    
-                
-                1.0BFC9FE27D3A590769329D70A0E2E872
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.D86253165E5E58A4859699AF6C04C146
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.DB420EA069BBE1730C24DC9EDA476595
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.31CBB28C0277768C8BF04A6CF1BA98A5
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.E6B45FD1B7595DCC123636DD01E50D8A
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.AD707ECC69A1E835FADE9B7B896C7C53
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.2EC7C7222330CC8A26AD23348C22DFBD
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.304855C6B7C0744B1F4E23991BD4CF83
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.94A9B934CC18AB22E125F70FBB9F82DE
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.038E34A9C374FC176986FC568A165527
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.CA5A1C1C65E431A37F3207B81767774B
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.7D48EAAB19465D9008EFE5C6E3C020EA
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.3669C6C2F1550E1554DB9CF129DCD965
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        207
-                        
-                            
-                        
-                    
-                
-                1.A6D31BBFAFCCC9A55B9FEBF72E1C2953
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.32E4082ED243CB134EA743FF78000BD5
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.38E575DC2AF4B7D7CF3203E6B7BDA74D
-            
-            
-                
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.220648A0923367BC4FA2794FA4BE4591
-            
-            
-                
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.38448C20E04942C66F6D9AB3169CEF09
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.B7BB4C29193AC92DB35E9E206A01E30D
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.4D5F15EE9801D7E3BE0892FD38AFDE0D
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.E062F9FB6D3FA8A9EEE31C6A81CC6604
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.78F8ED04EC79A126AEADE8C08B0E4F97
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.958F111DBDA1BCE46EFBC10511A145E9
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.1E72B20786B3BFB6DE896C380864C4ED
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.A62A0E7E1808EED079989D04772B6CB6
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.65E7DE349575885F8EC2E78FFBBD3484
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.17A61EEDF79D76747BE37B3DA3F698DF
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.F0EE6E4715A3D2CFB1F5795EA52EF0A1
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.59DE4FDFB535F952488C8C76082D1957
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.E8DD94F2A92ABC73D6632A48722BDF66
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.83B487198471CDEAF5C95D371BE1227E
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.B13DC16C4369B842DD1580DFC9A23E90
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.D9B3DB52DF8218B45D48BECB5131915E
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.CF52BFFDF3E12D43EFD5F614833B7DCE
-            
-            
-                
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.FF43777C8C33E41102BCF3F2A2A90C64
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.7B3676D45CECCC1D12D71EBA722FFF30
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.71B0E6E68AA99863BE55E1D6D037301B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.9F2F0F9EE78721BF80AD83701F9736A0
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.680A6E46ADDF391752A3A89F95D73448
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.496A9334045EC97BFFE3A5116694FF85
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.BC1C6F682219387D81896AD542BFDA1B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.21E756DE8577C5778B49EB14777DE712
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.9066576F0656AE6BA8E13A019F6A59AB
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.444CF76C0D4A6C4E637033A4D4E9126A
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.3BFB533872841D46A011D369D3FF2977
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.3426C46B6878DDEC0DA8DA6A0D5554AA
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.068DA432A58C3D7331CA5C906158F229
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.B316331E65CCB79F118C8A178C4D1112
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.27B9DEC971AF1E0443505F6381B2C647
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.F528AE4144225E113F7B00825EE52AA2
-            
-            
-                
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.8D68C314C554FFCC800097F48EA2B2BF
-            
-            
-                
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.B29CE6CAE05044347C6F6D2CD82D2B76
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.0EDA4B68A5E0A7A119DF093167A9643B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.25F94118327D5D07CF4BD4214EE33EC0
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.AD3D66C9BD4CFDABA55559C35CECD9E1
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.E336292FC529AF487A9AD66216D49654
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.673FA4C32058D8D329E1630ADA8E73C1
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.B74A7AA44043A326AC2D7D7934967B14
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.75F9BD52CF2A252C3696A912624AF4E7
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.1568E0AA2E38BC533DF1D9A3C26D9A80
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.DA7E3AB4DC966A21CCA466E87760769B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.AF491ABBC242A023D510DB3FEFC8D333
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.D0BF3B5963590A886B4DECEFF9D5DA2C
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.F09CDCAD633DBE0BD4D81B4ACD40F402
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.4F8C9F278402E6FB2E4EA057B834CC73
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.251760D30B95DDFD686CF5F8DD244B34
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                
-                1.0FC6240C9AA47CA11A06C45AF5EED527
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.AA72044B9C3357E0774AADA4EF555CC6
-            
-            
-                
-                    
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        409
-                        
-                            
-                        
-                    
-                    
-                        500
-                        
-                            
-                        
-                    
-                
-                1.6BCFED905208FF2C3021E49E1113053C
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        500
-                        
-                            
-                        
-                    
-                
-                1.E7E6381C3DCFF65F04EFF2E13C486B02
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.BCF75A4FA6D197563529575007325A9D
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.4D283CDE0658BDE321C6C7B4CFB00DB5
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                
-                1.6D32416F86CAE6372CEE63F13A13B4CE
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.E49AE7C4E4606611A2264F17092383DE
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.AF255C11E7836E705C16FB2E0C5C5893
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.EDD41D719CF5C94BEBC543ED4E1FA92B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.87C46F7702591A04073B685E3DCA18E8
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.DF34657F1F638BF55CCB59D8766A7A40
-            
-            
-                
-                    
-                        
-                        
-                        
-                        
-                        
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.227468E94768C87DC18B9D82AE1DD73A
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.A11EBAEA1FB195A00CA8C77DAA86896E
-            
-            
-                
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.ED42C99A3F7707AE33EA46D9155ACF67
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.CB843F4F8FA5F1FD7A435EF77E9AB382
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.55C741CC9A0D6215D57FB954E57DBF9E
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        200
-                        
-                            
-                        
-                    
-                
-                1.A37E5119D3567CC0420518DD48F43AC6
-            
-            
-                
-                    
-                        
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                
-                1.14B1920F8A49D1721022BDA3054DF8EB
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                
-                1.BB98871AF82585B0C646B6954309CBC3
-            
-            
-                
-                    
-                        
-                        
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        201
-                        
-                            
-                        
-                    
-                    
-                        400
-                        
-                            
-                        
-                    
-                    
-                        403
-                        
-                            
-                        
-                    
-                
-                1.6D706AB11308B480CF5A83510CFA437B
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                        204
-                        
-                            
-                        
-                    
-                    
-                        404
-                        
-                            
-                        
-                    
-                
-                1.EA981FC342B0BC7143D8A6B5C650CBA3
-            
-        
-        
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                            
-                            
-                            
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                            
-                                
-                                    
-                                
-                            
-                            
-                                
-                                    
-                                    
-                                
-                            
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                            
-                                
-                                    
-                                    
-                                    
-                                
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                    
-                        
-                            
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                        
-                    
-                
-            
-            
-                
-                
-                    
-                    
-                    
-                    
-                    
-                
-            
-            
-                
-                    
-                        
-                    
-                    
-                        
-                    
-                
-                
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                    
-                        
-                            
-                            
-                        
-                    
-                
-            
-        
-    
-
\ No newline at end of file
diff --git a/contract/5_0_x_contract.xml b/contract/5_0_x_contract.xml
new file mode 100644
index 000000000..995e50815
--- /dev/null
+++ b/contract/5_0_x_contract.xml
@@ -0,0 +1,17648 @@
+
+  
+    
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.AzureTargetFailureNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.Ds3TargetFailureNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.GenericDaoNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.JobCompletedNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.JobCreatedNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.JobCreationFailedNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.PoolFailureNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.S3ObjectsCachedNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.S3ObjectsLostNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.S3ObjectsPersistedNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.S3TargetFailureNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.StorageDomainFailureNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.SystemFailureNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.TapeFailureNotificationPayload
+      
+      
+        com.spectralogic.s3.common.platform.notification.domain.payload.TapePartitionFailureNotificationPayload
+      
+    
+    
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.9D854BF00D3B516D4045D9DBEB4B174A
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.A4F544C672FC8C11593427E560385533
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.B17329949B488995473FD573BF5A0507
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.013CF864155A896718908128F7C31E1E
+      
+      
+        
+          
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+          
+            410
+            
+              
+            
+          
+          
+            411
+            
+              
+            
+          
+          
+            503
+            
+              
+            
+          
+        
+        1.5B99BA64EAFD43AEC9359E716A93B234
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.416E43D944A06C824AD8BF5005FA1DC3
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.2ADEBFA66BD7C8629539A7399BD3A755
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.3303E206BA09EA57A94B52B6205C4782
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.032AD615FE7006F89222B5D50F080D6A
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.8D18E6B210A254D12ACD074CF265700F
+      
+      
+        
+          
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            206
+            
+              
+            
+          
+          
+            307
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.9E10AB31443900A845A6A64AACDA46A8
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.5AB983AB4BABA77BCD64173290A741B4
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.CF182CD57551902A475553F26582BC78
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.871E6F61ED03FE40EECFD0AAD53AAA34
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.12220CAF96D6A1B3E1BC31330E95EA51
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.51D7750881DA0A4621487FFE61533D85
+      
+      
+        
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.E368236F01A4B820CB381ABB9DC6A481
+      
+      
+        
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.8926D009A5E919B034A8C12CC7BC6AA4
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.AB6BB38915AC1B434AA79A746AEC1CA9
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.090A514BE0B269138CD466668F79C773
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.1AF4C0F11AB88FCD30536164F07B3944
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.878704403C5B1CFAD98804ABD56AB840
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.09B5FAE5CF69BF5E945E124D60852125
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.D65ECD1D08D1272AD5E969F80235DFAC
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.0500995E51CDDBCFFC70327F12F2428F
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.C6B44D720B8BC5FEC46CEDA4EA04CA5F
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.CE89262FB9070D85EAB1C361CB74D99D
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.D561CC0BB1BDF0FCD73E2AC1509393F7
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.14A107B85D096EB8554EDB0A4A8589B9
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.8A4F25C489C1E3C970CAD13B9AD4754E
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.4751D97F3546F963C451CE9F9AE16C76
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.7E0B95E1E222BB82689E2AA8CD3FE344
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.4FAD1C67C3247080EFA33C224B87544D
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.8C885A22F5F8EA044A104A020AE563D5
+      
+      
+        
+          
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.0EB0673D4EFC79596617FFE680876C1F
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.E697E63D8C13C0C06626E57BB95305CD
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.90CDE2CBD14856E5FDA2474704819E6B
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.581382F387E3FC77A09A11F14FEB312D
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.4E2A47F1477F1A5E5D4BA9166FD13700
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.8D11714D318A13D76782B5A71471D887
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.00FF753915ABC8F954677D57253B3230
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.002A078CDCBEB6844EA00B0BBAC7CAC8
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.B27EA3CC583291FFD676193DBAA4551C
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.8E0A2BF92DD79E7C634778C97619CDEF
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.1789113D83DC2E48D1B5D1C221DC1872
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.55EDBA2263ACFB0785E441FF2A58B924
+      
+      
+        
+          
+            
+            
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.15D63B0FD0C4257041795BA2235277C5
+      
+      
+        
+          
+            
+          
+          
+            
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.A1CF121219F77829C8480AC814898D87
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.7EA25AF7C646BCB2F9BDF8736B50093C
+      
+      
+        
+          
+            
+            
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.B6E603A64C60859EEA83B2E687FB55AB
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.CC4C9ED25B1509ABD0F30B375ABA4D78
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.377DC47D5802F8BBB59CC5D011268847
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.E3B3238A4639A19B8507203907EA2D52
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.5C597451975C08FCCE2F0647EDD16872
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.D032BD8264B23E052A900CD171273D84
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.779978550E2AC7519DBEE1B6418D7FA1
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.BD89C8ADCBAC5C96569ABC8DE7D841D9
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.760344CC17CCA3AAC0D1AEAAB22664BE
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.7FE0018AE76D9D24DADF93BC83A73EF2
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.9A7A0BABDD6E7CC2D134B62D158E6C57
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.8C6D05D23CDCAFBC0344FE0C91513C32
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.40C486BA0D58B76E2C253E81EB297D0E
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.18B2E9524D43E524BBB9C54ECA35E5F3
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.87B1B5789ECD05E15FC0C8BFED48E73E
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.B23CBBE34EDFFE1E8FBF444E4E0EA0ED
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.F85D09C772C76D466ECDE210ABD0F6E2
+      
+      
+        
+          
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.D90C5C1EF5D3968AE776DED4DD4B41CA
+      
+      
+        
+          
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.4C6316BC92D7D1285A6565E212E6999E
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            500
+            
+              
+            
+          
+        
+        1.8AB4C4419A05BBB45CC2EEEFE10CAA82
+      
+      
+        
+          
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.B024C098E863E2B1C9275B0EBFBA3BF6
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.E7A5FD996704C81CD95CC171BA42549E
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.F51C99474F629A2CFD5CDB6BC5CB7C6D
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.FB9358D923F09B224C1272BD9B459121
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.B030FA7F19336B7B8C64BAF6E6F6346B
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.FCF1BFADC9D4DE1CF114901B69B22280
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.3970B4F7B6EEDF3D16C14B1FE8D74EA5
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.FCDDCC0EE93BE92A31D4BD84C9DD57C7
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.39020D24E8F55C01FF5ECC4524C78189
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.823E96C86A899E615759E3AADB7355E0
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.C8788BC1989E0E16F93945C2F157893B
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.0D4C916F64F122E94EEB6674D7B9F4D1
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.620EF529E4CD2E0912AE9F42E2B802C0
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.2F15899DE5E70055E0490AD29530C842
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.DE22EC0929BAB9F743BFC38EAB10D680
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.905EEF6FC284DC0AC05F8F1B347829C0
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.BEECB5756E585359144D71D4410D4846
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.B8E477B5613F3E518F79082D5552E7AC
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.1480092553E000D0103C80E1BC65B2C5
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.B32DA5E8C42BDE53830D288C86582A9B
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.FD0CCCBDAB885F5E5C8D9BD9DA12B9CE
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.1B32364BA03FED4EBBD9BEF15A49EFB1
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.F1B9C4424263D106712B36B7809F8EFC
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.782935F97E84EF7CAABC357C5CEE2271
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.F4A8A78DF6670ECDAB83A83A18D1547D
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.2C0E9739F7F0BF5B9C16B620D3C195EB
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.8611905E16872C3EEFC979239C47CFFE
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.D6C48865C0991834DDDE193AB40E04EA
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.5A1DF310850382BEEFC771A9C1DECC3D
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.33DEA650BE9ABCDCDF53E462D402DAFE
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.E434ED8B2A34F104A95442717B1A4BA7
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.25BA53F39517497CBFEC02F6F1A1281D
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.02D9D53D22A0F5B6893643EE40ADB324
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.ABED6B8929079AC01A596C9E4C08CBF8
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.CA6D598467AAD4083472C11E72F28A8A
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.03E8ABF4920FE0C59B424EEDB8001FAB
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.3289F9FD0E9ACCD50787DB1A0055712D
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.4DB23DDC56235578526ECD74A1E1D151
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.278BB8E0110D66EDACF595107C09427A
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.5787086FAAD0AEFD6965BE9007AEF32F
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.F66A4CBE1684496D395E5546FEB8C8B8
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.EDC1DDE6748591464113649924079D39
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.5A6D3D3A60E77AC626275AF623C4C766
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.C079F9483A04E55B04DBB93A82435427
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.1212DC6708EBC50394B744582FAA65B4
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+          
+            500
+            
+              
+            
+          
+        
+        12.7853EE12A10929010211D370E4E9FCB8
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        13.33017D3F34393D367E5E63FB7F1DEB3E
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.1AA08A21610BD902ED2A0BFDBB145B9C
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.5F29580D0FA9D13D4F31F6FA1479253C
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.BE9B123D699A33A084C657EDA216FD46
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.366B50C862A74986AFC2A1FD9AFA0DC8
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.2CB8F6FD0FD4AAB2DBFDA3B66906F811
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.82806C1E451AD1325DD45123292E6360
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.AF1AD1BB3EE186EBC9E967F37EF47B63
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.81935DED0B11C2F6579420B1D503F846
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.96AA26481EDB1492BF1A4696FB4A5E90
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+          
+            410
+            
+              
+            
+          
+        
+        12.E85F3447509D8658DC4747BDBC1124AE
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        12.D2E1E5F53C787AAE87CD6C64E57DF392
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.5441D6DB77E32465283AFFCD70BF0BD9
+      
+      
+        
+          
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.A91ACBDE9427D4617DBA938B01FD4A65
+      
+      
+        
+          
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.46852EDC36DDDB68792432241EBAB228
+      
+      
+        
+          
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.8DD6A5CAEBA22CCE74B380322FB5EDD6
+      
+      
+        
+          
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.6F544A6CED02468AF923384632138561
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200 
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.0D522E432703A7E48DBEDD1A6BB5CEF8
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.A1093E63522E0D37459A3DD2ED4BA382
+      
+      
+        
+          
+            
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.381372DCEE1323BA3EC6135F28BD40E7
+      
+      
+        
+          
+            
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            411
+            
+              
+            
+          
+        
+        1.B66BD19B24CA38C77915186CAC7278FB
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.61F4F2C6BB6C6B6682D36487830BB9DC
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            402
+            
+              
+            
+          
+        
+        1.5A3E7F31249938D514C56E364FC6D02F
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.6DAEDCFBF2E71F76371CDB2DE03FB216
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.4EB20E46D4C9A9D1A9C832A10416B269
+      
+      
+        
+          
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.D6574E9F317A632E897CC1497A014EE7
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.2963DE5A15B7A94AE018A555B04855E3
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.BA2D594EAC2171E3CD66A1E4B261C157
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.F6F99FB1D96FAAB52761A4A6F9C007F4
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.4493BC861FFB54D0474875CDA923917F
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.BCDE69D118A54DB3850A7A1DCB9044A5
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.B9B14FD95B7365854A4873C9E03CF714
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.CB59926ABCCE409B8DA620D7098051EF
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.8A412C738BCEA2E768EEE5AFA91A02E3
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.F58DEB2156F89FEFF73AA296FB996076
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.6D9C05CCD6FC973CEC7EC06B2852C004
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.D2F89CEFD1D076C722E383CD47F54B0E
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.C3E90840E58C0ED58DE5285F1C629007
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.697EEA5D819D1CCED8356F3707A80CC0
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.F8EE6F2C578B71C30A5818323B8C63DD
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.CEC966DCE333C324697D8D2C014D5844
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.F75FB6B03E5778971F18B3E156F1C368
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.FB37D5236F8B9FAA3C23E1E1ED75EF7B
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.BF2D067D7C365D5CD381837F5376FBCC
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.1DCB7A5A19D0073AB4C529DE68EFF590
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.AE62860962CCA944FDE6BA7A1FA29BE8
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.74228C20E21299810A61BDC1D72981CB
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.5A5F6DB8F16E8BA2850BBC9786BF933A
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.9B0188827522EB7EEF626D5957F16EF7
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.057E27FF0C7515839363635ABD87011B
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.815B2098111720A02816432FD0B11263
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.F17A1AE113B6FD07741496A307004005
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.3FD065CF09A9346481021BC207729FC8
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.67E8651E0A9719E5AA68ED2783076E40
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.E7E2388B6F0182AA8B191836FDC2EF46
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.0B3D7131A8DE08F23DD8F92E4FB445CE
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.E016A3B10B9E2BCFBDC052B607BEF218
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.405BE0983E37E87E48E0D549DFFD22CD
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.D55B0B0605B4BF3AE32149CAD90110CE
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.89699929828B8EC5AE4F1F41003A1293
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.02BC732C912E22F368101E336487D192
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.58AA7409654D5D50B6B8F55539C0774B
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.6B1CA505E2302A04F63AA27099F0FCFF
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.96BE833135E4A5C8F7DE52803D85CCA1
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.3FB901D98106C97ED02FBD2FCB72A368
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.06D8300D0336BED673A20EF70C44CCCF
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.C9293362A29ABB99EA2382C6B71D0649
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.351B871DFA1717273590632B38A4EBC7
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.6DFABCE265B2455CBE17C12CF01CDDD2
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.C4D2CFB128128AFB2C46656E3EE693BB
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.9222B04244CF97705F3488EE7906FC41
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.AA0AD23B2F046B29AA8947581FDFB860
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.D74722063AC27B6ECB815A78C0E790C5
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.E5ACD5AF9B1F02DDDF00504737297CFD
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.8E55D6D6105C140A17D3438C4F4D8D54
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.86F306440C4799F14DD5F4C0719E507C
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.301D82C48DB6624F74A9A6BD309A9E40
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.098207A6680442B3FA9643BB23CB370E
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.0EC16E3818F17C785ABB022A34A7A083
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.F1ED654859180EBA33AD4697444A2636
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.84A39C5D428089EF64F5DB0558FA2C45
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.4C47E15164EB0479D97A05D6AF22162F
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.DCC71D6527639A19725050B37CB780DE
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.51440FCFF3ADF3834C5500FF554FFBE2
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.70D55191E13B0C77A5FF57DA126E0849
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.590227D4C2600E9BBB7B5082E4792767
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.445C035DB924A9B0FA42C72674EE4F89
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.AA975962B34D69CE1C78662B13A65313
+      
+      
+        
+          
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.BDA9A7110ADB2C122D0CA705B6B1D6FA
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.C861C307A1307B7D19E67CEF0C9FF674
+      
+      
+        
+          
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.9AC8B3AFD901820523B5E22441984736
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.E7BF64005D511F7DF092A1FD63CAFDB2
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.6DA60D3A441DEB83245CF9390DB652C2
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.FCB7A1CCECDBFE60B9F9268A4DA13E64
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.5EDB52181C6563D67BB04B9F6F9FFCB3
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.602C9239FB1B18A48D8E6AB920F9865E
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.613F01620D33F44F7F08DAD27477860D
+      
+      
+        
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.B3F196534F6B3239F608E60F46CC5054
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.9395090644FD6A9EEDE620F2EDB2659C
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.51360490B8BC0FEA841584D3A7941DA9
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.21D484F75EB67BB6E52299CFE9026E8C
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.B470C7165F3CC8ED799ED4F8B5BAEA45
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.C56316687F31CD493F7064FD951EC99C
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.79EEE017A84D0C323D68CA23A8CB1076
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.A4AFD3D5DA908492647C739E941CC97C
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.EF17F568146C914F8023380045758F9C
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.3D539F4D333D3240C489C26EC0A011EF
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.B0E4F8E2EF4C7507621D37BF97395F31
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.A426A6A86571D4DB76768FBF9E3540DF
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.BB0DDF53B3466065E0CC36A0EDFF2834
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.25C2373230B0EA43C94F0DCAAAEEA6FB
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.179F228C42340ABB9EEB830006BC5F2A
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.3DDC7451EA1620AE50203012113A61CC
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.28359B909678D42A4868FB82F28934BB
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.486CB0412BAD44C82290FBD64DEC1F6D
+      
+      
+        
+          
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.C2B1E38A255F67B820C6DB2C5A021FC5
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.AA7061BA6F294ACDB2FA2282E0742BCC
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.A5E2220D36BE184AE56CCD3CFC42D0AE
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.F6BC5F562CC7CDDF11632321B21D97DF
+      
+      
+        
+          
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.97CAA19C72182CEF847821F9E6FB3AAA
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.3B8F9F9CA8F1879985BF29BF5CBD21E6
+      
+      
+        
+          
+            
+            
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.E05E227E544680F0B4BB26DE70617352
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.56E9BDA78930F90CD41C280B94F3EC92
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.A7B51CF0108CAE7FF2ED2ECFEEF990D9
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.02F4797EB106FFD0157EC4EAB8FBB3BC
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.6D17F999F7BD299E10604D67FB269746
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.FC523C8224CB7FBABCF213D5D951CB18
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.0647B93684A5AF9B4EF9AAFC7148E0C7
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.04BA526681D926E6F55834C3FF14C800
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.C7CE8CF71088F3DB585CD3B02AD6C01E
+      
+      
+        
+          
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.2861D535C9E907B2316C6DD326F9C9DB
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.B3C891ADBE07985D8EB07F47AD5B9DA9
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.574C8D1B7983E679131E37FD5C3DF1B3
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.75E4BA966FFC501D192F53E91B7DA97C
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.CB9A35E0D20D8F23F42BE689119E06FA
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.D7B16FCF4CB54A4DDD0B7B96965B0C89
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.6C78CB68979C1B30B65792A51C4909B4
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            503
+            
+              
+            
+          
+        
+        1.1425C7EF3EB1C555EEDB3A7B83959C18
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.6C71667DFCE4EFE9D2D594675F5D0548
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.3FA058A0E73CD716458E4AD473D01D8E
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.7A7A1F423942BCD065A53F5E2C08CA10
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.686D026F4E37237A3FB0CFDDADBC9F8A
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.285B3B8B3249390AC0EA62095620648D
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.483EDC95D9050419855BB2C9B7D9395A
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.57134A2893313DE704198EED54D5B6E6
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.9193C0F29388C5B9B6583A6BF6C2B347
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.424FC52414BFE0A49B6746D5DB1256AB
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.13C8280BFD2D49B31E703A6C942A14BB
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.2ED8A733A34F7268A165FCB3C4DE0232
+      
+      
+        
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.BCD510DF25513B8FAF0352336F094053
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.D24D24F09DCA78579C08ADE1D5528A18
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.388A14B11DC4161D76E7BA3524CED336
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.E8ECE9BA39F0B03F8394B4AB5065B60D
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.67C819CC3E37319CB3D533A2D31F00D7
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.7A9D46F2B10E01B201BD25BF3DA3E3D6
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.466E11BA31409FC6549BB27A0C1400CC
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.5458586BDBC05CE6DC7B96FB305E279E
+      
+      
+        
+          
+            
+            
+          
+          
+            
+            
+            
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.221F7DCAF04AE6EB1BADF4997871733E
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.2C5C28BB2FF03BAE5647F7DA4B0FBC7C
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.EF0CC59153D7673ED206E0F310880A6E
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.D946E977B16DE185BC7065F3C36BBA94
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.6758EF1C334FA94EC51F836A229DFFCA
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.D96F1DC7CD390C447BBA6A55F52476AC
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.A7D83C4D167D0650C16DF8831A5503C4
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.D9690323F9BD00A1D7AC14D449C5C5F5
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.2BDEFEC679C5FBAF0D8D2934FE55F708
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.2ADB64C9DC3241D0FFE93910210E1FF9
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.4ABF850B6F44B9FF493B4AD55E1CDA35
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.F0DCE0BD19F895084F3C461358ABB31B
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.362B98FD68A372E063F679FC32F89B31
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.CB7932D0E8A306C5A2E35C8916C0E4B4
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.E62DED290400027E754EB45DBAC5062D
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.DBBD22390D6F477450A6966FDA2CA298
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.FED65C1D6BB97C21B4F9DA0B7A997D35
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.44F69227AF5D5C013E4E8870692550C7
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.6B2469AA8D0FC2655F6EB394348EB437
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.A0A9BFEABE8E3C638E51660CF67D0B3C
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            500
+            
+              
+            
+          
+        
+        1.642E5DBD51439CCB994541FC44A20E26
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.FF1951EAE0ACBAB8DB513DF209D32122
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.B73F07D16A191E0EFD397E3DCB5CE051
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.31CBB28C0277768C8BF04A6CF1BA98A5
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.E6B45FD1B7595DCC123636DD01E50D8A
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.AD707ECC69A1E835FADE9B7B896C7C53
+      
+      
+        
+          
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.0C6CA5CFC56E62E1386738F250C749D4
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.78750072F5ADD3E16FA02549EED9F0AF
+      
+      
+        
+          
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.94A9B934CC18AB22E125F70FBB9F82DE
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.038E34A9C374FC176986FC568A165527
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.CA5A1C1C65E431A37F3207B81767774B
+      
+      
+        
+          
+            
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.7D48EAAB19465D9008EFE5C6E3C020EA
+      
+      
+        
+          
+            
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.3669C6C2F1550E1554DB9CF129DCD965
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            207
+            
+              
+            
+          
+        
+        1.A6D31BBFAFCCC9A55B9FEBF72E1C2953
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.32E4082ED243CB134EA743FF78000BD5
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.38E575DC2AF4B7D7CF3203E6B7BDA74D
+      
+      
+        
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.220648A0923367BC4FA2794FA4BE4591
+      
+      
+        
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.38448C20E04942C66F6D9AB3169CEF09
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.B7BB4C29193AC92DB35E9E206A01E30D
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.4D5F15EE9801D7E3BE0892FD38AFDE0D
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.E062F9FB6D3FA8A9EEE31C6A81CC6604
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.78F8ED04EC79A126AEADE8C08B0E4F97
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.958F111DBDA1BCE46EFBC10511A145E9
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.1E72B20786B3BFB6DE896C380864C4ED
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.A62A0E7E1808EED079989D04772B6CB6
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.65E7DE349575885F8EC2E78FFBBD3484
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.17A61EEDF79D76747BE37B3DA3F698DF
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.F0EE6E4715A3D2CFB1F5795EA52EF0A1
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.59DE4FDFB535F952488C8C76082D1957
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.AC6CF1A6EDCCAC74410543D331DCEBFC
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.83B487198471CDEAF5C95D371BE1227E
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.B13DC16C4369B842DD1580DFC9A23E90
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.D9B3DB52DF8218B45D48BECB5131915E
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.CF52BFFDF3E12D43EFD5F614833B7DCE
+      
+      
+        
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.FF43777C8C33E41102BCF3F2A2A90C64
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.7B3676D45CECCC1D12D71EBA722FFF30
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.71B0E6E68AA99863BE55E1D6D037301B
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.9F2F0F9EE78721BF80AD83701F9736A0
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.680A6E46ADDF391752A3A89F95D73448
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.496A9334045EC97BFFE3A5116694FF85
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.BC1C6F682219387D81896AD542BFDA1B
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.21E756DE8577C5778B49EB14777DE712
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.9066576F0656AE6BA8E13A019F6A59AB
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.444CF76C0D4A6C4E637033A4D4E9126A
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.3BFB533872841D46A011D369D3FF2977
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.3426C46B6878DDEC0DA8DA6A0D5554AA
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.068DA432A58C3D7331CA5C906158F229
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.B316331E65CCB79F118C8A178C4D1112
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+            
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.27B9DEC971AF1E0443505F6381B2C647
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.F528AE4144225E113F7B00825EE52AA2
+      
+      
+        
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.8D68C314C554FFCC800097F48EA2B2BF
+      
+      
+        
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.B29CE6CAE05044347C6F6D2CD82D2B76
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.0EDA4B68A5E0A7A119DF093167A9643B
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.25F94118327D5D07CF4BD4214EE33EC0
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.AD3D66C9BD4CFDABA55559C35CECD9E1
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.E336292FC529AF487A9AD66216D49654
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.673FA4C32058D8D329E1630ADA8E73C1
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.B74A7AA44043A326AC2D7D7934967B14
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.75F9BD52CF2A252C3696A912624AF4E7
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.1568E0AA2E38BC533DF1D9A3C26D9A80
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.DA7E3AB4DC966A21CCA466E87760769B
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.AF491ABBC242A023D510DB3FEFC8D333
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.D0BF3B5963590A886B4DECEFF9D5DA2C
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.658474BB59B4C339A45CF52CB7F697C3
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.4F8C9F278402E6FB2E4EA057B834CC73
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.251760D30B95DDFD686CF5F8DD244B34
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+          
+          
+            
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+        
+        1.0FC6240C9AA47CA11A06C45AF5EED527
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.AA72044B9C3357E0774AADA4EF555CC6
+      
+      
+        
+          
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            409
+            
+              
+            
+          
+          
+            500
+            
+              
+            
+          
+        
+        1.6BCFED905208FF2C3021E49E1113053C
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            500
+            
+              
+            
+          
+        
+        1.E7E6381C3DCFF65F04EFF2E13C486B02
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.BCF75A4FA6D197563529575007325A9D
+      
+      
+        
+          
+            
+            
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.4D283CDE0658BDE321C6C7B4CFB00DB5
+      
+      
+        
+          
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+        
+        1.10D8DE9FF8DAD881A8553CC2EA0B33D8
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.E49AE7C4E4606611A2264F17092383DE
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.AF255C11E7836E705C16FB2E0C5C5893
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.EDD41D719CF5C94BEBC543ED4E1FA92B
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.87C46F7702591A04073B685E3DCA18E8
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.DF34657F1F638BF55CCB59D8766A7A40
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.227468E94768C87DC18B9D82AE1DD73A
+      
+      
+        
+          
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.A11EBAEA1FB195A00CA8C77DAA86896E
+      
+      
+        
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.ED42C99A3F7707AE33EA46D9155ACF67
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.CB843F4F8FA5F1FD7A435EF77E9AB382
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.55C741CC9A0D6215D57FB954E57DBF9E
+      
+      
+        
+          
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+        
+        1.A37E5119D3567CC0420518DD48F43AC6
+      
+      
+        
+          
+            
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+        
+        1.14B1920F8A49D1721022BDA3054DF8EB
+      
+      
+        
+          
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+        
+        1.BB98871AF82585B0C646B6954309CBC3
+      
+      
+        
+          
+            
+            
+            
+            
+            
+          
+          
+            
+          
+        
+        
+          
+            201
+            
+              
+            
+          
+          
+            400
+            
+              
+            
+          
+          
+            403
+            
+              
+            
+          
+        
+        1.577E0E6FD00116FDB9FCC4BA96BC82E6
+      
+      
+        
+          
+          
+        
+        
+          
+            204
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.EA981FC342B0BC7143D8A6B5C650CBA3
+      
+    
+    
+      
+        
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+              
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+          
+            
+              
+              
+              
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+              
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+            
+          
+          
+            
+          
+          
+            
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+              
+                
+                  
+                
+              
+              
+                
+                  
+                  
+                
+              
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+              
+                
+                  
+                  
+                  
+                
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+          
+            
+          
+        
+        
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+          
+            
+              
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+          
+            
+          
+        
+      
+      
+        
+        
+          
+          
+          
+          
+          
+        
+      
+      
+        
+          
+            
+          
+          
+            
+          
+        
+        
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+          
+            
+              
+              
+            
+          
+        
+      
+    
+  
+
\ No newline at end of file
diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index d7c22b50a..8f6f54212 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -35,7 +35,6 @@
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.channels.FileChannel;
-import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index 81aeba046..67a96a7ac 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -895,6 +895,7 @@ ModifyJobSpectraS3Response modifyJobSpectraS3(final ModifyJobSpectraS3Request re
     ReplicatePutJobSpectraS3Response replicatePutJobSpectraS3(final ReplicatePutJobSpectraS3Request request)
             throws IOException;
 
+    @ResponsePayloadModel("MasterObjectList")
     @Action("MODIFY")
     @Resource("BUCKET")
     
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/BulkRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/BulkRequest.java
index 22984cdb4..4a2eb6465 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/BulkRequest.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/BulkRequest.java
@@ -34,8 +34,6 @@ public abstract class BulkRequest extends AbstractRequest {
     private InputStream stream;
     private long size;
     private Priority priority;
-    private WriteOptimization writeOptimization;
-    protected JobChunkClientProcessingOrderGuarantee chunkOrdering;
 
     public BulkRequest(final String bucket, final Iterable objects) {
         this.bucket = bucket;
@@ -44,11 +42,7 @@ public BulkRequest(final String bucket, final Iterable objects) {
 
     public BulkRequest withPriority(final Priority priority) {
         this.priority = priority;
-        return this;
-    }
-
-    public BulkRequest withWriteOptimization(final WriteOptimization writeOptimization) {
-        this.writeOptimization = writeOptimization;
+        this.updateQueryParam("priority", priority);
         return this;
     }
 
@@ -56,9 +50,6 @@ private InputStream generateStream() {
         final Ds3ObjectList objects =
                 new Ds3ObjectList();
         objects.setObjects(this.ds3Objects);
-        objects.setPriority(this.priority);
-        objects.setWriteOptimization(this.writeOptimization);
-        objects.setChunkClientProcessingOrderGuarantee(this.chunkOrdering);
         
         final StringBuilder xmlOutputBuilder = new StringBuilder();
         if (this.getCommand() == BulkCommand.PUT) {
@@ -108,8 +99,4 @@ public InputStream getStream() {
     public Priority getPriority() {
         return priority;
     }
-
-    public WriteOptimization getWriteOptimization() {
-        return writeOptimization;
-    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
index 609ad9de3..e61ce46b6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/StageObjectsJobSpectraS3ResponseParser.java
@@ -18,12 +18,10 @@
 
 import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
 import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
-import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Response;
 import com.spectralogic.ds3client.commands.spectrads3.StageObjectsJobSpectraS3Response;
 import com.spectralogic.ds3client.models.MasterObjectList;
 import com.spectralogic.ds3client.networking.WebResponse;
 import com.spectralogic.ds3client.serializer.XmlOutput;
-
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -36,13 +34,11 @@ public StageObjectsJobSpectraS3Response parseXmlResponse(final WebResponse respo
         if (ResponseParserUtils.validateStatusCode(statusCode, expectedStatusCodes)) {
             switch (statusCode) {
             case 200:
-                if (ResponseParserUtils.getSizeFromHeaders(response.getHeaders()) == 0) {
-                   return new StageObjectsJobSpectraS3Response(null, this.getChecksum(), this.getChecksumType());
-                }
                 try (final InputStream inputStream = response.getResponseStream()) {
                     final MasterObjectList result = XmlOutput.fromXml(inputStream, MasterObjectList.class);
                     return new StageObjectsJobSpectraS3Response(result, this.getChecksum(), this.getChecksumType());
                 }
+
             default:
                 assert false: "validateStatusCode should have made it impossible to reach this line";
             }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
index dfe2b4eb1..7c1b5a268 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/EjectStorageDomainBlobsSpectraS3Request.java
@@ -28,7 +28,6 @@
 import java.nio.charset.Charset;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
 import com.google.common.net.UrlEscapers;
-import java.util.UUID;
 
 public class EjectStorageDomainBlobsSpectraS3Request extends AbstractRequest {
 
@@ -131,4 +130,5 @@ public String getEjectLocation() {
         return this.ejectLocation;
     }
 
+
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllPoolsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllPoolsSpectraS3Request.java
index 235dbaf53..e78f30c52 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllPoolsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllPoolsSpectraS3Request.java
@@ -18,16 +18,14 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.google.common.net.UrlEscapers;
 import java.util.UUID;
+import com.google.common.net.UrlEscapers;
 import com.spectralogic.ds3client.models.Priority;
 
 public class ImportAllPoolsSpectraS3Request extends AbstractRequest {
 
     // Variables
     
-    private String conflictResolutionMode;
-
     private String dataPolicyId;
 
     private Priority priority;
@@ -49,13 +47,6 @@ public ImportAllPoolsSpectraS3Request() {
 
     }
 
-    public ImportAllPoolsSpectraS3Request withConflictResolutionMode(final String conflictResolutionMode) {
-        this.conflictResolutionMode = conflictResolutionMode;
-        this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
-        return this;
-    }
-
-
     public ImportAllPoolsSpectraS3Request withDataPolicyId(final UUID dataPolicyId) {
         this.dataPolicyId = dataPolicyId.toString();
         this.updateQueryParam("data_policy_id", dataPolicyId);
@@ -130,11 +121,6 @@ public String getPath() {
         return "/_rest_/pool";
     }
     
-    public String getConflictResolutionMode() {
-        return this.conflictResolutionMode;
-    }
-
-
     public String getDataPolicyId() {
         return this.dataPolicyId;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllTapesSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllTapesSpectraS3Request.java
index a5050a9a0..2120c7cf7 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllTapesSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportAllTapesSpectraS3Request.java
@@ -18,16 +18,14 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.google.common.net.UrlEscapers;
 import java.util.UUID;
+import com.google.common.net.UrlEscapers;
 import com.spectralogic.ds3client.models.Priority;
 
 public class ImportAllTapesSpectraS3Request extends AbstractRequest {
 
     // Variables
     
-    private String conflictResolutionMode;
-
     private String dataPolicyId;
 
     private Priority priority;
@@ -49,13 +47,6 @@ public ImportAllTapesSpectraS3Request() {
 
     }
 
-    public ImportAllTapesSpectraS3Request withConflictResolutionMode(final String conflictResolutionMode) {
-        this.conflictResolutionMode = conflictResolutionMode;
-        this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
-        return this;
-    }
-
-
     public ImportAllTapesSpectraS3Request withDataPolicyId(final UUID dataPolicyId) {
         this.dataPolicyId = dataPolicyId.toString();
         this.updateQueryParam("data_policy_id", dataPolicyId);
@@ -130,11 +121,6 @@ public String getPath() {
         return "/_rest_/tape";
     }
     
-    public String getConflictResolutionMode() {
-        return this.conflictResolutionMode;
-    }
-
-
     public String getDataPolicyId() {
         return this.dataPolicyId;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportPoolSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportPoolSpectraS3Request.java
index 4addf085a..91a744817 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportPoolSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportPoolSpectraS3Request.java
@@ -18,8 +18,8 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.google.common.net.UrlEscapers;
 import java.util.UUID;
+import com.google.common.net.UrlEscapers;
 import com.spectralogic.ds3client.models.Priority;
 
 public class ImportPoolSpectraS3Request extends AbstractRequest {
@@ -28,8 +28,6 @@ public class ImportPoolSpectraS3Request extends AbstractRequest {
     
     private final String pool;
 
-    private String conflictResolutionMode;
-
     private String dataPolicyId;
 
     private Priority priority;
@@ -52,13 +50,6 @@ public ImportPoolSpectraS3Request(final String pool) {
 
     }
 
-    public ImportPoolSpectraS3Request withConflictResolutionMode(final String conflictResolutionMode) {
-        this.conflictResolutionMode = conflictResolutionMode;
-        this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
-        return this;
-    }
-
-
     public ImportPoolSpectraS3Request withDataPolicyId(final UUID dataPolicyId) {
         this.dataPolicyId = dataPolicyId.toString();
         this.updateQueryParam("data_policy_id", dataPolicyId);
@@ -138,11 +129,6 @@ public String getPool() {
     }
 
 
-    public String getConflictResolutionMode() {
-        return this.conflictResolutionMode;
-    }
-
-
     public String getDataPolicyId() {
         return this.dataPolicyId;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportTapeSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportTapeSpectraS3Request.java
index b126327ef..723b3e2c0 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportTapeSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ImportTapeSpectraS3Request.java
@@ -18,8 +18,8 @@
 
 import com.spectralogic.ds3client.networking.HttpVerb;
 import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
-import com.google.common.net.UrlEscapers;
 import java.util.UUID;
+import com.google.common.net.UrlEscapers;
 import com.spectralogic.ds3client.models.Priority;
 
 public class ImportTapeSpectraS3Request extends AbstractRequest {
@@ -28,8 +28,6 @@ public class ImportTapeSpectraS3Request extends AbstractRequest {
     
     private final String tapeId;
 
-    private String conflictResolutionMode;
-
     private String dataPolicyId;
 
     private Priority priority;
@@ -60,13 +58,6 @@ public ImportTapeSpectraS3Request(final String tapeId) {
 
     }
 
-    public ImportTapeSpectraS3Request withConflictResolutionMode(final String conflictResolutionMode) {
-        this.conflictResolutionMode = conflictResolutionMode;
-        this.updateQueryParam("conflict_resolution_mode", conflictResolutionMode);
-        return this;
-    }
-
-
     public ImportTapeSpectraS3Request withDataPolicyId(final UUID dataPolicyId) {
         this.dataPolicyId = dataPolicyId.toString();
         this.updateQueryParam("data_policy_id", dataPolicyId);
@@ -146,11 +137,6 @@ public String getTapeId() {
     }
 
 
-    public String getConflictResolutionMode() {
-        return this.conflictResolutionMode;
-    }
-
-
     public String getDataPolicyId() {
         return this.dataPolicyId;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
index 62d5f5adc..62c7a27e4 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Request.java
@@ -17,22 +17,19 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.BulkCommand;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
 import com.spectralogic.ds3client.commands.interfaces.BulkRequest;
+import com.google.common.net.UrlEscapers;
 import com.spectralogic.ds3client.models.Priority;
-import com.spectralogic.ds3client.models.bulk.Ds3Object;
-import com.spectralogic.ds3client.networking.HttpVerb;
 
 public class StageObjectsJobSpectraS3Request extends BulkRequest {
 
-    // Variables
 
+    
     private String name;
 
-    private Priority priority;
-
     // Constructor
     
-    
     public StageObjectsJobSpectraS3Request(final String bucketName, final Iterable objects) {
         super(bucketName, objects);
         
@@ -47,32 +44,22 @@ public StageObjectsJobSpectraS3Request withName(final String name) {
     }
 
 
+    @Override
     public StageObjectsJobSpectraS3Request withPriority(final Priority priority) {
-        this.priority = priority;
-        this.updateQueryParam("priority", priority);
+        super.withPriority(priority);
         return this;
     }
 
 
 
-    @Override
-    public HttpVerb getVerb() {
-        return HttpVerb.PUT;
-    }
-
-    @Override
-    public BulkCommand getCommand() {
-        return BulkCommand.GET;
-    }
-
-
+    
     public String getName() {
         return this.name;
     }
 
 
-    public Priority getPriority() {
-        return this.priority;
+    @Override
+    public BulkCommand getCommand() {
+        return BulkCommand.GET;
     }
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
index 1eb1b3e47..c7388485b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/StageObjectsJobSpectraS3Response.java
@@ -16,15 +16,21 @@
 // This code is auto-generated, do not modify
 package com.spectralogic.ds3client.commands.spectrads3;
 
-import com.spectralogic.ds3client.commands.interfaces.BulkResponse;
+import com.spectralogic.ds3client.models.MasterObjectList;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
-import com.spectralogic.ds3client.models.MasterObjectList;
 
-public class StageObjectsJobSpectraS3Response extends BulkResponse {
+public class StageObjectsJobSpectraS3Response extends AbstractResponse {
     
+    private final MasterObjectList masterObjectListResult;
+
     public StageObjectsJobSpectraS3Response(final MasterObjectList masterObjectListResult, final String checksum, final ChecksumType.Type checksumType) {
-        super(masterObjectListResult, checksum, checksumType);
+        super(checksum, checksumType);
+        this.masterObjectListResult = masterObjectListResult;
+    }
+
+    public MasterObjectList getMasterObjectListResult() {
+        return this.masterObjectListResult;
     }
 
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
index f4abf11d7..6851e22e8 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java
@@ -79,7 +79,7 @@ private static class BlobIdentityDecorator {
 
         private final PhysicalPlacement physicalPlacement;
 
-        private final long version;
+        private final UUID versionId;
 
         private BlobIdentityDecorator(final BulkObject blob) {
             this.bucket = blob.getBucket();
@@ -89,7 +89,7 @@ private BlobIdentityDecorator(final BulkObject blob) {
             this.name = blob.getName();
             this.offset = blob.getOffset();
             this.physicalPlacement = blob.getPhysicalPlacement();
-            this.version = blob.getVersion();
+            this.versionId = blob.getVersionId();
         }
 
         @Override
@@ -102,7 +102,7 @@ public boolean equals(final Object o) {
             if (latest != that.latest) return false;
             if (length != that.length) return false;
             if (offset != that.offset) return false;
-            if (version != that.version) return false;
+            if (versionId != null ? !versionId.equals(that.versionId) : that.versionId != null) return false;
             if (bucket != null ? !bucket.equals(that.bucket) : that.bucket != null) return false;
             if (id != null ? !id.equals(that.id) : that.id != null) return false;
             if (name != null ? !name.equals(that.name) : that.name != null) return false;
@@ -111,7 +111,7 @@ public boolean equals(final Object o) {
 
         @Override
         public int hashCode() {
-            return java.util.Objects.hash(bucket, id, latest, length, name, offset, physicalPlacement, version);
+            return java.util.Objects.hash(bucket, id, latest, length, name, offset, physicalPlacement, versionId);
         }
 
         @Override
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BulkObject.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BulkObject.java
index e2d214aa2..8f321418a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BulkObject.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/BulkObject.java
@@ -50,8 +50,8 @@ public class BulkObject {
     @JsonProperty("PhysicalPlacement")
     private PhysicalPlacement physicalPlacement;
 
-    @JacksonXmlProperty(isAttribute = true, localName = "Version")
-    private long version;
+    @JacksonXmlProperty(isAttribute = true, localName = "VersionId")
+    private UUID versionId;
 
     // Constructor
     public BulkObject() {
@@ -132,18 +132,18 @@ public void setPhysicalPlacement(final PhysicalPlacement physicalPlacement) {
     }
 
 
-    public long getVersion() {
-        return this.version;
+    public UUID getVersionId() {
+        return this.versionId;
     }
 
-    public void setVersion(final long version) {
-        this.version = version;
+    public void setVersionId(final UUID versionId) {
+        this.versionId = versionId;
     }
 
 
     @Override
     public int hashCode() {
-        return java.util.Objects.hash(id, inCache, latest, length, name, offset, physicalPlacement, version);
+        return java.util.Objects.hash(id, inCache, latest, length, name, offset, physicalPlacement, versionId);
     }
 
     @Override
@@ -161,7 +161,7 @@ && nullableEquals(this.getInCache(), bulkObject.getInCache())
                 && nullableEquals(this.getName(), bulkObject.getName())
                 && this.getOffset() == bulkObject.getOffset()
                 && this.getPhysicalPlacement() == bulkObject.getPhysicalPlacement()
-                && this.getVersion() == bulkObject.getVersion();
+                && nullableEquals(this.getVersionId(), bulkObject.getVersionId());
     }
 
     /**
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
index 255436d8d..a7d8db4c3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DataPolicy.java
@@ -31,9 +31,6 @@ public class DataPolicy {
     @JsonProperty("AlwaysMinimizeSpanningAcrossMedia")
     private boolean alwaysMinimizeSpanningAcrossMedia;
 
-    @JsonProperty("AlwaysReplicateDeletes")
-    private boolean alwaysReplicateDeletes;
-
     @JsonProperty("BlobbingEnabled")
     private boolean blobbingEnabled;
 
@@ -64,9 +61,6 @@ public class DataPolicy {
     @JsonProperty("Id")
     private UUID id;
 
-    @JsonProperty("LtfsObjectNamingAllowed")
-    private boolean ltfsObjectNamingAllowed;
-
     @JsonProperty("MaxVersionsToKeep")
     private int maxVersionsToKeep;
 
@@ -104,15 +98,6 @@ public void setAlwaysMinimizeSpanningAcrossMedia(final boolean alwaysMinimizeSpa
     }
 
 
-    public boolean getAlwaysReplicateDeletes() {
-        return this.alwaysReplicateDeletes;
-    }
-
-    public void setAlwaysReplicateDeletes(final boolean alwaysReplicateDeletes) {
-        this.alwaysReplicateDeletes = alwaysReplicateDeletes;
-    }
-
-
     public boolean getBlobbingEnabled() {
         return this.blobbingEnabled;
     }
@@ -203,15 +188,6 @@ public void setId(final UUID id) {
     }
 
 
-    public boolean getLtfsObjectNamingAllowed() {
-        return this.ltfsObjectNamingAllowed;
-    }
-
-    public void setLtfsObjectNamingAllowed(final boolean ltfsObjectNamingAllowed) {
-        this.ltfsObjectNamingAllowed = ltfsObjectNamingAllowed;
-    }
-
-
     public int getMaxVersionsToKeep() {
         return this.maxVersionsToKeep;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedS3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedS3Object.java
index 2228e4f59..78e2cea3f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedS3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedS3Object.java
@@ -67,9 +67,6 @@ public class DetailedS3Object {
     @JsonProperty("Type")
     private S3ObjectType type;
 
-    @JsonProperty("Version")
-    private long version;
-
     // Constructor
     public DetailedS3Object() {
         //pass
@@ -202,13 +199,4 @@ public void setType(final S3ObjectType type) {
         this.type = type;
     }
 
-
-    public long getVersion() {
-        return this.version;
-    }
-
-    public void setVersion(final long version) {
-        this.version = version;
-    }
-
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Object.java
index 39ff6894a..bcb4fc37c 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3Object.java
@@ -44,9 +44,6 @@ public class S3Object {
     @JsonProperty("Type")
     private S3ObjectType type;
 
-    @JsonProperty("Version")
-    private long version;
-
     // Constructor
     public S3Object() {
         //pass
@@ -108,18 +105,9 @@ public void setType(final S3ObjectType type) {
     }
 
 
-    public long getVersion() {
-        return this.version;
-    }
-
-    public void setVersion(final long version) {
-        this.version = version;
-    }
-
-
     @Override
     public int hashCode() {
-        return Objects.hash(bucketId, name, id, latest, creationDate, type, version);
+        return Objects.hash(bucketId, name, id, latest, creationDate, type);
     }
 
     @Override
@@ -135,7 +123,6 @@ public boolean equals(final Object obj) {
                 && this.id.equals(s3obj.getId())
                 && this.latest == s3obj.getLatest()
                 && this.name.equals(s3obj.getName())
-                && this.type == s3obj.getType()
-                && this.version == s3obj.getVersion();
+                && this.type == s3obj.getType();
     }
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java
index e6f5d6565..b788855e9 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java
@@ -31,15 +31,6 @@ public class Ds3ObjectList {
     @JsonProperty("Object")
     private Stream objects;
 
-    @JacksonXmlProperty(isAttribute = true, namespace = "", localName = "Priority")
-    private Priority priority;
-
-    @JacksonXmlProperty(isAttribute = true, namespace = "", localName = "WriteOptimization")
-    private WriteOptimization writeOptimization;
-
-    @JacksonXmlProperty(isAttribute = true, namespace = "", localName = "ChunkClientProcessingOrderGuarantee")
-    private JobChunkClientProcessingOrderGuarantee chunkClientProcessingOrderGuarantee;
-
     public Ds3ObjectList() {
     }
 
@@ -54,28 +45,4 @@ public Iterable getObjects() {
     public void setObjects(final Iterable objects) {
         this.objects = StreamSupport.stream(objects.spliterator(), false);
     }
-
-    public Priority getPriority() {
-        return priority;
-    }
-
-    public void setPriority(final Priority priority) {
-        this.priority = priority;
-    }
-
-    public WriteOptimization getWriteOptimization() {
-        return this.writeOptimization;
-    }
-
-    public void setWriteOptimization(final WriteOptimization writeOptimization) {
-        this.writeOptimization = writeOptimization;
-    }
-
-    public JobChunkClientProcessingOrderGuarantee getChunkClientProcessingOrderGuarantee() {
-        return this.chunkClientProcessingOrderGuarantee;
-    }
-
-    public void setChunkClientProcessingOrderGuarantee(final JobChunkClientProcessingOrderGuarantee chunkClientProcessingOrderGuarantee) {
-        this.chunkClientProcessingOrderGuarantee = chunkClientProcessingOrderGuarantee;
-    }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index 615b0c33a..b075bc0da 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -86,7 +86,7 @@ public class Ds3Client_Test {
             new Ds3Object("file3")
     );
 
-    private static final String SIMPLE_BULK_OBJECT_LIST_RESPONSE = "";
+    private static final String SIMPLE_BULK_OBJECT_LIST_RESPONSE = "";
 
     @Before
     public void setTimeZone() {
@@ -221,7 +221,7 @@ public void getObjectsSpectraS3() throws IOException, ParseException {
         queryParams.put("bucket_id", bucketId);
 
         final String stringResponse = "" +
-                "a24d14f3-e2f0-4bfb-ab71-f99d5ef437452015-09-21T20:06:47.694Ze37c3ce0-12aa-4f54-87e3-42532aca0e5ebeowulf.txtDATA1a24d14f3-e2f0-4bfb-ab71-f99d5ef437452015-09-21T20:06:47.779Zdc628815-c723-4c4e-b68b-5f5d10f38af5sherlock_holmes.txtDATA1a24d14f3-e2f0-4bfb-ab71-f99d5ef437452015-09-21T20:06:47.772Z4f6985fd-fbae-4421-ba27-66fdb96187c5tale_of_two_cities.txtDATA1a24d14f3-e2f0-4bfb-ab71-f99d5ef437452015-09-21T20:06:47.696Z82c18910-fadb-4461-a152-bf714ae91b55ulysses.txtDATA1";
+                "a24d14f3-e2f0-4bfb-ab71-f99d5ef437452015-09-21T20:06:47.694Ze37c3ce0-12aa-4f54-87e3-42532aca0e5ebeowulf.txtDATAa24d14f3-e2f0-4bfb-ab71-f99d5ef437452015-09-21T20:06:47.779Zdc628815-c723-4c4e-b68b-5f5d10f38af5sherlock_holmes.txtDATAa24d14f3-e2f0-4bfb-ab71-f99d5ef437452015-09-21T20:06:47.772Z4f6985fd-fbae-4421-ba27-66fdb96187c5tale_of_two_cities.txtDATAa24d14f3-e2f0-4bfb-ab71-f99d5ef437452015-09-21T20:06:47.696Z82c18910-fadb-4461-a152-bf714ae91b55ulysses.txtDATA";
 
         final ImmutableMap responseHeaders = ImmutableMap.of(
                 "page-truncated", "0",
@@ -241,7 +241,6 @@ public void getObjectsSpectraS3() throws IOException, ParseException {
         beowulf.setId(UUID.fromString("e37c3ce0-12aa-4f54-87e3-42532aca0e5e"));
         beowulf.setName("beowulf.txt");
         beowulf.setType(S3ObjectType.DATA);
-        beowulf.setVersion(1);
 
         final S3Object notBeowulf = new S3Object();
         notBeowulf.setBucketId(UUID.fromString("a24d14f3-e2f0-4bfb-ab71-f99d5ef43745"));
@@ -249,7 +248,6 @@ public void getObjectsSpectraS3() throws IOException, ParseException {
         notBeowulf.setId(UUID.fromString("e37c3ce0-12aa-4f54-87e3-42532aca0e5e"));
         notBeowulf.setName("notBeowulf.txt");
         notBeowulf.setType(S3ObjectType.DATA);
-        notBeowulf.setVersion(1);
 
         assertThat(objects.size(), is(4));
         assertThat(s3ObjectExists(objects, beowulf), is(true));
@@ -1407,7 +1405,7 @@ public void getPhysicalPlacementForObjectsTest() throws IOException {
 
     @Test
     public void getPhysicalPlacementForObjectsWithFullDetailsTest() throws IOException {
-        final String responsePayload = "";
+        final String responsePayload = "";
         final String bucketName = "BucketName";
 
         final Map queryParams = new HashMap<>();
@@ -1445,7 +1443,7 @@ public void verifyPhysicalPlacementForObjectsTest() throws IOException {
 
     @Test
     public void verifyPhysicalPlacementForObjectsWithFullDetailsTest() throws IOException {
-        final String responsePayload = "false10000t1false5a7bb215-4aff-4806-b217-5fe01ade6a2c2e5b25fc-546e-45b0-951e-8f3d80bb7823PENDING_INSPECTIONfalse20000LTO5false";
+        final String responsePayload = "false10000t1false5a7bb215-4aff-4806-b217-5fe01ade6a2c2e5b25fc-546e-45b0-951e-8f3d80bb7823PENDING_INSPECTIONfalse20000LTO5false";
         final String bucketName = "BucketName";
 
         final Map queryParams = new HashMap<>();
@@ -1481,7 +1479,7 @@ public void ejectStorageDomainBlobsTest() throws IOException {
 
     @Test
     public void replicatePutJobTest() throws IOException {
-        final String responsePayload = "";
+        final String responsePayload = "";
         final String requestPayload = "This is the request payload content";
         final String bucketName = "BucketName";
 
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
index 948977f86..4520a141c 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Metadata_Test.java
@@ -63,7 +63,7 @@ public void testThatWeSendMetadataWithEachBlob() throws IOException {
         blob1.setLatest(true);
         blob1.setLength(2);
         blob1.setOffset(0);
-        blob1.setVersion(1);
+        blob1.setVersionId(UUID.randomUUID());
         blob1.setName(fileName);
 
         final BulkObject blob2 = new BulkObject();
@@ -73,7 +73,7 @@ public void testThatWeSendMetadataWithEachBlob() throws IOException {
         blob2.setLatest(true);
         blob2.setLength(2);
         blob2.setOffset(2);
-        blob2.setVersion(1);
+        blob2.setVersionId(UUID.randomUUID());
         blob2.setName(fileName);
 
         final Objects chunk = new Objects();
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
index 5e360d3df..657e61316 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/serializer/XmlOutput_Test.java
@@ -102,17 +102,6 @@ public void toXmlWithVersionId() {
         assertThat(result, is(expectedString));
     }
 
-    @Test
-    public void toXmlWithPriority() {
-        final String expectedString = "";
-        final List objectList = ImmutableList.of(new Ds3Object("file1", 12), new Ds3Object("file2", 5022)).asList();
-        final Ds3ObjectList ds3ObjectList = new Ds3ObjectList(objectList);
-        ds3ObjectList.setPriority(Priority.HIGH);
-        final String result = XmlOutput.toXml(ds3ObjectList, true);
-
-        assertThat(result, is(expectedString));
-    }
-
     @Test(expected = java.io.IOException.class)
     public void fromXmlWithInvalidElementThrowsExceptionInDevBuild() throws IOException {
         assumeFalse(XmlOutput.isProductionBuild());

From cb38f09f46f57e4a242083cca2995831f67e01b5 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 9 Mar 2018 11:54:21 -0700
Subject: [PATCH 152/229] Adding undelete object command

---
 contract/5_0_x_contract.xml                   | 32 +++++++
 .../com/spectralogic/ds3client/Ds3Client.java |  7 ++
 .../spectralogic/ds3client/Ds3ClientImpl.java |  4 +
 ...UndeleteObjectSpectraS3ResponseParser.java | 49 +++++++++++
 .../UndeleteObjectSpectraS3Request.java       | 86 +++++++++++++++++++
 .../UndeleteObjectSpectraS3Response.java      | 36 ++++++++
 6 files changed, 214 insertions(+)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/UndeleteObjectSpectraS3ResponseParser.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/UndeleteObjectSpectraS3Request.java
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/UndeleteObjectSpectraS3Response.java

diff --git a/contract/5_0_x_contract.xml b/contract/5_0_x_contract.xml
index 995e50815..2eead26af 100644
--- a/contract/5_0_x_contract.xml
+++ b/contract/5_0_x_contract.xml
@@ -4602,6 +4602,32 @@
         
         1.BDA9A7110ADB2C122D0CA705B6B1D6FA
       
+      
+        
+          
+            
+          
+          
+            
+            
+          
+        
+        
+          
+            200
+            
+              
+            
+          
+          
+            404
+            
+              
+            
+          
+        
+        1.94C075BD3FE76604F8ED084C9B9E4AE5
+      
       
         
           
@@ -5442,6 +5468,12 @@
               
             
           
+          
+            400
+            
+              
+            
+          
         
         1.2861D535C9E907B2316C6DD326F9C9DB
       
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
index 67a96a7ac..3aeefa580 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java
@@ -1379,6 +1379,13 @@ GetPhysicalPlacementForObjectsSpectraS3Response getPhysicalPlacementForObjectsSp
     GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Response getPhysicalPlacementForObjectsWithFullDetailsSpectraS3(final GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request request)
             throws IOException;
 
+    @ResponsePayloadModel("S3Object")
+    @Action("BULK_MODIFY")
+    @Resource("OBJECT")
+    
+    UndeleteObjectSpectraS3Response undeleteObjectSpectraS3(final UndeleteObjectSpectraS3Request request)
+            throws IOException;
+
     @ResponsePayloadModel("PhysicalPlacement")
     @Action("SHOW")
     @Resource("BUCKET")
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
index 425e4e8d0..e8b752e61 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java
@@ -858,6 +858,10 @@ public GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Response getPhysica
         return new GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3ResponseParser().response(this.netClient.getResponse(request));
     }
     @Override
+    public UndeleteObjectSpectraS3Response undeleteObjectSpectraS3(final UndeleteObjectSpectraS3Request request) throws IOException {
+        return new UndeleteObjectSpectraS3ResponseParser().response(this.netClient.getResponse(request));
+    }
+    @Override
     public VerifyPhysicalPlacementForObjectsSpectraS3Response verifyPhysicalPlacementForObjectsSpectraS3(final VerifyPhysicalPlacementForObjectsSpectraS3Request request) throws IOException {
         return new VerifyPhysicalPlacementForObjectsSpectraS3ResponseParser().response(this.netClient.getResponse(request));
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/UndeleteObjectSpectraS3ResponseParser.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/UndeleteObjectSpectraS3ResponseParser.java
new file mode 100644
index 000000000..709f25eb7
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/UndeleteObjectSpectraS3ResponseParser.java
@@ -0,0 +1,49 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.parsers;
+
+import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
+import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
+import com.spectralogic.ds3client.commands.spectrads3.UndeleteObjectSpectraS3Response;
+import com.spectralogic.ds3client.models.S3Object;
+import com.spectralogic.ds3client.networking.WebResponse;
+import com.spectralogic.ds3client.serializer.XmlOutput;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class UndeleteObjectSpectraS3ResponseParser extends AbstractResponseParser {
+    private final int[] expectedStatusCodes = new int[]{200};
+
+    @Override
+    public UndeleteObjectSpectraS3Response parseXmlResponse(final WebResponse response) throws IOException {
+        final int statusCode = response.getStatusCode();
+        if (ResponseParserUtils.validateStatusCode(statusCode, expectedStatusCodes)) {
+            switch (statusCode) {
+            case 200:
+                try (final InputStream inputStream = response.getResponseStream()) {
+                    final S3Object result = XmlOutput.fromXml(inputStream, S3Object.class);
+                    return new UndeleteObjectSpectraS3Response(result, this.getChecksum(), this.getChecksumType());
+                }
+
+            default:
+                assert false: "validateStatusCode should have made it impossible to reach this line";
+            }
+        }
+
+        throw ResponseParserUtils.createFailedRequest(response, expectedStatusCodes);
+    }
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/UndeleteObjectSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/UndeleteObjectSpectraS3Request.java
new file mode 100644
index 000000000..353eea17f
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/UndeleteObjectSpectraS3Request.java
@@ -0,0 +1,86 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.spectrads3;
+
+import com.spectralogic.ds3client.networking.HttpVerb;
+import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.google.common.net.UrlEscapers;
+import java.util.UUID;
+
+public class UndeleteObjectSpectraS3Request extends AbstractRequest {
+
+    // Variables
+    
+    private final String bucketId;
+
+    private final String name;
+
+    private String versionId;
+
+    // Constructor
+    
+    
+    public UndeleteObjectSpectraS3Request(final String bucketId, final String name) {
+        this.bucketId = bucketId;
+        this.name = name;
+        
+        this.updateQueryParam("bucket_id", bucketId);
+
+        this.updateQueryParam("name", name);
+
+    }
+
+    public UndeleteObjectSpectraS3Request withVersionId(final UUID versionId) {
+        this.versionId = versionId.toString();
+        this.updateQueryParam("version_id", versionId);
+        return this;
+    }
+
+
+    public UndeleteObjectSpectraS3Request withVersionId(final String versionId) {
+        this.versionId = versionId;
+        this.updateQueryParam("version_id", versionId);
+        return this;
+    }
+
+
+
+    @Override
+    public HttpVerb getVerb() {
+        return HttpVerb.PUT;
+    }
+
+    @Override
+    public String getPath() {
+        return "/_rest_/object";
+    }
+    
+    public String getBucketId() {
+        return this.bucketId;
+    }
+
+
+    public String getName() {
+        return this.name;
+    }
+
+
+    public String getVersionId() {
+        return this.versionId;
+    }
+
+}
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/UndeleteObjectSpectraS3Response.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/UndeleteObjectSpectraS3Response.java
new file mode 100644
index 000000000..a271af52f
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/UndeleteObjectSpectraS3Response.java
@@ -0,0 +1,36 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+// This code is auto-generated, do not modify
+package com.spectralogic.ds3client.commands.spectrads3;
+
+import com.spectralogic.ds3client.models.S3Object;
+import com.spectralogic.ds3client.models.ChecksumType;
+import com.spectralogic.ds3client.commands.interfaces.AbstractResponse;
+
+public class UndeleteObjectSpectraS3Response extends AbstractResponse {
+    
+    private final S3Object s3ObjectResult;
+
+    public UndeleteObjectSpectraS3Response(final S3Object s3ObjectResult, final String checksum, final ChecksumType.Type checksumType) {
+        super(checksum, checksumType);
+        this.s3ObjectResult = s3ObjectResult;
+    }
+
+    public S3Object getS3ObjectResult() {
+        return this.s3ObjectResult;
+    }
+
+}
\ No newline at end of file

From 1d210abf61c32a86054c7162966e3324faa8c22c Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Thu, 15 Mar 2018 13:23:34 -0600
Subject: [PATCH 153/229] Getting the 5.0 sdk ready for a beta release

---
 build.gradle                             |  12 +++++++-----
 gradle/wrapper/gradle-wrapper.jar        | Bin 54783 -> 54708 bytes
 gradle/wrapper/gradle-wrapper.properties |   3 +--
 gradlew                                  |   6 +++---
 4 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/build.gradle b/build.gradle
index 1b9e50da4..dc50edb0e 100644
--- a/build.gradle
+++ b/build.gradle
@@ -23,19 +23,20 @@ buildscript {
 
     dependencies {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
-        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.7.3'
-        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
+        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.0'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.2'
+        classpath "gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.2.0-beta"
     }
 }
 
 
 plugins {
-    id "com.palantir.git-version" version "0.10.0"
+    id "com.palantir.git-version" version "0.10.1"
 }
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '4.0.1'
+    version = '5.0.0'
 }
 
 subprojects {
@@ -43,6 +44,7 @@ subprojects {
     apply plugin: 'java'
     apply plugin: 'kotlin'
     apply plugin: 'findbugs'
+    apply plugin: 'net.ossindex.audit'
 
     sourceCompatibility = JavaVersion.VERSION_1_8
     repositories {
@@ -62,7 +64,7 @@ subprojects {
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '4.1'
+    gradleVersion = '4.6'
 }
 
 project(':ds3-sdk') {
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index 1454129915e967f2239442034b31dae2c3787c80..7a3265ee94c0ab25cf079ac8ccdf87f41d455d42 100644
GIT binary patch
delta 17061
zcmY(qV{qn8u)rJJwrv}mjcwbu`QK!dCr&oD&5ezXH|ECJ*tYN9w@%%2&s0rS&vZ}M
zeCV2)uKD$66=YKt-)xD?l$Tt$@3XqEIHjn6>X@JeT;%ue%*@4SvlL32Fs9kwru
zqPBCln?X%_j#y2u=-IXq;
zEbYA$)}yEFVB|V3K+Sk4-Z<)oa(AL70K1}A-8chEa
zF=TJ_R?>JcYnJHDH8dm;A(}WzI(`rh(-SJnepU9wx$@?VEE%XfVvP4B|CR#!dGurt
z3Fu$*iv0!%z=;9-zn52!%}=mhPJc=#zzPshJdAJQYd9&bNE&IHThX&wR5#(ldqp+CAAR%J8@`6wVd;0P$F
zxD1an%M;W=EOwrzc0Oq|CrGNS1Rk@$0mC5z#Zd0q;^^iMn?;FemPLF#+*42V*&&$r
zODQ`Fr@0{-Lx;to$&QEg?pc(1|DLyGb4H*pd$%`TFxqh6eX$gl{b5qe5z;WxT
zKpfH>br?n&RRgjn8=+)913#g*MEfE?|}8*`z{lI-6_*8=Xg(HxN%%Hf5K>kA@WYbDCXB>O?x2G+9e4o
z;lyR8M<*(^ddsS6BL;j00((<-vWo1ot?2RIW3BO&d^oB>X2an=MFt@vzyV9V^+3cZ
zX~|(!>99oJ@W*qs
z=RD`&SJ1Qdf2;+!mfL<^TtsNAz^aMmY{@gf0XYJTCHp+dGrkWVQF9&c_Qkx8k_P#I
z+{B%IS@=YJ;}qsb+%9T&H^1$dBqmn2hgQO6L2Tz@^#8b_6g-s!_%tpE)e!e|2YhnI
zikY@ZSTna%$2N6>tSot?E!hHd1&us}NvOLV1fdmzn`^MW+4j~$e?Y9^6y@SP{;
z)LdEiO5NcbhZWP59{kh?8`0f4H+T<_HcPL0wCbJYLc1iMR%iY6sw~oR@3x
zibUK)?CexC8b}+vlI8Fc2sjZ&O>?st!z>lJD|;*9itH3|_GSxMw1eq1W~F@e!;0+W
zAO%-Bf(@py4EoJbTt)P>N!wZbD9#4&UWciM?H<(hC;YeSFDfq#j#02f;t*-d_i?dE
zaYS)(W$;zNQ9=@}bPn5qLsuVIWY@{jlN_a+xo7iRwsvEvkms6^8+yQ3JU5<5B6B1uadk4$j
zZF&aj{;j&W6JIM=QTuVN!7A6XK96ICLW`?^hrhqTYKL&Q_Iqnlu|0yZg-nB-l9^FO
zoj_Ytmu+)sOHx)EjHN^0b&WiAE!|Kb#3?gCQ>phgG4jXj6XMB@j^?+HZ~9RjV{*=v
zj<2xiwv0(T-Szc*BzjV14{Y+cl)K7yroP3av4n_eQ);o;nSyyTezuw`IW#?iX(eLt
z`)az)rJk0KxVAM0drZfPht|oncuIh5Y<+>kU;{>F$qfR!ta;1|sq9o+zi#(1no%t?+pRk-$U1r1)7v2rZMPCtHZ2kez*hvu`z=)I%4O
zmIl_YJ60oK3Y*3fKPTs^Q5sPgq=;QgZ@iE0rdc^(JkoRSAENU^Je)~>T{p&dbuY3#;$(B?KXDyZ6Tv0tH;HPoY)q*kXvk+}GM_b9A
z8Q`LVp>>-x`dpEmmCB7yjF*+!Y-u#REV@(rZs42@&$`uEZ0z}r+4KUMa7t=O+#LzIXEC
zl2A9-Azfix3~{u#??9Zh5NF{wz6;XnqcH_)(!HZ<{N-;9Df39fj*qO^Hpex6mM{@zGKN1nJ?sr&yt4oC
zK*3UP_}7L=4C<-I)*hEQCmM~l+>t~zRMVkohW^8U*?t^-$NxI;#Ji1vTBbY~OS8mJ
zO{W>-+Qw)Q(;%UD-O)*3LR0@$Q=`?>NHU)pqbl^Y0xEZl)D!yi9VC9fymyMr~mnfyo9=rr`lM25_!Qm
z933~hyc@<{HqCEnON`K?5Pn0hC1ZDd9LERAncQjL+Z
zLCeEJ^FpPwqx`wvx`6iWruEF`gRDlle&e#MoZ_m#9Y1DAaNS!IFJYSgiAY!rngFGr
zr@JD~`805v6BYh&8UyA0B_68r=%g#SWs0dIZ)7nSG22_|1~Ua5Uh?m`~ry?w+kB3~@)Hd0Sa;}Zf&K8|ZZj{DXx{Jz$2
za690p{c0BgFL%WVqYrG*HvAmtgvMas?lXvA{vO1dqmPIM?Rt=U&fh&
z71LPXg&N{jJ3{wTPuqkyQ-_rL4p2TeMtcq4I-`M3-1_q$IjsDk2gh{`$5-;#*L2z@ruc-R&5XRsc*ScX);NnbIemTy6a&?`$$CVNF0vI=mX@Xrng^6P1%=X3bujmbr(m`rOW4H1=R
zDKCo{F0B(kz`nm8V;EmDF*K?dBs_AiUIXqYFC6$^wUd8V?Xjx*HuwA%rcRs!U6y?n
z-)@v^hb?y7hFy7p>PH<_W^))-rW0BzXM9Af3)T_(rObv?mS%pf5-QA$5jAxGpwq>^
z&?j{oV>qyo!m%tO-ke+Vm2|)PoqaQ48oh<|koXDSGntY<>_!@X8>=zCsY*5;P-7>i
zo`~qKjKk;?v-oq4OMQmBg6NjF(Kc)pk({TNC>pD)Nugnyxq=eg@z!f3)Bhxzw1fMB
z92CKZZUh4cwvPw~#`!Og#7U|iAp=sh^xZKJ@q{1zA$uB
zN%d)vxi`4(hJIa~3-9)6uBz4gCa)|jzgq0*C$|c6o6kC+n9s1HK}VYoHE8DSb}H-5
zPisF`wDwbd3FvG{KyAaEeiG~nKMdSD^w$R7FQrK#`9jwOa{Jp+gd*X&6#zTFpOkU*
z$j0*tgZ$0DVl|~}*M*tJYP7fuSDxGk8UxhGtgQoTaC=BT=N}+aqe<}=J6rPW%-;c%I(bnKgGXO@TZlz3H
zTCWGDLg|gR)cSXCD`(ZtDa0Aw2l{_1?;1yH)3W_0dl*I>q~lWM8UQWARs(yj7pTmp
ze#qci53KNW%anDOrYhK$BxYZyii_7d)PLH`7xuXnrp@;5wvil
z*K)TIf)rwDUh?j#r*dUD0w!3u45|&-#F&j3VNny6vlADe-=*^npQB?OjPyjDUnLIc
zWrTb7Q@iD`0IC+p6=11d)0?Y|dzPis2vbaCRi(w*QWmFD-g2dt{Ul*@benQvNtZ$w
z*})B@kx^%D$XNcmKuVf1QKNX5Fs)JfTNXswD9rd>mcw=3i)bjV#RHVGYEX}3WwCi}
zcjK^;8a&`}FMNk?Bq2%5DkvwfA3U;{V9JGQw86@K+WtCw1`w)^!GK%_egzS+u|+^a
zNZH%K^n^Jh3iZ5%5!b9K>At}9e7_`kA~7OY24&B61#AE5L+f$=3)^PhsOJ%SPl!yPsA4ERGkKYV4-|0}P6Ete=fFWV4lsajj
zO3b!7^LiuCUc_q;V+2QIZTFNC+f>$7j3WPDN@0`(&xsSZo-|b*G(sS$q>F9jBx}8B4`j&hZ!u$P`kmkF#nM1844`%$
z$(USru$qUF7bipIJu`QU~giuC*U0nQSr`?lP6d`
zqd!WHc#d!dd9h4tA%FrruW7^hrp2>7X6lS=O8fL)iOGb9G28@^f|+v}T%NJbDda5J
znz}}Qpenj${km0{fN0>Vgt8{nAlS{U0OvPC)q!4ZS^4xwV`(&7MBQY59_eY>6TOo-
zf~^G^+b=^T0YA;@za84m$WyMH$mHYKWZv`1w(5CT!GdUzYoZbkCL`fC{-PQpSd~Nk
z5AN&VQO<^F&Iaa%*eSUKNQ@(bAVZxb+SKCA00RKsxDab&a7-p25p9}@LZrJlc#tA-
z-aFrQW%G;ljoDM2r@6ksYPA3mer5GjtX&$7V
ztZwiL@*K&`kfdZ@^8(e82GW(gtazH%fvfthu0H7cWw8FQ7k{q}C{k?xMjCLWIFMl@
z03uKR48FK)YVasrvtG9>ExoUQF)hvN{@kOyXs{40uXIMAzB8H-wS{j$@Cy$?a&pON
z6evOj9FdiL(l535C>*|L&r;Z|TYr2;DogUZFEjI0;aj3(X1MekPfj#dF4xq~
z9A~}PhvD2tP%4dH+Y>^$H%YP@kZSEH6HP*aZ+eFFAtod}TFCJM63RWCo;BA1cvHDY
z%6+?JFQ>Pw$N|K+L+LWv`?3MS{CAd@Rukp@_;Dg-(BYx@gcQ~MU|E&2iuxzhq0W=W
zM@((7|1#|r%g)fFFHxBZ?uOLC%MUnXEh5(IHwtH#tCg;GvmU}S7d^>o33@c^PgoUs
zNGL2YFgQ3cFfcJNs0zOg0|>Cj3etQCCz~Q?C8wHXW
z7^_2ALg;enLOn8`1)+nvnao#|=Q>HMoCKv9D^Dt7*{l{1nhj1_T3P*-*rXp*O~*=)
zn~WR&@t>W7;|(6)fOlxojn~cArB|cex9Nd+@MSWdie*rzro*88-i=y&mqMQ{HZ_Jp
zNE)M080pWuT}`(~JM5o*fN;?I>4-gcmy%@4U}gX5{x+dT7rL|Bl@&HO$`dm~%Wh5L
z;o9DfS9_QEP+*&JA0)!&@kl4t(FQ~6t}w!8l^OPCnHl~RosY^8!RWq-Q1RdQpnxxu
z#3JPs(eTgQ^l7N0`=+9$Z$1)Xr14|#<_J@jK18Hi*XRKnLnBwN0CwIlOazgNzv>Jv
ze`T@rhOOV8s*9hqN%OR?pr4;5eV>wr-V!bDl1YtIV|sQ*w6OyeSKV3-dM-wYZypJQ
z0@R0EuN4
zxQhG&WR!>YHlbgt>vi-@3{7sT*LZKQWs`V#WR6DBZO=#u!(+w@t#0`6Wkci<;bD|Ym?`UNX&K_hM!4{O=?>bk
z&EJG1R^6m-fTM^WGQ(e~EN*AxI`uw2Ff8XDKIcUZf|F3^OtD!-MQSV2U4>}%sIuDo
zUptftW@9WdfLPp*j}@J{b3@vq;U@k+FTZ69P@%LIr{jjta*(P|B3!wQRKIPi;EjjX
z@;r0kEZ`PA?<&Yv=&%qcg0MH2wkA2n_{XimVU?(vfp?23ykj}-i604aYX)|umRQL<
zuCRR-u$74%b5=Ci+^h40?02ER3bUOyMz3S`zsXh-PYv!+zDq4c#Om{NWd-)o5T3GT
z(?}i|bJG^y?uQDPWNj~u?-YhOCO~i-M4xiS>}Vmk{
z0s)9bz}x{6d)7!g7C}&?G5u3BXzOq5AU}#euDBRAk%rifydiioSkBgpZ1CbHAC4Av0Dl2Ts%2GmjLTkwBZviGGULhI)Rwk{
zYy3KoH68Lnp;EmdzZrw@2Fp@rxt3e(?Rkkvx<8@)5eTB1n3iohj(d6_!Dgv^p*)H5
zLW4P52VD2#>!|V>iIQ@)K?p@AEO1E;`$8!uwLvq*g(zx|RbRWrkKEC;4~4lX>X`Kc
zK!dV8q9n6T=HyiHD<^lo0u%1?VgYSZDK-K7Q5+N|IaciH@g1Kqp@3QG*V+#9O`-xy
zdQSAHn?%9tvRIXkRS(1QYOZs^fQ1OXOdWD9xkz%w>cpK8)6T>!btU@gn^`+ocJVcP
zkVEp1XpGDY3=Q%mqJm_CpQ*Kx+-5a2z+$98@c8e6bMAuT4TrDCu9O6V7@SgH>-YGd
zTImXnFp=Bh&R2@M+1!Y&Kpu_4T<4hDM$ud?s_2TDI;@$xaUl)^nQc(5jAq}fqZW*#XcU+RYic5Qp1kq{w{&f{wa0`BTJeb>>u
zSWO{1x(gyomwO_XC?S@!7)NAie|`zeYOj@hi2JZMF0cDAX?TWly>(-sM!^j`_gNXX
zvK5OKOXx@a`lashsj#pH_|A_mAf}ro?0ERnSDZRm_viK#AF6g$p^#
zP%WP*XczW*%nigUG85G{m7Tbi=$=!bmY`0>3wcOmap)G?q&N+Ak;^>F;d_i@I_B#u
z+tUa$qILlP+~3%pG+n#~zxKsrqHc?1xITuYtQTiLNl@EHe*m1KZVyvn*C$Mx4aIN9
z2ih$Gw#HE03qsdisqG~i0Nuc_cDN&*h*;a`t6loCl%z~IZK?;{{kgtv181(
z(X)mR-yR;FeZksvk~GZj+dk(BocHaBWnVd|cnR}4yy*9IAK1H%L2>^)3|pe>G+z|t
zG0Wt&8paQQfkYP935#8Z`4vtEs>>U=;$EK=$A28~^-p<)^nFDDE>I878D+`$AVH`1
zp`qlk_W8xG5IbUo`)EsQgl!PKE`Nxij`$|FI3(D=GGRo|V`(cb?dh_-b>EWf##v7d@oA0OQ+G{D<_8SDAv2S6+X7QF*6Oi@Tgh$#lh~
z&H-V(9Y#yRY#-{Yx(+wdowCV26iaSDfty~|PzQdeH~BA5=8dkn)WumtUr3j`941)aR94$#ELfOGnQ=zNM892myOK!=CZ$xU(
zDK#2#4Y}sC9Y;Uf<~4S9N-x^uj_>un+rv5m5klT$+N48m=%0pU-hAOisVFx4LR&d2
zndZC@cO
zuUma+?4D1JHno!o??7jf|BWJ;1o14d~o<4d~*C2Dqvhz}0(gHyr
zlttq*q&sGV++sLyTbiJcb(C#m)+RR^^z*NUrlk(pa&X#KEyUG1p8b&%u
zJTixf8Ld`X4r*>0$TZw)~lf>Bc?rhBbLsZ1PWn
z5;c&-x*iu!0T9?NR)%o`YZ-%r6p9PB*6R6!(T2jOW$a$^G56gFK%n(#Aoc2z-SGv=D{fnq<;=jP;loe!6)8WQH-;i)YzOFOg^Cn~F>Zx38b5vo
zz_{Ftt?JdZEG}Bx2cB02{&YU}|LIJB30OHfIVFny+Dmh`>e;XUbnm$h
z^u2dh7M7Pld@e$}cl|;jnVHY!3(SSy%M+RlYy=aLlC3yB|0YxrpWunzY@u}8?mC_EP89W1*MI;KxO;0iko
zw?TpbblHS^+;j8gSH1`U%Wp+7
zBSzv$a)IzA*+!AU!&{gz{jW`+5m?_MAzoQ^{GG`z$!$a|rxaOz@us-eu<{I64Ilpq
zy-;UPW2`;m8`pZS6ogZ9z;2%>Y)!l=b}fVmz5#s?C83owzH=F?h6Vmc$yZ7;J-z=L
z>2H4aW2OWSX2geKyjplGBgj)JXa;Z~Ar2LZ^(Zzec(l+u7%=4^7EK$Re~Fa~#}KT~(-
zT3uyUwpjH)s6+VL=4(9L(rFZQ!hjUjXvdB0kWc
zbb1Cc2V)lLr&42oK&*%KJ!oJZOk(&rjmj{T7|rgUa`T7(eo=hWarPpR)z|Whm`CIkBG}!aC-guT2C0JZAsj2>P=fuYS~MeCT10$|
zTRlsK*gUCCGI^vbHK!P6(pPMko3DRc4R8p)G&nYsCxC%)V&#lh4=uC|8^^
zHfqP;(C0%0FgLLA;rNtHm`D@7_>4Rvnr59hDff<0TS0B<$Xh>U5OIPB2*r!_t`V+Wt{fY1~@v
z3zz6vbbb&|;EJ@DZgZnIG1cZ&#$;ZNC^dh_HriBd%6kW)(W2wpZ8|hzq4q6Cg+S4<
zu};AXU<(PtC6u!GULd`mo!&$lnVbV&IN;~mA9dVvu5cxtFz}_U6Zx5My!^B#)M81K
zp7TVlKUXSgMCk{*In>mWWr(vR4AfNAFrSWIg%h5rzua#{D0>X{8T`Yb1!^;P;W#?=
zdJ5B;NuNIoPVBqOY5C1Ri#-^z6sWR1bu)0;fkM!p^7w#SdO4N)R1moDHMYZLh*>JQ
zjq>IJ{u4TRa#obF)z@qO-1_A62m?wd{Xl$UKCj_9yS-u<`~)g!X$!Y+P*Z3Lp1SXe
zhoQ;zS%IoJ^&lSxD+`R~Y4mYYc8gXO7DBlIN%QtHa)AQj*^RW6@9C9tLhebe8Z3(@
z0CV1V8er(z8p<0D6B>d(bJ;AL$q1i?tZwa?z%JvTG>sV
zjYXCb3pYuNS~#tI$Lo?^nYr53^BR7+fRV~EA{C{$$DgYsn6!k`0fV{!Y!F)ZX$N&o1e0*3pqP&7NFmm1o8P$d*HM
zMsDWE
zl*_+@33^@jFkzQ!&slHGDVGP`3|j+_a_)I!kI^uwJmyj~`kaOV
zwM9Qt4e4xdqmyo%W%_08u0v}I-kLl)Ymcu*6dLPRXsEI@ZIO?2v>P$?U>n-3?#zP#oGCJ!D+DI%}qeZBCM2-^-T{fgq{apwF=33lerxdj#VKQ=?7J5MTl
z3&ljq9Nzz&$`1#r$Lh|Xg?DWh6#SLpfuW!|oW+o1JlNRKhYL^Ymy17gQ|_tz5i
zIVqdulenXnWYHrRQ-X+M$*Z*FG33$WgpAR4>WNlNdl0KUf;@rn
zHO?>BlKBfaVCRcZWX`?^e^Ig1+1tds+)%yeeI`j0rnp-SsqUsYJ?9GAX^X1}k_Vsd
z3jbhCL62j#ztBDXU82GTb<%+)TLCHyp?OS9C**L6Oy8)g255OSVO3bRbJln*RY5|%
zh5VY!E$K|$5=``^PSPUpb4fgNyYb8-{$IMG-&5!Y{Is~sj0-o6-yN3?^mSvU{N`0F
zEH`4%wwE>fQV@bnafHryyRf_E9Xup)q)|44>6+}Os$Ee*O~MPN)B|riIc{qO%%OQPpZ8@a^TBe05>fQzRjgGscv4
zR_U(?M%xhuZi|2LA4tjGSRV9CHH^G?RE&A7!e>|vp-ruWB`z@YYGmngd~W|H(AnPm
zRw{vtL5mP;O0wcE9_-z7t`2ZYOBTf!gus;FzL*)yzL*tm*}Mtk;hR5amu4yFh)U7ueQKF5
z%i!&26&zUnLxy)P)nY}@F3UnP4SW@9;qw6lJ~kP3z488F7bi*IeK72Y`Gv#sqghqA
z)UP8PDz)FLQ@FlYl08F6PjR!f#$+=d15zFE82fYhcgP+18ef}2oq_`cb@ZiL=ZI$<
zVD7q-F8Zt6xsrM!TfqVmtKM~dc^Ch7otzBB9wD0Ut%lf!(|39OC~Zweor$s9;`l6%
zyz{7|X`RVyOtaDgxxd;TZ~z6L7ezoV+~EsC!CrNsv}ol~*$VfJo6;$mEq9bNpj*d=
zha%)p%A77CJrLagcZU&mo}SxxyDKR3v^V@_U~CZj-Qy&+j>7P^IZl8+ak%!%PehLp3{H9=mBeN5#q>dxC4yYMU
zu%>kLeiS+%cC9T$%Kcg2rsRJ+d!?t7v!3u{76zjvH>*1l^Mg}U$Tw_?e;BBSPA+yf
z42@_{P38n_jR+1Fx_Fzu)pBmFB(*!IvH;FkPFZA1g36fH$~^um&t4
z5@a`jk!>R+lGi3mKLx)|JBXxcMC=m&t}+h0_z=yn!SkdnxJhaH7es$^!Hr}kV-;Th
z2?0=&AJmuAg!fISt;8f@C&FlR?nHWICv|asEOqhAGHd0+o`{q@l(PJLxb2F|#FS^Y
z8#JDgV}mFYAqWax%pAy$&HpNi~pOty;vdtjawR1Kb%?W*p&8fi?208V{a90)#7!
z{C99#nKBiAVR`AgAOe?~MBF-PtjZ>$&1b&%x%aq&t_s
zetn8%Zz64b1)t_*Cro=$-@j`t=o!_yeVZ~GpZxBjWd9ex((xXm=g0lVN?1`A26LGO;4F~WJmLvLw
zkV?JDq;Hr&a-y}OndXTCJ1iw+l4W6_FEwbj0oqp*qaS+m)uwXXi_z$IV+k<}ymQXs
zU`?7ctuG|{$_=dpm!j~IGXD3i>mE9;1FkCis=vv*ziItzJ#Z}a4Ig#ZOd=5Z;OEzg
zR`Py-v_oymo}bbRmy&}*5p>I_0;mI<@jLIXIxlEu=4dWL5M({5f8SB+Ft)rPvxOsu
zagV8W-t8PjS?3x5wuM{GR(&%=TW9i%H40C&x1oKH*!-48I=oIguRP=Z+t+_S_+bQp
z8c-1R6cWwY><2Cl<3P@HYbZ_bdmN12u1-T~zk>Yw;s!jf&5Iw^_0@gF{6!_*Nb(In
z!MB1HjZr6KPrjolM3pC{?`ErbnxZ3C`9F_7*!bh<-Ke6VT!k?GGo
zsou10(6Mj}Qok9pTXjqKhA!`-lO)ar>bMh3d@@S!?c=*Mml+Elj3rck+|@3@C}BYH
zBCCa;dFQY+bWjOrCFUx28T7LaE8$+u3YN=oRs4v@^EfW!Q+Zh=qXxUaSVT96^f(67
zr332^ItJuGkdPLL>*}vI*@$l%I{XcnN0a>?X&x{+tu9x;#cZ003DftfJlA|dKTu0p
zI#9T0J&Rg?d$YfgD28sD+K8YEXnh%PJQBBeT15S=f7b338$BrU;B(n
z7Z?)oq8um5`-|kZZX-yhH`uY&JnNlV3-(JZtO@Xe*AiC4fzuuNLEas<4wVV7mEq~)LtW(p9t6}CdNQNQ`;UofO3Wj}fPpT9E@@o>o5e+p~UifIAi0{$@
zI+fSSy6uUDU9wddP-$E^pR^vtoqR54(VqyA?a}sg$7Fx8C^oA$Tu?2V5Zd&Dr#@#s
z{bsL8`D6t?r2+XYU*T#>Sz0*GB8~^*JGtV$RMtME5;D5>S3fi_ug7~bV&e9n%1Hf6
zXU9`S&OCV+Q=JX#@YOCTuU#>Hn@#biNMoj|%;E0Gw@NTmV*T!b({4pM#?vQwV~|Wc
zDZ;kzFq;At7}y~~l9-JIpiQWW^`XRJYLf02S&UtP{bM1tco4CqS#~yAjSv}E5!%-N
zj0(Sh#)4)_f$a@`%W&M=cr8uUKwdlVo%jRAGf!3qlV&cNS~uJMAvf^(Vbl6DL-h0G
z92M-chPHNYvz#4KW1ZJVnss^n_IQoCeov?L9xWfIYUa3+flJuE7neP
zR(}1gvVasrojVB$*tNWuII?N9vIkAEp+{PwCW}+6s@}O#SZ8G#(ag%m9~u79YKqb`
zOTW0DV93-P-%**Jd2Djqp!&W2+qRB)rUh%B!axWelKXLj?a|IRkw>p~D$$98(zbqC
z0epZMm0-!DLY#3$L26@>A=0xXH72*P!j1cT)GrQ=;$q}+YtC|P6hg==e8@Pw%UT(g%-qEF9Y~9(|7tS@R
z8d-3XLZv4MO%1inPM(0zdBDRlkDpTA<15Ubof8hqLSasfpDt3#=ShQ6cZvY65%0gN
z)CuR0KfzB`pt@JdXs^YhD^eCzKkxMqjK#-qOh>PAIAaTljvsOAB1RlJr-dWT)W(UfD!gi
zknterEmJb-)?Inx^*Y8e(PfuCBNQuiqeZ(qZNWbPJdOIKJYt_h;c%Y?6tfOc7XRT^
zPowsEpHpWEWM6H22@u~|Kfja1
z&as2=d1{Hpt)9^BJ=%UegnyS>f6vFac$_l6zrZF>Hd8R!qsVqk{4(lp4(%TrOo|L
z-SFE{4zd0TQGpLmSg@!?ZTwa~t0(mjmQ&~JSnvOfsw8bq5HvV?VIl0Lag_)_fPpPS
zC4CE41ys?5y`Ob-aHE?fYbt86nT@^Vjbnn*QU)zB%@l23f`Sn@tyUbp(uR9?fWJd&
znNKDG57PLDEQB%8LxX4+6FI`}R#RtZkEF&u5H^1Yzr`g6#|WtT6vmYaplG&L7gZLO
z7ES$dlGO$|f+X7Arhc6Hi1*darqucf=G=t=!0;=IB!Z#yuN?OadC&OoqgUNXcn<)r>m#~9Iw#CTkwmp|_M;4b4#iey1rS(E{T)fLG
zdv=4!-sT9#XugX?I4umDw)^+}Y}lO#7XEH$H!hWTEguUrT*-0w_A)h=
zv5titewz!V5xi>M{X1&ylqnYdznC$l>lBCv%zkebbJRyN6LXEFeZF!4=6Q%_%X%dd
z+a~W#cs#p*_-@bYPoKa3v0gT+r?I1N01&wbJ_P(SK}w|9mb2EWp``~Byh%p37b_*vEBhAi0{<;$(faQ^G&}?;Deg#>(ieNSG=B}VbVc1^KM(9^t(+vdn%T5$b2aJx)m0Kl^@Pfx_n>=3
ziJS1t_m*FX3RQ$@x~{jRKkj1gPm=__8O|WRb60KSiA68fy!;H+3%0ZeqPp>oqnX{9
zm)29YdAD@Co_7tmaog;n@sa+KUCi)5@fRyw@McsPR--$$V7CJ!q
zp<*_~;fQu4cncF>dhb~Za6$D`Pc4f4VHX%&9~uu|o6~w8e;wOBT5Rg%+jMqN+d)U)
z;Sd%h6BK&q!v)6^x$5q43D~y($OeSNuVUfADeum9zj}qK7N($mV_%0g7PnKicuUEq
zJ~qe^6yJf^q4kQ}^S(3ux|(x|;TVA{M@QxL6)PHp8y*Ga4DWnWC*D>FT1Xl9c8VXdO%+%94!fRwu3
z8?a}QqeNbbdTN+=jpmYLi!ptbeJ8{k$NI%Fudz+~kn0aqEQRiV1dCcVmTc5Q1W}+{
zluz)wbE9&~1r!3_yW<}4FS0eElUPD&|AjS7l1C;vaIBYqQ)JQ{p;J)uTkQ)RJ$WJx
zjUoYi-;X0lx+D&ZPI<4^K>%85Q)oFFSB8r{_q`~_xdl6=kekG7GyZwgf2+&4(YQD8
z4Jkj=6A^wwzwXYA!r81ZBv`Urh^M}|Rd$soRWTV4rX{(m-w~wKQ#Wm*(Ir00rlwvS
zuxUpE$b6RYx%}qTyFKY&!R?vzTCxq5W-yVt%ncoLfBb6P9vihJgx&)uoJX$fX?Uzn
zgbtPHlEppkyo_v2BrsPiI7Z{3-WJ}mjJORuH@%TvF=4Vu867osXg{)}u^i_@=mo2L
zagSArwJW(Iml=rq*UmpI6OG6>NQUp>CA{7ZsF}IwtFI2X5SOhs
zn;gUhl?C!^Nsv`Jk!^elJJEPR>3bJ}GjyeigfGTdTk_>}csrbUORb(~0$pcy46&^N
zpENcsM8`b+MMotwE|YiBeH_Bv(FZ#?!U~WEys6cu4GxK191*MrAplvJdFUee`5t|-
zjItM4TdeERcHx7^_bp1PAd@nm;Ph*p(rXkc2mfnr`;FhZeVJ@i=!fTL?OM}lysLR8
zU1n(!xbVU=Jq$vM0K%0j(k;8iGv5L51a%XDK`j!dnHvdL;~C`zZ|jJmjy_7fP9BTd
zu4&{I3GP@z3BBIQ;)tbDy+0YyxqN
znC4G(yf#)O%P5eJ|kwNs(7#OS(b^uu#
z8Ix_#NKTH+0M>j!H@rL}#dJ1j^2W6{dkbQM(CmWjU>%uK44
z0m;fBnsW>cZYVAbZUigbeLjHcV>6hkf5DwexChLfb-|5Mbn?Rs0Wv5xIEn!rlfX*z
zFFG=cP2PAhl}UIyRDgZ5<0TmxLn8|yRm~6CmKORs@CYnq!%h{c-RQ`Pkaza09g=5I33mo
z>6Mg@*3@W2ga77o^ua8TJnD%+=AuGo<6r&D5OipT%VK)#h2^e{X6e{Q`HW_V^z5^E
ziqGv0(TXJrH<;5Y-mi<#@8|Dp{@Yha-*Z+FcR+X~BucBX5fA
z27qWWC%!bM2IochL!bVOpa%Qpr;q$dROYKB_1T5X@deJ8G|v4@{VV3$M}^&&JjXXf
zhU4h5V4?7L*+CCn<{;qWr8wq$?uH)h{Wc-S@t(GID+sAxQ=Bl|224_12vb-zD5w}#
zAC=u@d{(@QK@Q$=K`e{-T>K9M4z+sa96(G9RlZ%TBGN*>@zf+Mz`4!<6ItIb&V_Rx
z#W|9JoHpkK!P(&kJuV7>cx01*0f
zm>>c^8+&&#Mj}9IJS(k^+Wa8yV#;;KLxaTpx8YsotLMfAflA~%I9rTlC3gk|TviK%
z&KC7V+F8u2!F<|YecuuEhAgGNWUlk}FH1h&ECQKg-;MZwURr3uLjxC;>Le_B(vr?u
zUunuSLZnk${h`XFGMSoq^2
zimkr`V~JH{HX0PPmueBWiC0hJ?1d++VY|_JEl>6*h{4
zqVYn6_$mQlbJsRDFQP1@7Yi#tJ+g{hbLQIZPX`Ga
z;?danhzE$4^}}*^)@~u$xm8qbfWW~*a5KB))eo2>i)F@!2+#dhRTs*_QW=-7wo$>;
z#Clt6+jWBH1kj8LzK(`XZ@{LX3TjnjFu~u3`LVJ)tE)<^%p{I6CT!IqW-vN^u2Z=V
zIs~A``gVhNZ<@+WcUKdr1C6=uLDbbrYNzo=j@wxqJh^K&e^!3;-%Xq|d0k{otESCa
zT?1GJxok>cR9$Ardl09QXIWvP4MQGHf3Z+&OAu4x{^)c&3WEB32w)M^bD7DogoS#d
zWhBy}waSGp`C7YeN*P@)r({sJdZa2SC0w-^O>09(Igt2-!j6*I*1_H(uExVQIXU7N
zF=rNSD!W6wYyCP{@@
z{O^Wt!5wzS3;#wvQ~L#&!+P;EwN;u^$
z1uEg-Lh&m7UyE02Gw~c5ZC>Sc{mJrq80B^{^~?Av1NlwU2ygM&aKb)T3WMHA*C4hY
zRZj!dcRA;rRmTXheLqz^w6641R`DLL!tfueco>p$&T2SM4gk;IB?@!=80d+#ACC0*a0}|5G3uLB4S&S&6rJiIvLF+TK5IA;zZ{ib0TBW3DjO=mJn8k?1Z28AV~61u5iq9ugPCe4USpE?k5>BJ`|m7ci88}V(|l+6Ciev%(`Z1K
z6jc&q7+txBY^iOAa(TDMvWUWPY}H8^4H?t2#oDkGb3haIbR?|jVPd@_t;^zx%oH`J
zrK*9tb>E!WINGm3UZ|KHo=s{0HWNQ={eY%25p);SKSSLyN%LodQU$K?5w?qluc%i#4;^+Nt?2N&v+i&^obPP059dEqoBX2@b0lW
z8$dwa=2SR?s(pXGlY}TgGc%`)HR#M$>=Sx=1+dD;g+xDCVM(6`SiWhFMswIfrVa*;+6g>^|@g)QS@grMadu|C`g)nflPm
zYR=g4^gv_tnw`gfwKHTIw2ZEhj0I~<0gy+Zj@ql2pNn)*#~QljFZvmK7rySi9Oe-SbKrObJC;#ENB@tRG3P~PyH#dIQ_7WKK-a%!?n
zcm3l6ba_0j@0&1VQ#7o)S>>!A>X@;cMV!}g1)dL3KQ3El$hUeZu#`u|-_`K(1?XT*
zAt3*4oVH6l8nJSC90B{#0cU40_4GZk8fb!D(ZBR<1d-i9>mgHA=^*2}%+sRlzde7vYd%4+@b
zn6AeT7M)Ms7}k@+Z?e{g&0@?#<
z&A7+10TDBdEatzH-t0a%*9LVPh;m;_QfY!4tnj!qV4xd=v=e!)+7nWt91(Obt*%~9
znWRpO9($zDH-`S2(J^HoS!!e36LQ`>Z41>X4!VM5chMSxgxN^mm62Ds1(31dpmO9r
zcjjF-`AK3mL~*Isd^s&mw_e|7HDS>-MYD9DNMBD`1>=;=Sk$Q=nv~SLUh9X?&Z!Il
zlhC8ff8v=W*F8rE{Pdj-r*Ng+vWQ`<(cN|K3}587)S!I}iJ?2MFBRNc-mVJ^adG~w
zvZ;T0t+AiaoxHDsa|6Cz2~cC(h4J&x6dd*QXEnVP)K=1P%ha+e{NmX@^a{LMnM)l?
z&z^-3I^di|E7@;~v?2%(s<%3WZfueSjntA8#k3XLNvvhi!cK%FF1!`Cv30kLU>7
z+xCf{RAs|x6`j97wefKNLLr(tbNLj$EpVj|?Mg|$08;S?5uNaA9Qu`_8u$Yggah~J
zB)kISg~&#i_B`CYU&Js9$gn&iC~FefDYO-n>Lft(Q9KoQA%_q4pwqTW42$D=Zn=hg
zm^Io*)W{r4gAqB^HOm8t%o`55(3`2UA?zMMb6y_t`f6zeHr*Hy)z1&Z{>OjEr85p5!L<2!+e
z7dN_G345#Peqa|jPF&OOg$EQ4<{FNZl9^|IwRMz{tXXYghU5a%493;s_>kA;V%w6c
z+cWbYKZ+j~$m_P%*(_2-NEOW+4^9^I%vp%50p(hnV>^OJQ*R{ZGGtk>f}Cc>O4$K|3oii
zcf^!cnAiG-Sn$jK5$GRyE8x4%CbbhB23{Z7&`SP-JZeGdw_e6O(*z=a2d&GbVtdQm
z(~t*>j+V1B%B(WhV-{xew@w>`CFZE53kGYuF_-qU+2Y~2y#n;5`dI&U(C3~%e*
zhQ@i4c~bycFg_buxX7pD-VXe=n?~y{I4ZfMwH5QW;HYjtl#x!WJ!4dzz|IWo@Z3`I
zn)G?O7~-TbVbZJvJhm&8>PoCS?(7P9Xg?F!FQkrUXmzI>k2Y*}c%pvZb5xl{W(%fd
z-B(4BHfZ`*XaN154x8#1N;fnQTdzjzT(p-jP?G_+Q_?v?Kn)KfrsSCY+OEn3+;I2k
zUiW(Q8G241!**c~NA8$At^{rL$F{YJdORxAJDsT~OzeGY5tq)pz}~-7vqf8Xok#8S
zWuXXILxD1hdeJfA+L0pS7E9I^JCeJ|R+?WVq#Ma8Gphpeo-iskj&0~QI75WtS9-T@
zep>+iDQo&h+G-E+1czv^Ml~F61>Y?4gr;5!OQ|Gk@9Cz%5XH~%
zITZ)f{8&Jxinsy`UfLBmcH-zD^oBE<;rcwbhK=r$PG*wlNX3LDZZjNSam9B)>-`gn_;{+^UeK>Lm?-79g!=+;Poj_s6xX8{3YXu&!-xy4ki_WMr
zW~+DQHlnxSB4;-|sccRg^U`gHgpH!4JREJ$Nv|{#bZ4=Da;hNWX|2XxHTYZDX$#IrNDPpA!n<#1r$OPMftK
z2k>1reB`WNix#`VMlQu6EpQ)rj@{)rYaI6@?n;FxUfKwdtX&r#NfrFwY$3>a+(M}bTy4S&@L^KA3nk7`{snl+7
zv7%)(@zCu#=mkz!?8!j0#R`}M@6}s*%)Dr?X@lF0HMq+>%<(H32bhF65BdGXm6;KD
zJ3Y>LHa$dYMXw=U$d%i*XjK!=WqCs4@fglAqw?><%3JY5MOxR6h2X0C7E8dyUlCe6
z-|4sfDjyVg0X)2cn)g`4fkWG1!OkuTCM*fEPBd=8sqA=EG
zL`w(>r#=OVYg^OpKtrSn0RA?bY&Kck2I=K8rCB
zyGV!B``#diZPmtDldQy_zt9dZYWF-H)8py($E4+Te{FcMrAU#dd(Z(=eFd$|6*Q$=7+L&F2boMj#gh}`!YS0|4B{C{I_g4vdlYV9*
z?#X=;D#?O@qk%NU5Y~Xgw}cyAO=KHg7{6!0$HA(&3dK5zBP;deE<5p4*>axL1n&Dc=1&Yf7q($b-y>Sj#D
zJrqZG#BF?+BZ-8ur8fu4Z2-AGZuHoL0UCk8*zy-FfI#&=s)tA)CAyEy5C>n`{=!QX
zVNMcT&i%nXOgE8W4;5lgLa1RvRQBD$M8cfr$4hx&jkNCHLoS5a&_+RcX_KgtvHoYAf6WwLUSmYHRsX50evU%Q
z0+(x0-(@W}^4jhRxtI`1;G5JWRF2&F!GQz|pmSoIz{!2fi%qz|$N39}u8ddBs-9k7
zF($z#BhQwkY99ue2_8$68v4oDan~|0#RCt38+mq_J72W)UPi)-<
zAmMTvW{@^+DvQbeR8(a}#|M_;&eI85cIm*Xz#D0Ld|ImZL{dZLx)xek;Y5sI5+$|H
z65g{6``v3tclH#cifp=`N#koqw(}FT;@m>g^hp6y!)=fYr)E;LW+wKCL^DGVJIM{g
zDzH<7t2f9v_~$#%yCNmsr<#WG{)?#`K=(CE9vf4=eIA|Fj7*h7;;}KRv%;fa*y%5r
zx-Rvf^u8t?
zb54&t?`gw=>)BkoFW5c-lwB|Kf-P!a1HNU~0)E7awPG%qAtA(Y*Sei%Co2jE@U6~M
zh`TVj$u(R!dn5ZH(^td(0og-7?HrJQa8@duWB)2xog1>V_l)!9FG%wO3=uqFlq*kz
zZPHbYcDp#h`HJYVe&PMfBM#teYd}qXL2WcW0)ods(qH{3wd86Or
z(15D*FH6oKU|Y$vOQ()ZxDpY6VN)iHV9#U9?^j$BgLbgQZcE-%s79xnB1bN#N^-@S
z81k{{BpGK@W}rb+E(;eY$ql2CSBo5q(8>?TsY_JIIOv|=VSSxXQP7$Lw8rx%+JK|i
z$)RFO`}#^+7gHohqJ#3TKZx}=ijYEXAzP9JkudWo;gnd4N=})Ls&fubBlD!Q9pph}
zY$EfU2idF?*<@H0SzpJSN({~q%GAloz*(C#p<%*paC(}~w~aUWG)m}A&OW6Q(PHa5s3U?+|v{W6z!=uTXW{UXtO~b$FMu{6D33TU5jgQ=iiWcxIoRpL{
zxg~bbJ=r4g7%HRaA3eRf#fqvlv>!p;UR&ARr-CJ+X(l>6hQ1bZ
zuLB_wlV5$^IaSFiMsjG5cY`f^`{C+;X)4}rb2BrJ{WWi3t0d4z2-TU#iam3ok}meB
zt#U9he!$tVIgymPr?tqnuXpC+nCFSC@<-3za^0r4Eq2reMAXSjR4wfJ;6>wZQVHB8!X`RkY*qOi2s!NVn)pYNkQW#bEb!C&54ZX=Pn_ymlhG8_W?O|Qr06Cm+
zEdAQw&;^_*T!7XEm`&F15Deco5)|}coFwqlHx_*nA$uGh@EmvqOEMvN&U!$&)cvgT
z4ZQ@fAI@|FJfg!LJsP;4U@!Vk&o(zs-|NZHW>I&QydLrkUayU7dfu3lkua-rgod>J
za&;%V>(CWOw{%+FD(1~9*}wu`WtSU<{cgsr&LO18F>pNm4W1rJ
zsk~GP07n;zEK!nA9Kxm>%cU3v7EOKS0uIOmT^cIe%m&C9Y&8DZ44@?={**1Q4YV~n
zBGZs3Vhdgx5dH>VmfPK@t|TvU!>-J!^F{>KTbY}>u(qb(Qjf)9GlIaWL&vSfm0wjX
z(f=d=Fol5?l-+OM3cum>GyRstZUkE|{FhY|z?bvyOQ?tB)h>U(>L5
z5@DRFmJ~PQuEbzi6CuUE)a(i8O{_G-(4iC5NPLipb`_q$$Zknvl}~2T*cIF(PXI7#
zgjzR7h_mO_>(>AtaSg;rRS{*067MEbX@<}z+37Y8=3**Xy0a8FI@r2$FhaXoiTxr9
zAQaspAEmY@W=*2(j{Fj)b8)IdrQX){D|>39dOGuxrs)&3zLk6;e`%XO;oZPMTRj@W9T*L7L{FgagPpXUb{^q@5Xg%i`
zmUJ+BY`Y#zS#l<)qHE<#EQ8fq!SD(Sz)jSzYL+V0*0)OfiiGaA%_TE(bCAy1g7xny(|9IM+0_Icie38HyMhlhFiS;&pY+Y^EWn8t-&4Qa~!LQTwlilRH}x@
zGpEGWvBj;WB`JA#PDmcs)rBr&9MhV?)!Bi9>S)W1BlvTex9O{<88441@0s{_08Qg;
z-~5>U6QdJ4uqR{3`_gbEd+LGmMO|cSLJPzuRjML2!ZvL>`oTWcpgU06(Rlr@asOZQ
zdKp|?s=WB*ZQ%{`Xq%z?8=rnncq3F^(W^{yEO{kJ+|rG&n2Z>1|T
zTF!UYQ=mYO*tgta)@36VakRM{&377{mbD@7*jB$j!?QiZ^Bpx@oHaJNoDq{?ti$-8
z%3l}cP8*bqp>MDTe0n%2cu0G~bbM40kV|S15TgI8SUGlNfZ9=$ASG)WZ<-zvxCquV
za<~~HFAfsCblfUZTrC#TEyXXOUK;s!n<&bK2BC(|mBg0)5WbBydJ=eag#A`ivqnW#
zQ&aQuQ`1vdchU7f>-=u}ZNVbV07}v0^StfIees=n?PF;0`VO2yngjh}68gLZO2iK{
z%4f3$@Ua12=S%On8HCLF4ECT2_FJ;ymHc7N?DE-ye`F5=i>P64&=?SAtN7H@bNd%S
z+0sfTKCoeC9_}kaW{%_fOCYRpW}t7*_%yv_3b{r1Jb$E59f0YLoBt?3aCxnaD^^J_
z_o+U{2|juE4Cse7AI=AvIvASo8Jv+lYLgq}R~-gS!zeMYym_|qX&&CLb+YuH+_e4Z
zT%zuz5t`_QHGQ0ClUkrc(|`@W|LBzPN=Fkg+?#o6Tbr>hDj*j+x#O{3{`msBo$F-l
zzkK5%=$LpyVOFo!e6fr%y^#H8
zwG%+nzuCDXXTWdybCdec{m7XUH-D+fZ<~513<_193nN!S3q_tU$NPoFS}qv%$NE_G
zycP>{*$qRmH`pTSuY*lW*f;UH39t|CZV`c+xE9(WN=+&sbX4Xt`MOkxWywY{5k)2l
zTpZ06dzZkqw0*+4fRVhTKBNfrX}gN?9wCq}rfaj~`o
zY6gzVL4T2CL;D4hNh3c=;z=vLRu2Eq$l^0!{CKne)uI{>5-)wp&}^y7C6jb|-Dm)A
z@bCcx2=@*Qsf8A88Uw>8c$a`S5H(W2Ic-dr5b;6|H9b0YJ0Yt1;#dOy&whYMD8^c4
zT8K{xPnSl^kC`c_Bf9QzF(TjGfkk#%MnI^1S_F%McbqQ0cj-4-^5&R$+3jY2}~
zF{DyBxJLN*{i#Fo5M|@tCUZrvHu5xlNE)(@mgSN~_tI&+<0=Z*oVS^_$lgpJJ=Ud^
zgwZx#z)50Ep~^7^f3`zcY&iS}dmk{1<292?#H5&&Kkql0{J?pQHY^=z7(75CCA-$l
zyo9TZYM_968Q3VKxss1%p4zvPeR+vidSa&=Y-M{L`phnCIu52P30hocTBu`Q0~(F3
z->!}c>$ogm&>*;>fxGW)Wc5Y$Ib8nl(>1i#Z9j{OUp#lvI*H36SE{;O+3QhQyjJ^|
z$%1nC$#1
zJR#qgZV+v`&8&HTJjG8YXqNmy{;8rwe~1ujr{0xrJuMz*hilL2C>Co>G$I6Cv2h(@
zqqROa)n`gCj+8E@aIL6A%&i}cxNu{&AwvN*3!1mG$p|(l`Gfqxy9$^cz;&g+JRA%{
zkD;uBL$Y74IIvJ_pz{vo48()v;lCOk(U#D6947Cmmcc;|MR{UNXLApXIwf`LWmWDG(rtwqX`ai$Hh&Wj|aYVG@Zv&<$S2Uu3SE
zXT&#`KQ`W3OE57@>saoCd9C{D
z&@Gwnt=_H3)@pe4^VI$lbRd$G%rqH!iFGagEaBcRSY$0&#L7V0#DEHK1vO7=6DIGY
zk&P5%-sCgY!lWTu7PJC^z1V#_4js{rBr^0tz-~VNYc_DQ^UG!tYv5z0$uPnpiSY2#
zq!x?@0Em2Mwim2MqTYtC4GYcBQ2dOIDL
z0hfh?hbB-(rh#G$CZT^K*7T*+(kzSEbm2v3W<5u+Rx8>SvZSw%(g5qiic{at4jWC7
zexlX?ViD=vpmB9k#WYs5-$Q3}cY8#TnM(xhK?MHL6V
zv<9)B`Xh-DgH0g+j@VCaZ$!;c4-b&~vsN9kWQu2D@s0d<(jPzLQ4Wh`rz!AJSCOy$@I9u=Bs~jOqinSQO#0BiG
ztQr+YGTc4~kL4U*Gylir#3tS47hutRMyevK(dF-Bs-kR``MDk<1xI}yraM{ut*M4Q
ze-ID~-+5%uu3E|IjZ)8Sw-@fw{?i1177#Ge*LexfNetAR$DBtdFz!2SRyUWQrCLK>
zrlSVT?)5HWDunOS(*to6!bw?yzOt!~Lo0BG$!JL&ZTnN92Qn#3_9Fo|02JPdGZ!!G
z&(+to_oVTIi*t@G$Dsw6C_?RRQEJjTtgAXFfd{vB>otMFsJw3bN^iYKY8Rl~Uvkvp
zBc{Bc47jFIQ60W-=2Cs(%4+kMhf-IWg;}RHwu*zK(y}QYIBn8T1uXuA{HrAOGv61)
zQ#ab~39_7%bEJmIZJP7%0Jx2Tn2}ral5W1X6l9@|6#{cp2EDszG0}FTKTtYY(|n(U
z&-zBnAX(>4T6`fY4Et2tf&nz
zlc##WUh{dgDpq%E0k|WZi;HF(MuYT=w>K*+8b>h`v9rFmK)9sXT5obp@n6k3`X|XeV=|
z7~vDVQiqc?@hFNq@0ec4D-af5L^(onXB9{Z*SWQ@^T%9y0A{JO4W;0Ymb1}V)W5Cf
zQ%)zuo-#%UpE9J9Onbuh2p;@`yCVqVkD(x>`hJ-Yfyiq`7UvdyIOD{ogg0&H{`QBl
z{dL4#wjBE9EWCpx??8+A7TA`AuKq`jO-MDnXkSudOdAq^Lbwg3USrO+Jmm1+o_mFB
zlf|XFux8Fc3XmvkIk$2~p#(i^(vJP~Xk^d5CDJ0=a5j67(2m-0JPY>*x$W~HU-@Km
zq_6eozPtdDGmp_f{sN|iAa8e_IDJ`zS!zL-Pqqsc_LLf6?-0*vNjOysbBr(>RkTfV
ztv~Xmf<5_&COnq*9_y>cE!1F>cXZh-46{U&UGzl)0L(s5cCi6S@L}H~J|e4mL`s&9
zA?_iqVxAqw5osXqf1WUkhf
z6<@K%GKA#0&!#OR{0chbo3{pYn4#yo5<4Y+rK$lPFq51P^gRsm|5H&!|{sr~%0+bPOU_pAxN-SXDV%hcJ2a3DYy1?_x
z#0G31oT}T?O3fHQm|grU5~#DXLm+}|O_yNR3=O!Y*qgR#gg-xct&l~QnO5Xk7SrS)9;XDfynW~Vh3qRUwUCX
z0FcWas}X-hUmBBJ8K{I6)V|EiqHLvstp#e%i7~i5!Uc2rEwKBCjvsGXlJE{{BT48#x)Vgg
zP7U7E6kW!yywZim%P|H`kuLIbi}0>4;1n+`3E@U1_tTG8ic%GVUsSTrkH+b%GP?cj
z=Ir?LdjGkf6lwcCE3aQB+qYg@YHgP`>=@0*n9-0m!TUx6~3DH8i5BNS^+yBz|N$h
zy7vn9fbGmi0Jg{{Wc$2uPHbb1o9|}N0{A4-XOhAtQ*AWq8`cq@Y$>Z+{su+=nbbdp
zu=wX|`Nw)agr_)4e=z;Kb;N)>t-*cPN~8`*N4?$gh9qlBUw`DS3sgHMaXunZ?C3r7
zA@WZq}$qG@oqmbOS=pdgz6&DB(
zV&8V`?kBU>8Xq5|9rwt
zIjl%H`1p(%v@n}l<*bH7*pyaz&_B(Y}zVuE%2g5leP*ICUW0LNKfj>cby
zEVNC|4nrxuemZ+Gq^@HHh44z=zr>A4K!Sh$Ez!Fcp6Qc2MPKKO0TrB?N6WRRQ6j~W
zbEYcvs9?+9t{b3{beK@}A@ENOoH9pu=Tke#t9h)<)DD;5ZFIpkNMgkVHT-j!1b>Jy
z{)o4E6|dE|2r(eSb^A>M2Y5iy>WVV*fS~Y3r@mw-`$(}SM#Id*_@fVP(IwmCzj}a7
zkTA=y^$T+HTUPhb9Qzl2f#KTmczngzV)bugE@`Ao{CtJA>vlXdH(4(ZGJaf-wnXEo
zEQC$c7S%c{6>37|Qx+lS+=nURe2xx{RPRhO>taN8#@|(OD~A4yFZ#^W;xBB0?E`SALuP3rYL_Fr>Q`1O-zc9q#WXTD%#Ev`54fxDyv=Jn
zCQXGeUD&*TAu6+?O#zIhFqbfK834p`g3;QacwbO!3vEsC2LRBYlr7u-0s&HEVvIHs
z$q_V*DE6{lzvo}ZX*v>4w*Z$jAfkl%T-@zp@0g9>2i-O-PSDZUAcIw6%G!p|Z$1@=onj
zOowuL0C{G$(;XEKWukn$;+R`XdZaS)2Pk
zeL~aw>uM-BYe%8)Hk6~j*i}9;8oBp^4?Pa9oj(=hr@!ZRR4oh;FQm$E2PR8H`9t-n
zxy2&|MaK|jrUf1ThGu1LZ7PmP{5^*7TMgIWN;{Z725xy;Ci4cMPaDT(Ft27kIY$X|
z)0H1NG59V|cW_Jnnx3iRgJ}BkE!f~I)kX;8L9wr91P%g%0}B+g5(E6kQAhhifRkfM
zI}Xfar>K&KLkESgYPP0llUZ0a8WS%Ix2ARXhfYJYaIy#!`~GuXedKL;0c)=(tC{nO
z|ApX|6F_8ETcq8;ulKZOdF=>z&-8scs4)cmbb=6+;tJg$NhZ%6!r!ixYK>GRuOyi&
z$+?~5hsLgy)*2Z`;sD&fB6SNTb3XsPDFR1qhOFfea`qXb1<>`QJ)Ax&h8mUx$JV&k)o0`rt@!K^ofWkGGg
zl4PPCjXAEvm(*gXO1+vHp2r#$
zN}A?-Bov;4-i0=^UR<0a<{Y1^p=zs*M~C%Rtx=u^&W$Dj-3+!~
zY@!nsiMm(AV;Zuyash0fbNToGNx?FY0V@xK<(wR>?`ga>$PkeH$?{mq`;Zw0{g4cJJy
z<)FZlP+U*CWhgeEG}0L~#l99rj!#h*QXZx=RIa1VAJlvjvtoI=6%*5zfM|yjl|bJa
zXX>lzrN`#5fs1qYzcyQR9?iJMo^~CgB@OiC2`Hh_MSH?!x?27_E0b2FBffO?6mF5~
zr{9$4f(!6FomME!>A#0L7+KQXR;@JE5GTAtG?)uyBuk@HQX2&
z`eG}qovqkcQ{5*<`dbNtJ+-gt(5lg*t%yov68|yDQrp4*^
z5cf?i)UYFe)AcN!0{@o$%W=-1PZ2Ogin2%Eexl;u>#73(`&JMrP)2wK!G&{6Peq<<
z<7)FpnRSD6MhV}ZzJ`M=$KH%B5YQ$dxNAiB2{K76
zULU}9K?gN}6=p6rej7?Xm+8u)(<2V!`HRx?Sv4opr3W64g9zU^%=mL2*(@ZdR^$&W
z9OGUawH#Ey9;5_@ukg+cR^*)P1C*(Oo!}7JRHv{ZSO7HfDJlWyjfU;nHD;aVVc$h=
zw@_l>Diks*>|lTBplHO+6;i!qP^4t7SeQ8mQjufSY_Jo0iWP$q+%pwKO4f_Z)r-r6
z<2_jDBc67L%&ordNq{Vdv`f}7sIuhA3^O
zLE~rjq1!cyZ+_7pib!VcD0(v?ik2}EA+r%#c>V#IX|ZPSo-*9C57-?_Nq;f&`Inck
z>2T#>AyGwBy-%jwPNw;PKb_P#?)+f8;R6FxWRW@}58&@ggjaKIwaInZEer4sF_Yz{
z;W7zJqNRkpZRSmc!__|;{ksJV3T~rr@nn^LR
zhX1GS!3yGSA2w+7rkM^lhqbhuik~`?DiY=XtDUP?lJ~9+N*9Az){uu2jim)Rfq8%b
zD#HVxItsHm$w6R1m9%^-H10JKl4@ywl~A3@e_#{}aaD}QN0w_ssp`>p!=c)EBQ>;b
zdne5wf7mSJjB`vRa4;_0>{eij?_)>(X_d+aPMy{vV{U?~3z^N_`LV-nXp+M@Sld)i
z8nX3h!)zxJd%SuTg}(x97Z?CMMf@!MnQoAa9dU6gjq*o;b37lB38C68KTDc7k`A6Q
z{9N;f@zBZ4TK=3!ai8b)7|$R6FfRvM8jCy@3ZF8gW{DArcl|YLa9ZyDb}f*)2AMCI
zOp$bem8lK8UgIPUx#Fo=8H)DE8OW#bo5>lr1$m+?KmB)px+8J{VO*8;vLBIh$u~qo
zLlP$AbU-AE_U2Gmtr!OH;Vqt2xQWaAbkPdn|X+57I=DcP!3+
z0j@Cv(U`w}Idm>%I}Nvg$Pu9NY%%Jl{!OwCAYp6Y-dwZWO7G$`pD6#Xv?YK7Ril%|Ke~;KmJF92ytJfYsF=
zD|yczz(VHo>I+$CMtBbAi#{;_J0luw>$MU;wGg{>m3?Y+1@F(bXwj)JyI#oTcAe#_
z#+U=1ZOri+!XK4px-IIo^o#Q#t@2d`{VLw|ys`@R1tsW>*-BA@W8iUXJF6$KV3cU-
z10Uiot|OS}hHjYj&V0Yox2=~6`Fb{xab6!60Qrnlif2ukZeydNU*#2Jyc8J_O{Ch!
zaj@MNZ*U)xh2saax6$ubK08bg0j-Ac;##Loi{`mRaSdvAT8-l-d!No%ko1k;bcS90
zRs)B7tG!NU(`Lh&mELW7(>&@PlykBO!*Z&51F4JBjID9KiXxo6kwJ@Vzx$zzvQ0w;
z*u!@_ni65Q8w#Pbt~O}a+j4=Q;hN($jU6IFi`wNM@X%Z4a!AZ`x^8QAugi6T6ICuM
zz=(B(rP0@a4kThL{cQ8m#Z`0LPH5RAH!{X@h;>YNE%s*+{v&KXqn>{`2d_CP89mfA
z2Xj)b3S=}5=yvR4rC|MBPH@4k$CFmXZ>h`%u(
zc#aT_^65v`XhLK4WG1omk6Tz?$vet+2I8w}_tNDd*us}?ZwE2HKR)Xb%aP!i%v7^k
z;C->}cub+nZ4XefpTNppXz9&+?rf7NpORCP1G4Zmd!DU^8&|?HkxnVa8NGZ-J0NJclf6jC@K`o;b-AW<{vqT|sXWQ*>2}H^5i#l^
z0rY31Mj6D(If6#B;|^OUSY=WpUQ2v{YMpIdn!kj
zwg#kNK*{%3jeV&7mrHBY$%v_)WG4hA#3M|yf_UAUBQqQ(g
zlfT|ZxQsU+=k|F*(_4%?iMVJ53n}^H{dFjzJ!tbA1Uux(d;nQkHN5_jv3yEs{g{xv
zr9F7HSY^0-94Vc8Nt!GbSR
z`9T3VMVy6{{>fCW^MG}`N+mr2W59#435(#V2Ra>V!v&|$_$9;&$KoxkkA+|4kHkI(
zzc#!cv}}j*dYYNF4w7Lj__rLw2vijq&8VtWe2x{E6Yhc1FY!CEP(fk|XQL9QfYhfS
z?N7)$cHXzwTkXmGf0*;KNT+qzejQ~KS2uNwcuz1OGa`BAE?N8Je2Mk~B$+2i;*Jhd
zk5EVNrO{r|G(0KD8kZe#kX5$HibWE5TSW;dT`ER*8RjNPLcI^I%kck+RKt1-<)Z9d
z@^0|oNIn^uR?lA_S?!HVv=gWlLZ=cEGc=gk*_HR?>-iFZkIAfCkghPpWkxY~%zOy9
z4XLp91g2)JXoMm^3J&>Ui6R$?wv2u##=Sx`%w`f%g5&#P%K{$;IN_FAAwZO5e?l#L
z&N~7*2NnNQXP2D?mV4LH_qKkM)23
zfH2rZKR-2cFdg#Up?B#^)!LNws!2om1^LyG+$&wppUB>$f?CNOmh2}n!-
zzax>;{|742BLYMJb%W0Ezd=NX|3EH=|3JL|hq3VgbV17gpQ9W*{C`m*rT%vmKdJw~
zYAHCN;}{Jnvh2T05}<-C)_=pHl1GC5pIEpF9FrCVxEbB8}+w0X>Z{6Ufq>9B?Ls=~2$)g=I36
zubpvZs>lN~|6Gxu9CB8Q=|(=7+jG{8X
zD&EOEfLxVYup+-&X^?+qP+EN`7H2eq6;5hYot%3~VDhr_QcPkkU?GDRnaO8?LV_2h
zm@In0LID?CnXLLJ8}{3SWTfHMJaGGsIf|*kIJ*eUx>JElH78HFXwBp^9n6fF&I5K2
zywC);LTyoW>dyizw4NmfbdMI3^jxsT{Bs3?Hfk{y%?ER3E_*O-STwnDk;de{%UVp)
zTPO2f0Y=zXHgM#A+k+gr#(RO15lof`CUYIIncR0qVDgqLQcS#uCSSOc%Jksy
Date: Thu, 22 Mar 2018 12:48:27 -0600
Subject: [PATCH 154/229] Updating to latest 5.0 API

---
 .../ClearSuspectBlobDs3TargetsSpectraS3Request.java  | 10 ++++++----
 .../ds3client/models/SuspectBlobPool.java            | 12 ------------
 .../ds3client/models/SuspectBlobTape.java            | 12 ------------
 3 files changed, 6 insertions(+), 28 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobDs3TargetsSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobDs3TargetsSpectraS3Request.java
index d310a8436..52d39f14d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobDs3TargetsSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ClearSuspectBlobDs3TargetsSpectraS3Request.java
@@ -17,9 +17,11 @@
 package com.spectralogic.ds3client.commands.spectrads3;
 
 import com.spectralogic.ds3client.networking.HttpVerb;
-import com.spectralogic.ds3client.commands.interfaces.AbstractRequest;
+import com.spectralogic.ds3client.commands.interfaces.AbstractIdsPayloadRequest;
 
-public class ClearSuspectBlobDs3TargetsSpectraS3Request extends AbstractRequest {
+import java.util.List;
+
+public class ClearSuspectBlobDs3TargetsSpectraS3Request extends AbstractIdsPayloadRequest {
 
     // Variables
     
@@ -27,8 +29,8 @@ public class ClearSuspectBlobDs3TargetsSpectraS3Request extends AbstractRequest
 
     // Constructor
     
-    
-    public ClearSuspectBlobDs3TargetsSpectraS3Request() {
+    public ClearSuspectBlobDs3TargetsSpectraS3Request(final List ids) {
+        super(ids);
         
     }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobPool.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobPool.java
index 6e9c5c18b..4a304e644 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobPool.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobPool.java
@@ -40,9 +40,6 @@ public class SuspectBlobPool {
     @JsonProperty("LastAccessed")
     private Date lastAccessed;
 
-    @JsonProperty("ObsoletionId")
-    private UUID obsoletionId;
-
     @JsonProperty("PoolId")
     private UUID poolId;
 
@@ -98,15 +95,6 @@ public void setLastAccessed(final Date lastAccessed) {
     }
 
 
-    public UUID getObsoletionId() {
-        return this.obsoletionId;
-    }
-
-    public void setObsoletionId(final UUID obsoletionId) {
-        this.obsoletionId = obsoletionId;
-    }
-
-
     public UUID getPoolId() {
         return this.poolId;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobTape.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobTape.java
index 8d162caec..1f628d2ba 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobTape.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/SuspectBlobTape.java
@@ -30,9 +30,6 @@ public class SuspectBlobTape {
     @JsonProperty("Id")
     private UUID id;
 
-    @JsonProperty("ObsoletionId")
-    private UUID obsoletionId;
-
     @JsonProperty("OrderIndex")
     private int orderIndex;
 
@@ -64,15 +61,6 @@ public void setId(final UUID id) {
     }
 
 
-    public UUID getObsoletionId() {
-        return this.obsoletionId;
-    }
-
-    public void setObsoletionId(final UUID obsoletionId) {
-        this.obsoletionId = obsoletionId;
-    }
-
-
     public int getOrderIndex() {
         return this.orderIndex;
     }

From c84b4873faa7b1fcf69d9e822b57bf36b50fec88 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Tue, 27 Mar 2018 13:11:00 -0600
Subject: [PATCH 155/229] Removing depricated serial id

---
 .../ModifyTapePartitionSpectraS3Request.java       | 14 ++++++++++++++
 .../ds3client/models/DetailedTapePartition.java    | 12 ------------
 .../models/NamedDetailedTapePartition.java         | 12 ------------
 .../ds3client/models/TapePartition.java            | 12 ------------
 4 files changed, 14 insertions(+), 36 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
index 0dba73b94..2607b4797 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/spectrads3/ModifyTapePartitionSpectraS3Request.java
@@ -35,6 +35,8 @@ public class ModifyTapePartitionSpectraS3Request extends AbstractRequest {
 
     private Quiesced quiesced;
 
+    private String serialNumber;
+
     // Constructor
     
     
@@ -71,6 +73,13 @@ public ModifyTapePartitionSpectraS3Request withQuiesced(final Quiesced quiesced)
     }
 
 
+    public ModifyTapePartitionSpectraS3Request withSerialNumber(final String serialNumber) {
+        this.serialNumber = serialNumber;
+        this.updateQueryParam("serial_number", serialNumber);
+        return this;
+    }
+
+
 
     @Override
     public HttpVerb getVerb() {
@@ -106,4 +115,9 @@ public Quiesced getQuiesced() {
         return this.quiesced;
     }
 
+
+    public String getSerialNumber() {
+        return this.serialNumber;
+    }
+
 }
\ No newline at end of file
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
index b54a5e473..2fe64a482 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/DetailedTapePartition.java
@@ -61,9 +61,6 @@ public class DetailedTapePartition {
     @JsonProperty("Quiesced")
     private Quiesced quiesced;
 
-    @JsonProperty("SerialId")
-    private String serialId;
-
     @JsonProperty("SerialNumber")
     private String serialNumber;
 
@@ -180,15 +177,6 @@ public void setQuiesced(final Quiesced quiesced) {
     }
 
 
-    public String getSerialId() {
-        return this.serialId;
-    }
-
-    public void setSerialId(final String serialId) {
-        this.serialId = serialId;
-    }
-
-
     public String getSerialNumber() {
         return this.serialNumber;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
index 757d401f0..46787b2cf 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/NamedDetailedTapePartition.java
@@ -61,9 +61,6 @@ public class NamedDetailedTapePartition {
     @JsonProperty("Quiesced")
     private Quiesced quiesced;
 
-    @JsonProperty("SerialId")
-    private String serialId;
-
     @JsonProperty("SerialNumber")
     private String serialNumber;
 
@@ -180,15 +177,6 @@ public void setQuiesced(final Quiesced quiesced) {
     }
 
 
-    public String getSerialId() {
-        return this.serialId;
-    }
-
-    public void setSerialId(final String serialId) {
-        this.serialId = serialId;
-    }
-
-
     public String getSerialNumber() {
         return this.serialNumber;
     }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
index a90f131a4..1def8e1a1 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/TapePartition.java
@@ -54,9 +54,6 @@ public class TapePartition {
     @JsonProperty("Quiesced")
     private Quiesced quiesced;
 
-    @JsonProperty("SerialId")
-    private String serialId;
-
     @JsonProperty("SerialNumber")
     private String serialNumber;
 
@@ -160,15 +157,6 @@ public void setQuiesced(final Quiesced quiesced) {
     }
 
 
-    public String getSerialId() {
-        return this.serialId;
-    }
-
-    public void setSerialId(final String serialId) {
-        this.serialId = serialId;
-    }
-
-
     public String getSerialNumber() {
         return this.serialNumber;
     }

From bb95fd6430e417cda78ef0bb3ed69cde045bd193 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Thu, 29 Mar 2018 11:02:34 -0600
Subject: [PATCH 156/229] Incrementing version number for release

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index dc50edb0e..2b96178cc 100644
--- a/build.gradle
+++ b/build.gradle
@@ -36,7 +36,7 @@ plugins {
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '5.0.0'
+    version = '5.0.1'
 }
 
 subprojects {

From 9dad9b4a51560a9e314e0f60c15005d4089a831f Mon Sep 17 00:00:00 2001
From: scribe 
Date: Fri, 6 Apr 2018 09:19:20 -0600
Subject: [PATCH 157/229] This prevents a null pointer in the case where the
 versionId is null

---
 .../java/com/spectralogic/ds3client/models/bulk/Ds3Object.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
index 38e7012e1..a23d4a84e 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
 import com.spectralogic.ds3client.serializer.Views;
 
+import java.util.Objects;
 import java.util.UUID;
 
 public class Ds3Object  {
@@ -126,6 +127,6 @@ public boolean equals(final Object obj) {
         final Ds3Object ds3Obj = (Ds3Object) obj;
         return ds3Obj.getName().equals(this.getName()) &&
                 ds3Obj.getSize() == this.getSize() &&
-                ds3Obj.getVersionId().equals(this.getVersionId());
+                Objects.equals(ds3Obj.versionId, this.getVersionId());
     }
 }

From 704de1b058803a2f940852112168168a009473f5 Mon Sep 17 00:00:00 2001
From: scribe 
Date: Fri, 6 Apr 2018 09:52:00 -0600
Subject: [PATCH 158/229] Now name nullability check too

---
 .../java/com/spectralogic/ds3client/models/bulk/Ds3Object.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
index a23d4a84e..4c4a831f6 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3Object.java
@@ -125,7 +125,7 @@ public boolean equals(final Object obj) {
             return false;
         }
         final Ds3Object ds3Obj = (Ds3Object) obj;
-        return ds3Obj.getName().equals(this.getName()) &&
+        return Objects.equals(ds3Obj.getName(), this.getName()) &&
                 ds3Obj.getSize() == this.getSize() &&
                 Objects.equals(ds3Obj.versionId, this.getVersionId());
     }

From a608e27dadd03f19a1347bcfbb8d4bdd696f0f3c Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Tue, 1 May 2018 15:57:28 -0600
Subject: [PATCH 159/229] Reving version to 5.0.2

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 2b96178cc..97c871e71 100644
--- a/build.gradle
+++ b/build.gradle
@@ -36,7 +36,7 @@ plugins {
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '5.0.1'
+    version = '5.0.2'
 }
 
 subprojects {

From aca0e95d4d7211ddbaf32b52b6828f59a56813f9 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Tue, 1 May 2018 15:59:31 -0600
Subject: [PATCH 160/229] Updating gradle to the latest version

---
 build.gradle                             |   2 +-
 gradle/wrapper/gradle-wrapper.jar        | Bin 54708 -> 54329 bytes
 gradle/wrapper/gradle-wrapper.properties |   2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 97c871e71..6c7b9012c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -64,7 +64,7 @@ subprojects {
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '4.6'
+    gradleVersion = '4.7'
 }
 
 project(':ds3-sdk') {
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index 7a3265ee94c0ab25cf079ac8ccdf87f41d455d42..f6b961fd5a86aa5fbfe90f707c3138408be7c718 100644
GIT binary patch
delta 15814
zcmZ9zbC6}r6D{1fZQHhO+qUg9ZS%Bk+n#CL=1kl6bWiu2d*AoPFYY^kRn%U&t9C?I
z*2$GCTWi3(D#4SuA!rmwUz7QDptJ7W?%Y5@lJJ^Z37rvJua-cEAVEO5kw8GGfUIeR
zz_=L%K%SPqH@YUqClz8A2ks`C&5|27Tq&&MvYv>ZC{2eQvy45xWSA{mdFYZtrb1^_
z%*zGyuMzH}5oj0K+DSd8f`D7=SMlW=g>s-QRWcK?pH;!s=d<_o=dhIC2~B_nm`%8i&h{0-e1qO^IT
z8tj;A!zK<`{|oQ2YISi&tmc;XGHm#gvD2bTt)k{)%&V>~Npp0WqW+C6>cM>>==jCJen&TS66JYLJXJN~`vU-n3X6~ojkti!u9Tjvh9XYd2k!&UAcjHtf*=Kwo_Ci{(ckRM~2kA$4HBnEuSSMz5dC8kWoH8y#%kmSD4R{0^-rm+|Uc%!W|
z(1Nek<661DHDigQkuHt-;bf!>enzs#7ZDj!c`>CD;w@;3bF2se+f)O0_))jJis4+9
zfiUAOu^?NxYM86^`dho`Vngpe0ph8-9!jYcuatVxmF7vfkwAAY8D
zQ2Pr#U~+M#kot*j1n58t4f)<^Jir@JP7;a%^s9Q#H`H>0vc@E+bXwk3atcUrg*5&p
zzxSRN(r_0a;w*W(Gbg7(c$ZaXuQ%&8%<^4oYa^4}Q%gX=`|iUwo#wS3AQN?d4L>fh
zmbUxCMio+{!Q&}dw_~>KvhcR-ZIqT`Q_TmNRFX|ycCB$+i*j$WbZoQJth!ap)bxT-
zr-OoKg+)c-pd9~|VIIb@&KiL7fk()V7^)G50*T*`M}LfnICod@iu9=#MMByNxfk^}}96GD~_Ooeh%4AN=S~4+5k8gHaT#JpT-{
z_c*tH3zNJVldCJKF}OUS*(%=|>BwP*_UB;f%ov5pAbgOR<>l+`f^=as>8KN0HGx06
zK!VzDA=L4(55DBkBVrXoSzr#`HXlba_j|Wg5~q)L2wZ$y$Pn
z$mb=7cM9A?iC;x<#hgfcqfQ~dM(6CMlgN9pcTzQscN){RT(A%Ku1$p0xRA8lIxu%@
zM+8lY19D>>!D4+6u!e9*F>08^UmNQFOv|V9I4f(_aDhyNdlwj|2ybgjiI!7f5^L^7
z?Pp^}M6L_pk){g;k8UZ;r<-DAHY1B!i1sVo}PN5V9QP5&!XRfFXfVewfJc&SC6yv>HMgU4Pv}+-+gg3}O9BDmQc;=ncf%Ck?
z0p`6Tp2pytTKViw{Opust|T+JKi=VJ#emHlhk*-U&?@ADK{y2@4Do1P!0)guqYmiw79dx==z_Mfsdwp_>F>-fX0C6rc*Mh_7RmG*-6n?k>%wTv14Q3M;o?r5JMf-t+bytKFw>kcqc(
z@K+w!s%8Wi)A*%BUDdI>Wt~h)LTj>i41G-1U7+si(KVY#xsh%)Z%GycfaJMTa70a~m
zQrVft8KbN66q46EE89p$OCD-luwjsjSRh9CS9Hx_3A6S3^Pz^M%|>(WYKON&3RynO
zH`50(pJx&{RCcfFUcQpM7+9?hgnPi_>q4-65X3yweibo$
zCMN62m_R=3W2a@P>Gn!8_V041%X8{fkr!J!;rcy3Z-->NwWpA9^ViZ|wlPI5FR*Dn{+KHaU}I6|3KJ(QYP1|!6?BC4
zC9F`nv)fIr#l3(^hleUx1rb666~>7ENz!D)EtQAWy$!kKC`vrdF)@GdMW3#3UbVUG
zPh2-*h9&hzt;Q!f0&VoY>At?_e;Sr0=R#tLeou8Q_q!fw52}WD94%L$+kP-B-ek=5%Br@8C(Qa
z9#2ETPR6`CgTip6HG;y5Gc!|NRfBadsjBmd4|^qrks)Fs6eir)f(5t{7DRq*a&AtaQSwe98hQl(6|!3O{x`u>gdq*
z<&+5+#k8+~RaZQ2KuG6b!6IhHlO6GdVyPuM$UK-BDa1_e<9(xJizBp#4Q;=^_QSj=
zV$?V9oUzuz%94cAed|t^WahwxM~H2DM7-1Mypf1p!aw7pG~d>6rH_@|w#wvHMP^ed
zuz1HkR^BD_2tDuyq?s09N%Z7tlg=jYiB}kXAtRzYjsO%p7@2lVjOv8e9|Vn#XX{sv
z$W^3*`!j9FpNUfiCDo8IHmohq$gfas;o@=g2t^j+nPo14)rAGSe`Mw-Pfks9=GU>@
zEbyJW$rIH@NMspGPB!%!T*y;Qt4Cz_Sn7FJA-Ao4H@mw2mKSVWAh
zZ3Qw?g+;(gGHe*;qXM!&gQ2Vn|3LlMOV!J{&VZzWdH|bJQ4;1b5IR{E_~j0?Xvb8p
zN=I%}MQxy=bxYM}_-EKBJpU4s-e~(zz9r^A#D7rp@_X>bS%MK0*#Ed0&jw!e&ILYi
zx4bm`4f}{9c*vHy!;7qTY;1!%`{NTyjSS?m@}LHzmB>+?<)${PdH_X74vr8;-;}s#4+&~Sjpsta;WHul
zp+;K_$CYMyWrrOOo_-S?*D)ssz1E4fRgcf)-+kHDRZ^Uqt&BUB$K8zVyURTR=+NKo
zG{Zd%KTo%^kn$zmps&Fc;or-vg1|cn6Fp=!ZAN$)atss_O2$DV#%?ImdUY%EJRSsd
z&;g2i91jQ!LC>t?k}bBHG>Q^P30|gF;oERZrV0ZzyC*G+S3Igty#q
zZObH^r8=W>+>@&&Mwc1HU6_U8kEHcRjsdS)!7)1v{igxgXZWFbr*h;WB7y*X!0?G{w%P$9*SHh!nSt+1$RIN^&%opc1&G`
zNp!tOw3b*&cR(OMFqpddi8BcDvVN4~eBt0(m_b=#
z7}Z5Wcff`9g5~uJ-;J$7`Xia4lhYqTtAOsvnq8EP4C+7{0YMm;dnjv$r3OHu(Czl@
z1gn!-c9&*suUIW~89L5mVZ~lV8e&yGXSZ(sbxJ7}%X1Tu2%nG`%On1W&el66f2d*t
ztt?0lZSII+|B+5fHSrh9u)8nfK5Om><6wBzm%;rW=##PhJ>b8m{(C!YwICP>$SMQ~
zh}7TP%GnLbl|~Fq38n^YIW38x2woU$Lq$fkBC2TU$rW~~atDgis6t4H#bAJ{Rrfom
z#BWSC(Qj3~Vt$e`_hLjt53>WpI%hYvP!vgIXeMVbU#GKp5AG^%ZiGQ94`{#%M-a4@
zlLrMox7V5)1}OKxZ~~%@B(K#j@|*If<&D}p$o1Adu@)_?;&XUUWRjQ
zjETsOrbrI9XkOEKMb*Dk{QGZXyj8&
z)IUyxS|ujRY^VSdq!4P|qyI>LX&{7*29$0u+fpX<J
z?qe2ZQ5EmPOks{}Iu}UQB)^UG#6nUTdtJySdwq(t1V}9@avMFzAdj!aDaA%Wn^&-e
z{X3%CmiiHO1$%j~SR=DcOT&1hGs}8nQX87(Xzk>d1Zi#RnCCsm`M?jvZ3(iMFgvFa
zPGT}qVG@14svwDKsW14_3V^Kue%g4jhVvLj;<#OtUAvAaDcJOH`8_VLQ(v$i(odvk
zF@Nl}!F*0U*Q{!XJuKZrtst=f`kAu<@AscjARvvXARvN&xg`o9OBfO0>>p<7tB>m{
z%;N_djgCs(;v&}$dI$T!fvBTAlt+2e7-oY~KuhVsHzd4{*ybnFd>Y5+YA1dw*;(-#^Ps`r!6Z_6`;|$oVJ^
zUVt?(*xN_!+U8^vW!zG!a>bR*H!H4bws02dq}#zRci>y~3ONDjn!@ym6>XmCq?8kj
z$2K_)cPcZ}?b72jj44%4^-wyG$%^X{T(Kp`qI4w)(3@OCDAY{;@>z20;*0h0c?u|%
zQ&$%}L0)ajvJ`KrBUf8#LS;g>_R+t+W}0oCr7E!2F!^m$_M2t5YIZ!u77m_vyK;2%
zcItF35wV54E97+{JQcEoOGT&rw2B@+P}}JEv=g4O9~DmQK*kv1sj};?ArG6GWOFkgFP25jA$*
zRr}hDzeBRlA*@)RV{E0DL30$~IW&~EMr%9m8q{l9=NE9;wgQwX{xj?YFZ@9
z^yr!VOw3yjD;f@=>^0+!J(}*J+7WII=$p%I?3P5X66||U
zXw|7SZW8M&U#J*a);Sd&LXc!O=~fyf#f@~8kb(eQSQ1D^#(B?^?imIj@si%mGsHGe
z&Jgf5djnD}%39`bl2@Z#X%i-Wh{q@bNDjmaGVV~3Uy(?d|#IWQ+$-Ya$3|I7;8rnp}4Ugb(GDPrp`4
zse^2~533%#avIaZ=lm8uhd@T(v`%rBife};dXCK)9kx*5guARvIiok_>9a8GGDq9!
z31?#QyjBIKQz}tXZJEZ&WM8P?<~iBED98tZPXQvQE#RCUNfnlL?*!_8o7)Lw<9iAB
zqv$hZaHQTaVB-XJOE7ES@b`qGuVMR{2frw)Xh?YwaqJE=g2q|3nd{FyY7p|423(Ia_4ktaA{Q85X|PK_cntl=Nm|L8i!jP(e>
z*vvsKCKfjR?W!3?!p)Zr*|o-NZ09|j!I%F~<|NaQa$%bvsW~#TEk-yYa9!Ezj$Zdn
zT0kSB*y#u3Yswn(af|!fEv3Q|Hs~^gQtEPr^0BA(6U8b_J{phb$i|__6_vv8l|V$~
zQaYL^3UVK%9xYOpDWOo0U<7YpHv$Z>ny6I^nTz|LcQU>H5MVvp|DKEMZ!EIA6QaK)
zB!>i`5Qr-V*YSaJ({?Gp(Vsq_Lk9(S!SQAe2&ZjN&YZ#C{dxWfA?CyD@4ftX(`
za!{(1&r+<)CMfK4CPCPt{%;e}<<@t1BiO^9Xry;#=1N|ezQ+*X4F$**6I4|=49E-0
zpD@_+9up_tm+q5OfoABS6C_^)B1pw%xsgnEWxA?40~bp=zw`{(uVV~Fk*iG&&J>7l
z3FY$wRO!I9EfasxMH{1PJwB^VIu;fln`2{{kvmc6S;G$T=RydIMqe+&@FTBT^~NYUgwOzv->{g)sLoP&
zp!UwQH`R=72VjP)WK}m0bPP}K%YbsDh8p>US)RU-WCuBItEZ`csM7m
zWpL}SxY`xoVw^WT*wj4RyZQ#o${uf7j{DkZb{0<3FZ7CuNN%S5bXL@E7`pw`XwRU&
zIxSPE`hF2gPG|vs-U%ADc2#1p3CcRrcPU2DWO%0iY|20gO*j>C@DlY1eqQ^NO1?_q
zibJ=;`V)G`?S`a`lGl@-RUvOUT-4ax
z>!|&0T`iW)YuZZdY3K*a;(5D`$QUlTWgNcFAR%~$%f=4?8`VC^su{q4n8iICi+#GF
zKNtbnw>7{L7_tWuucULmhjjM&!`neUJZmc7&(z~PW)Gse4?>pF?D#x118kcdxg8#b
zZB!-f!UT)0vP|qedzv+L)FHeu>JBxh*fGfijqkO|x4y_(irK{t9g_riih&G_^aW3%
z15yv(FuON^7dvcDri-N=FqD0U!{qwKh^)EzB2!ZCnkb6CW*9QNuRb(@+v
z0~SHQa{#4+BCix?LYoCO+%7sxPEl^NGHKF0FLs|R!
z5+3ccCXpR%2qXha(UdMsfMUvKz7f3f5BUVgCY^(X^SbBvJGt&)+waA-H(4dw!Lq|P
zJd`-BPw4cf80XkpgPM!fXVD(}#Q*i?(BCLkS$}hg>c7tX&wo2Q;E#DMK%M4>(~=5`
zAh+wmu&XSMqt2w-Vk*^j3Rh(+6GK^XrJ6=k2w7T3&M*Y_nA955RK+c1?=1)#qy(kV
zw?wgNMMar3l8l&)IP2qSULe2w6|wO5KSFGn2ozVUuJAMv;#U6q6E9>Y8
z3#!oHB~OfW;UeAqtpk_naAy
z_kh`V;aI6f=Pvw+V+~Fjr`UkQawt$1FZ$xLbJjWJFxlz`u^9g5)h0)8#4)zJ%W~pi
z;<4zYEF=MAkx$87c8o>otTWgZbBfQ&Omd_;fwPvKA)L-x;7PH`5_Ybk)m}8))I&w5
zddDo2pXg-oZT`n2z)0I|zMiqbB?6)4%PpPm`Y-#;7trP&o3|3Lzk4&F6n4co#Ny0x
z{rwrT!L7&f(=x02k4E4*hr3ZjPJV{IUSTZ1XIu?U#^W2Ht9eW?=Srvz=x#aWItkH7
z>gne##X_e|AX9O)s7?VF{%jLQsUUmW$0o#L|$$tjcPB%%1vU*pG9lcjWJ5W|)t|w2yX#%VbXW
zYhy?ST9ZwPR8B+OVncg)m&_f+Q`?VCJa6nLspq$0r(br&-Pzd|Chk!!Z$vw4+8r3D
zQZ8-kz!BbNRw!EPvv)VG=y_c%7s+JfEHEqBWDFU)df{9oQBB&QP)YhI5rWK7?@d`bOsYP@%=HjY+u>6ly>q@1OSZj2bdJvko8x*
z>UX?3L;PF^HHG$F_c`BL_u0P$?(;>%oG}Q-otP*dJKXP)666-PV+@&0?eTUJbJfw5
zkkWPPCI)lK9|hw&X-!n()~C8B0InBqoPA%<3{g7Xcd8+g@U3I7mL}_N=QzPw4spy1
zE1jN*sSzO!xljIURtqnr!0UC&3Rj!Xh?`D?s%FO=rwKNJIf`Mk5!Va79nNP)^%`9R
za)+VD7s<@Eh>I+zYc}X3sx>`+xJNnwr0B0IY4T+@S3btBaTeSw7f|CuKnqvrEnYPN
zj&y_-u9soWZ>Ek5KNIqCjsBU=7H^Z)fIjt-bjw=y!tg`}c~xx3=)MB$aYN{tdHo~L
zt=pVpr!)ZN@?%%oAKGU>4@Ryiy(&J%4y*pd;x}``y%PveI9!DCTMT@*eAZfnmOD|#
zEoqzD9?{zq3y1Q?@DmPyK*rWUBQ9>Ug^$(7v%=PakT4s+xv~U_8jf`Io{`211k;qt
z$IaQ-g^B11j#PJB0u-+wL#3zZ5^~7AC&6@$qqz!a;GfAB2nK|bt2@S6H@MS%j!%u}
zfU=yv_aA3=Cl*HCB7d2zk0r+JV*aLVD~@dc+SFT6SrKvz)OhCq@Q78uEC6_5HfhH#ZLHuuY&Lj6|1=6IsLI8?{D_Zi>!axxb
zJ>tYt{|4;bzhVz|AZj)X&?o~5(Bk;FRAtF%TxOk!QE=F
z+q~oGguSe3?8n!
zI20x7Y^mxh3%vYDE)*uEDc#_7g{yL|p~{`ehxVAT(U$H+Xn
zXwF$O;U>A|&_}6tn#J%`U@eQ*iG`vjXvYf2z*WHCv!lxr`sOv=K2|9xz6W2xMS_4D
zxX_Nyc52^j+Jg|@AUKCq&4etT<5h<#Y@-wZZOw42x+_8RDM#^oMKZQS0OpDWfTs+`
zhRE-C7RHzX&^=N-XE&e$c%f$}e6%<0qk`k?8pv2X3t{rYA3fxf&zj0CnK$ri3~;m`tCvIosP|b*Id_qaI_6qS
zcMnBDtG@0=x8q868#vv8_qpxbZ5ZX9dhyRE$8aubxc7CEN((cIf7dLEUgVjWilJS~
zPU#}`b05NRR7)-NoEH!qWFRE5o&8%LkUxzIqdW^-j5LM&tqX>mK1_4u?uaw$LYS0q
zGgL~AM|-&BIP53{n8eXx`%a#vNLYMBPZcD#BOs<1_9xZ4l|bx0;_e_mm>u7rI`_8(
zW4RoWUysvtXmkb2OREj0KkWm`Z{NlsxLmtXL{cv7&@~w%*hsr{tAxVKg7AaHu$(J$
zYIYS?+&Phv)ZE}(C-TIw5Vse@pUGG1B1pWm
zhfHPjns4S*BphMm>OkggR|$irLH8*!L6U?_*hFSUKp9F?JFJLe+iwRsTZtXzz1{<_
z-VMf|qBqd+G8{$>_po73!?Y2vNm9}p{z=@Gj6mB*u_Cng(JzKG=I+43a;6{JrK?Xu
z5eE5#gp9!d&k#VTTg}G*Mun?buAxGwMZ{$Iwf}@(+au#GM>68$dm%2Ax5g5>syuPv
zfpVB$vRhukpUqRjYpSwl?+>l{q`B1o9tY(vNWxSxxHF9OCbK=*p{0I;8nRSQ=2W9f
zGEXx9$X`LkVf8#|gFMX$Sv9@5tDo*xLGr7%SVQZcJ7$2)YXCY2cKu`ag6M~uO=tP<
z5tHQnY(;lg3FIY+wRlSVOom8`cmy|ew(dpJd$Re9SY5swIb)NVCGUjbXJe&;Ym6xy|LPozJc^t41jC=*4$v+W9Kq0;<$>VH<^e>B
zX1t*}6x8FwQDjoGITX>;AUj&9lnMdR|JNZg|Npvfw65Yi^xrVr^Orkk`!|V0&Lbj$
z?5J{A-BvMiSbno&$(G?@CEY@sCVKV?j_{=IIBsVLD&|oD)}*lbx!l$#O6(3`^YtqA
zZBX2p1JA}9V6yxU!c4pdi==g~elPO(;=Bkxi9Uiqr}pfLKv0XlZRYdLyiO;=gqs(U
zDL1fL**{k#f0Xi5?{B%P^ID(irq)d@KBcc^+j
zHNT8ZX*9Y4dp?MYKifNTbX94$UWKjDxY9v
zse8Hdv#(DVp-c{`7^MWVHx!heU`$peb-39S$$}JV86$YkuWoS(36pDl{Y_C{2H`nx
zci;T(@Yz`zcT`ZZ?jdSE`Rpz`vaz0H)}<%JVjDXE3jW(W-$4nV=K4exeEL5QVYgy)
zdbAjJpk7$8?tUp4|C%I&Xc+`(BmHhNIYqrib80$jK8z>nc?Tcm)y%
z&WWBQ|1)R0y*yJD?#yO95;%;_5HG9GJ~4|?aEV#KKva}cDyX#I;hJ+q#TKA86l3cy
zh3Kv8)D#B%g8y$!GKAvT0rD1v!P0a0V>xYiE*Q6r7Sw?u_8K_rX8&cn8)*Kol9C+2
zWbR<D%p^fi
z^;Oa$s)y&N*n%X!b|Wf@w04xjBxq*wg|gaOgeP43Mi*o?NM7HQkCutT2-W7s&&NxM^de%TT3MG!K%_tY;1WcO&L9YB0dV%MEQp=sH)NT>SC(;IE9RGN3>
zR#{m?9y%Q5Q5nuAyL)bs%z)Y!QN=|%EL#E{kj8!OS~P)-U@&pzGQ%--@Huy#nAp?X
z>p3uzd&q-3@DTSw8DcX(8ce
zZnPa@S1#_XrcO+I#Y|XvprgRu_Q9ngN#=3$HWiH>(#V0p=7wXG=aznJSrNVINbX7}
z)fOcCWg5G9lTUG5L!RFiJ9!>PHxJAg$KcwX-&7+ai>L^ZQ
z#yeTBqnNcc?ZVUjo!8^Xxk9&m;RolA(Oe9+cX>wTWz>fAg*d>qMbKd(`ZKLahNemH
z7cISYYTre1UixAIb=%{WeTSJoMp2=Qc5XRe}UV4oco
zPoKk>kN?z|T;m2G`=N@M(by^dj4P_Rj-9}~X_?xI*g$hLLN(Q>JgrdI0fPQssWoJU
zqK(Bwr$7b9cTB*Ug%dlAqC$ub-BPN!;c`Zjh36wnL|}QI2FFSKDqIs;
zmh4@`xXn>&YCQU3`dBxk4LN+1ee=X6)u{cb8A1iOgCu}Q;Q~kh7qPLXEQ$;@{0cJx
zHCcYVc%4=VLWNaBZ7#v$X
znqD7k%@mTPT^WgZ9b!@E;&RK{Ikkg#&iQ!9stSoQO7}
zZt#cMWeXsfUw=5O%5w40npW{Bz|=3a#Ja#&q|u}n+Hk4EB2ILSRDaJZVheg8&Hrqe
z7)@K~Cf_Csx4oGm6ky0f(9U-m)l=3Auk#l5NjIJ))7}(_98R;{!hLY25+@j16s3Qb
zKJ;^UZKceN&3{|e{)*7kAmjJEH6U6Vvc6U&q7LBNA>*jDSgCBVaHh6AlYm4|#H`zG
zlU&&9V1`0-Qo(ofqgZv;GUj1IW~(c>SCr*g
z$roEyp{ae->9SHUC;t1;Oq?nCPXs|ykT_G?R206Y{r-_`j3y^$gaMN_O|derTbY5R
zL+i^*`{u}rE4+X9q25z{Hk=gz%C5|>Z4{uxfX9AlSN2$YH-ZZ1UEPhk=^1ZeKkn()
zM_Qnj&?(Bqe#q`tLdrd%84-Zhf$P>1paj%M=+czM$N%9f>o|UL1b^9TV4_Aj;oSn(
zOKGKx&}ec;@m{*{9)5rAc<$Om>OG!E%MV<~-LUthVoy5amEALU=8V{;^tR4gL;=KR
z{ejEkd?Q0Isb2pfyjOt8TS?#}bIp1v@76)}g7?}Ku#xmW+E_8Lne?tX>MMUxmRvjR
zUO1oBAhVn_bgxOIW+#1SuTbja+GNIZ2&!q%z9Xs(gvR^>;vXER6;-(E7_p)Le09Lh
zx^vS!;;O)Gq=RRyLlA?VvzKY<^$fuFzSqZZ;lR8H}Thd5J_59D%RFUd8DIEm%0jj}RbBw7XZ-M16{i!ZFL>r
z=k;Z9hS9#n5T5BiT^nK^MF=n{&sU)zqxs*mQ&D8!3iSnup^o+FDO3I2Vcrd)EcobV
z!=(|LEekw5{N=Gt7}WeE@Jz*A`J2K!W#U)ha|1})ScNxjUkGcbn8#C&YdBLi<~nP?
zC>`<78>I;_Y`8{#W4UZYtsp=z5uHyil2Y4#%XUNm_6C16mia#a%5cWu$g^$uL7Pb|
zn%u4-Y>@7;<+d&SSOXfb|Flo6CyD8o$pF+Oev`3PYFdwH5#;;yrLaQE1`u~%&Sw?ftmQ1wlN^Dw#Mq=CJ>;cYMW&`=!K2`
znQmWsfsFMwF_a3tB4)x;+&2<9{%i;*%UcEY6n$PSrNTU*@?P@5m$-C8tM!DRp%&z4
ze$$LDsHsm3TI&DP^l4~GTv1S*DMFUrKb+oGo&VF3ay=uix`AI%ab3;l;AEtqf5E(h
zmM{rBzEy&}3IuCCl@`E`mll-ow8EdapYHDzod=5LwRE(HoiVXY82kL>1YhM2&t4?D
z^G9=+F}L#`h|341t7gpZKQ}joH@KG=Oq?F86uQWLYOBk6{ehNT5dSSfd`
zvkJkmCV~&t;lOP=e~yg+!@Yip?HA^xp!46c&0uk`jSU&?lg`dk`zTZs{dXBPao@`9G(Hjgqe3>g*UY>zrHz
z3dHbsv)_92obC7a;c>Swz8>5m
zXgH>PC#g#7)g*A|Io8r33g(LXVQwoLlvA~U#j->y2d4sP5$}wAH{Rc#x-9C_l&5N!
z_z~F*36=>!Jz$ViNOelD!x`o485G$Zg)tJN1#c(X@5|imt4y_|Ymx2Dp69US7bvS1
zgZTTyw%b?PWPlVSxjSq*GKo7xE#e^cCMjIvDag`mUN|nC_whx6a;iXJVHHVOak75R
zL5@+@*YgQxlBTk^(F`Rmrqc~Y(@vmD2n7M5_%{gvE;Kl`O9PU->L?q_*w9I}-c9$)
zl%Yg3K-~*-?P7!MsZ>UgvAO)H+W?p6c^hO^1$#n+yG!`nV)Ub$L^olxBqKO6V>Qw6
zK3$qJ5r09x2xdVJYfgA0NS)0OtlZFtw{1A|twwC@LSn~Gv@KmE{!DNFI}oouT7#(X
zLrxz69PB;p(g5%)A7VnD2&F7Ad8;D%<3jzI1@fbH@(S~YRx$?uM&)M`y1vsidY7i{
zuuOrdPJzLfiYDX!_#j`C3cXRd~Z^;A6YBI5QJ(k6fo2Ac1TduWeE0
zgyxM|th=L`j@Afi{rtEMUDM#ybbRap->BCU()IK`n6uu@2({4@7RGIb7*m0vHqA`)
zv1xO88PgM`xb<~hlVbq>+#-9G3@^R3BXv-u+!oQDL#aS<)rSkj)FxeIWA3f4vX0?`
z6ovtBm?q67R8N~k?V^wPArzE54tF#QgX^1b$(H0nN1`AO0!F?kg!KbP
z^?d1Y|2F~sXHuK+6PE$}X~gp(p0$!wY*y0cNVCvpUE34w2@`v6?U|HKTS=jCVg
zgA;Qs*7WxST?1xfpS;gKe-|7-=VBt&0b
z_!dD)fbOB=ZmC4%-TI8e5Im7`#ji32hDINE-G0$@C*D|ZSamaW`BjT(DV^eXT~qHS
znQQYb0?WZaac58R<&!xI1>T+api4*O0m>c$1xc^D7{7dZ5t+wveNH2pu0*S;E@;;hIqz+}ju9#}6htXUV1AYa#8>=GxH;f0KgppPcqpAZ
zs&c;#nQ4mdM!20z^y5J}I@pLn49cEvo7y)OGv_w_ko@>&pF_J~q%NCB2R~~x`jh?0
z{fM_Wgt!?d9gM7Z?nLpr3|eO0;hc$_CY
z{Z2i-tUf4S@mcu$zKo#DEX*d(^wrGnN*eQ^vCnKojOpdLgMv!7gTi^BI}&L}Oz>`3
zom}9hS5=nv(~qV#oDF#CfikHy#x|H+jXdh(1J1$WIT#{g>iLflfXD;i?5`6C#x|Sb
zhAW%eaxP7W4x^#(^X()jvhybu_r??O{?G2ewFXA?1`NzdbCx{>L2F0>63AR=-Gl}mc#ni@B}kW
z+S)RtD!H4yc|*|i1%O~U%BmOjAeLKvj4SJ!mH*N23gPKk{X`&E*)<7Opzx8epT}g`
zP?ym`7Y^dA5g%l9zD^15YznnYRr-Bs_J#X##(6`fH4;aXFANle^*<4H1W;PYbKO)>IU5k$(Hu&&!dmis8z+4c0-kzcdIzD4%O
z&og@Iygu+;7D9ihrVN)$!f$I26j&G$i6%dRjYJ(=-xh8@)XIGct-s6*9>1TqnL5!G
zIV9HUfRDxcqKe13#6&!%NJ-hcV*Br1VWVi=njnVT2#T=w<&z<;S
zj@9rQ9n1{05j2qhc_I*2?2dmWH@ZLh6mCIjBSeM+2T_rShB*^TeE@O~EC1U`+Y&Q8
z2?O`P?pdJGj0M=FC$Pwi9C$K=P4K@qD-aO=zZ%>B{o1|$f&_t)lPCoLHi`Tt6KSFU
z)@1&N_&bvK`wOB08v7vvJ7-Y{{?X~5|83d|fd4BU`43Pe@ZTks0`dQSy#rv_Qoalz;XD@o%+Cc>xQ|I`!ZAqyjWwr$(#*y-3w$4;Kueq!6UZ9D1MMh6{slCST%cii_ow|?zWHEY%S
zF~{1q*PN@P9K5*-Jb@d6C-FoskzX4c_ul!=85AS|uc4Xn2OOFAgl-Zf2uKDZaISy^
z`2GVK;IgEN@|lb?)gXz6FuYGh5@3qHg$0&e{0*i}L?)nERy<#_K|@+SQD5g|@xA8*
zzWcG}xCFUbZLoOS^=(+(@{RgUe8XQ_)9h!Xd?_$;Hg_d=`-exL;5{+m>kU63?arQn
zpM}ztFOEyW&4_>JtRKN5^s&@)n$i*c0d{K`zzOFq%K+C{42)JAhJ2>9mT0EH5QH>Vsxhb##`hlRD4oFWpmhLKo+7RLWtgEjE^H
z$e~eYVvF{)+DBO7fVjeNQc9r59X&+tC8lz1VlK;`a}G^Ow1HBO$GPmBL6wE)Mvm(Q
zU{jf&^wRSfkdX=7R@0f6YfaS(ov60ST9%8rwCHUV$yt}-hUv7@OIfVGVUXk5g4t+{
z?kKFmLSRhee%vZ=X)ewTb=qF0+8%d6TR`1j&HCL_Jhwe#A6Xo*Z$O8CN*5np3nfZmq8ApX|7XQBeOG6Va$7KCpV@cB%AHmy>J$A
zRZ6f`lzv~bWm5y=$#ABA5XPstDP8fstiW$+aULTeTjlzkzIMEraGE()-ZTf{|gy)$OO~o|P@4QOZI#Js%R^_rkMj6M!mr9b61&A_to=e=Lo*CW{sws|oxtE8xTgqd
zDJwd-n}gh#cQ_&=g@N}M5it)_fU=oy`5w9NG}5Ym{H1v-|4QK|+>>!%kn*pJAaJoa
zguLi$>_^t`wqmOM_n?w0ECyoMTW1f
zsRZDx1m&CZ+0nxgYRFU06b~3JSz8M_sED&i2MtsYq=#IeFMKt|@oH*Nl3~FSkRrJx
z)D)!}qX6@Z;ipzptGn>rA0E1bci(7|g3OJI+HH$^6;Wn(pC9tF9go3xCTm?3;h5gs
zb^J#u(YzAwNp1Y4`N{N|0p_`4lLF$KTWZb1q7)D8tAe7CLsvzB&$q~(jl@G2WE0<;;xCNO`Dmh(#w_iy$YX&LR^4uiXavTv
z#J$^$4}px6EG@_Mmk2m%7KrvmB?8KO(`3wAjTtB5IxX6q{4^U)0Pju63?JxqgKdIK
zG)?mRR=x)HWjBZhHTiN+7%&%Ts}Bjo3&c$EygJ47SOFKvb~5;_KI2>6xR!yZS#bW`
zkn23uJzECj5`$;Fl=w4Z*e1VKBwt)07vAXR7nm&1_VUD=g%VQNOC~RGu*%#-p5fpI
zyKCdQ!C|%GvkQ$5pn=JBh+*x>rMPKo{e|%0=S{D&ws3$s~2BW)$`&VSIx^C`+0D5Uz>o}+$5kY&t2dHi&N=is#*ga#dV^*he4)LTAgi!Oy;n6j|c-R-USHfSamkef8C
zwdG2+{bWqDyi{HKhVrST2U83x2RY1dVxG^~42fFrNvpx6N~0~g@V#Idj|LV9b%*H-
zvI}nM`Yprh&uL8jfjG~NEF6Y>)*#DaUXzdev#+H`Qx|XFr|&xwNth(+Oa#Xw8s&LPf$>e;_Vyr%e<#?4B
zWztMlVsUQ+scjXlh8mVxQ9h1SBas&8O7|Gc%B9_BT_~j?V?*>aBdcf9Tjk*VAmoD{
zK7G`ky5R(HIB&7Z*066_xBlYkl@~78>t1RsfwDd>kst4NBXyaam^`-mN^g%nI`*#wP=rg%`~94uXq}OL
zZ|r0F80KJvK;2o^2wcCZaXKJfU3wf_7APQ>2-ZIvxF=o}AdN=FN@=7!h(N^d%4;5M
zJiU-;xzSnr!*(oL#F8>4M818RFso?8t=p&-%vYLH5)jL2zJ}nYcAnbBIhV#U^s8c{
zGBy1QAAPg2fFQOa$>i6OpV;w@zNAKUKUX0iut}%n8%&H;g<)6MhyQ|n@oT2vDI<5=%Zu-{imXgJi3boL9Wb(BMut0t**PQi5dh^GK^2s@_Xei?LVuzE
z68QG|nfL2fR2wO=lyuyEp}h&(DP!|`po;(wD08u^#dS^4esCIa#jE=hzPZU9Vx4{}
zC*N-&kRMiGn!9751An-K=8Oic9kwmL=n>06LCxwtbpR^9FcT8!u94lW9=3!lwVWQ3
zcjPYyzNC?Ig^|Sl$mRlVRYi@7L9B}2bG*-6yS%46E}MCMy&oXoMx&rGm+%_Xg#
zSgS~5y%4Fe)RQ$B*p4pznn2^f$@Sl~Ze;@!iX|C^wIn
z767-jQ|Yn(fsXQ()%R@W@~go3pys9S+_-?5EV`4b1@riX7V1
z8E<4FOs7QAqpB(~e>K7pI3~ZNyQhiPA>j#1itVfAOgSzcM2fDj}fpg6D(o~BU&Yb0*{)27HOZ%
zwce(vy|iMs_S3EsP#|3JH8|TGoLcduXZcq0RU}-vDXf49#klMklvbX5f;QZ@>)YSnu3^RumbxkGBdQ)%x|RAXYC27tYO&kd4pwa%
zRkb>}V$2s|4$Q38%HBiYrFl-ax%4e;vP35{yYPl^jn{3K2<11xayX6S9rWoA^ERV
z!kqK{S!iP?6o@=R+JekHK~_}jfpXv_vH{ZV#yIJ7b)+Lm2ZACX@5<46H7W6*pZKtm
zXEp<@EN}~Zu1pi)Er#gSG9w5GZu%AAJ210{B7%&&r<}q`y~uD!PP*V)?;yw0;G))z
zBRplx_lSB;(w}2o*F3G#<71g7Ja5V=A+cGgwEd<~g!%d#HLRC8h7A2wS2OcI#g@#A
zGq5J;_Q@P(C8Ey^vGWIJ^6S?Af<{h6Q_tP$hQn;(2
zdr+LknS?bNuUQI;uq>4Q2hI(>$}Srs4});HeCWh_4C!)4Ay(VK<
z>VMl@q`t{u%!xzMnPJXYw67~I%_M92Y$`${oInQO8RA?yCgcW+Uwp1BrnG(Y-mnF@
z)@4F{w-m$7R&6RxVt6p{+~GH@&}>2S_(^~8@c?DMez>;xf&9`pte8DaPxhD(x5OU<
z8C=-CG>)8C{`C0l9dE&HeN$!_IRXe}#!|i-_+p>&^UT{=GKn=1C)N`SFc^b2!!P9=
zG)?v$!c}RZ#2=DwUw?e#hMJe|X+;K9;3(G%KH87?2j1eoq0bHYy?}$mFGf64FV?;;
zE5lriGmTo~MW7mbb;b{voHal4&tiN`46;K!vphNz7Zk215+oXu?s{z$`VeKGSwvnS
z#x!cgG;mU^^Rd|l??m&4f10OhNB4%oDxlf3W)>zQgISAIX~FsiFWVbUMA-
z1!`oLU8EV^^ELY%hp=*3Sg{x3hFIk<*ezWr7UW}?4oy#e^qqY5N5Yru65K7%d3|#k
zR4&FS@AM5tvJFRQFHi{rJee#>iH6|YCEHnk=gA8BDt>S${UQE$2{=mQDB%MG0a=9r
z0g?KK4-~dzlP~Qwl7LTv+uU{
zFt)mWDlV-JF0C7qKKx*g#h4$A&or7g!weGBp!LJ2^W-OR`N
z>fW*PvGsF#o+CN#x2;T#d8~aQoA=H#aRj$&&)|V-2YHIwU_T?ebiD$>kjeONFu;V&CYa8Hej|h3Da_;qp~%#d6&c&_HEF+X&Aw^vUO)8Ky*x
zX}(~Y8d7>9!3}i4(%AqhuFNa&Ba;@7n{9@(vEHKE%?7Uz1Y541hdDe#EdZLqx4dzp
zU2n(BT+b%wWizJ3!4;S%tTwW6fR|cGRj+tiOjmZ>V-(3jZc!>15h-j9`BB{0m7gf-
zKk%YGf;+EhZg3Z0JoZU>Kc$VjJp4UhUTmiyLFzDS&S+LK&4NaMs(MBUdgu20hX6u?
z$Aw07$}MB1^b6?tJw>oyrD3No
z-k(x%ZBsbt8{G7v@YNXg_7#lrF}EA?W9tq4aX-;!=8L~;Ja`*+CpfzGJ@EDry8F+!
z!T6UK9%5mVLmZ;yU#7Tj^@A_40BSEUnwY%%XJg#>Jxc|^`vcq`uxI%f$|^7Q!8(d;
zlyB9=w%R8Fw)iIzA;6HIX`u!7b=DW|%3>}A+o)F8O;?GlsCv=VkE_~)R@V)qqa5rz
zmg#T)Gw!ZkR9S#(HvtI))GTQ%qdWECCY;EUX(vvtG{>lRKXfP@(s|Xb
zRR(V^H+^|P@k?roUwDgC4d*dlyWnR~w&5K#Z4aQeVtTIz@4}YhR=^;j`bh65Ti12sa=Jq&Sl7kF&`SN5
zI;<_OSE8>ISk?M-=JnI1w7x3iyk?&rVU?7tNBqgOo}uopW0eQ)fdZB5lUznxBN(Mr
zZ2v}e_m!KQo(tn_HQW^8%C(;;E}blM^SQlY^g6H6R!)G
z*Y!9bppH4*G_yp}vin97MwdRo3)=)YYpOKh;wd%w0B6`A4d*{LaJG>p*ABH}Bh?8g
zQaWqxHG5^=e@6Sv-;2Kr7Ty<&yZyLV`Nh`tG3O~cAlXenv;C-5hm
zz&Og*ID*m@NXMC}3sEq|DQl9)Qfy{`Uts4!51SRo@`g2InH|nj`O**OUs8ludfIM}
zc9@>tm2f>j7z7Hk$ZyKas?rS8ws6Ddbp?6@_-n!pb6^Cewj)D06TRZk8d4D~xv^i(
z(TDoV`Ufa^$iMCkE5YfTwG}+FGoNO;Y<>#LucqLFzIQk9NU+EMTK|cyHd6kLXYh0Uj)cP@;^^-hci5ACk
zxi>WFc+DI~YG?1h0%mvgGJ;94^oW~A*&FJ5vZWU*xM5QKliU}Tcbp|Wi%Dn()k2?3
z1_s|3?e_hwZ}MMny@TZIyBd?iG2@#;EV21sDXxhmXpI9(tqw8d3QeE@B@4VEVV2*x
zLhr4XFp8}nFa{Sc<}oxSs$l8!?kIL7E>jY$h6O-%1fuBfaJZs??Q+>-HJ${q2e6N*
ziqw}~7gL@3IlXv)CfWeqynqG_I3
zJqOhu!EU`z(}-_()(8pqV9<^(Mu#Xm%7
z-*mmbMwJiWMqG0FMzAr;(FG`>ExSEwqlL2p`+b!5#hw8KvPxLy
zSL3qF<~nCSHJ$lun!i2?xZZTRf4|xh7cRKr5kCdYg(?;Q@HgiJ)547FXV^yf$+EaZ
z)zp*hUkS3Pvwg$QDpPT}_yD|`*f8e@=O?+W3xxB@jlIbXaTGGx7^;
z%9!bfw3T=30>7;J!^Xt!zd-)I?W>Bx5|;n{yL^WFd%*nX@6wXl(%IO|&YT&z>y8D?
zjV1&bC@&ddjD5{<(4tJ;LDGuQeE>Z&%NfXH3L3B-w^XdFfmz=G=^0ofE<)7Rdi_S
zsg{zoGZL)$Bb-np8*Yg560Bn^kL$oZ$4$V(HDX#9U+|L%%(fwW$Ke^$kp9ocs*%bW
z`clI!*4TqgjVdPt@JWoSmx0bdmviTnmWc#+V{j3lBJ?=2{QE18@97ucEl)kHWSmGfQ=AAyfIyAS9Nql}Rjt$zJ{iZHHB!(yL)+7Ss;cy?z|ATs6g
zFER~tX&0~L{$-$+W^B+SIvGZIyS;>RVibpBql4N`ZIue41R`4a?+P5ncFGN7
zOklGR1^j@59i>cnj%gJa=03*xA#xZ?u+)x`k0l@|q7pav*`}Owa|Lg5RTaMED^8pS
zrf5tSuthpmzgrJp|Iun@$StI?zEwYEcv#{J2{e?{nPZpc1jm)I;-R!ceMP4?hPcPp
z9F$ySf0UqmLjE_oTl?!>(t(a~a)4_k^uOn%HiOP<_Xf-Y|2=3E)J_aI&hauhG*$}4
z%w>)NK-;x0c3aoU&1$^#9qgY&WJQTs3ZVdG^K89T-Nj7FgS^?<=?s@rkCU6vKfgbq
z1H7uK6NV8H`5N!7#ftK>@TgczNTJ0?nxQb+=`1)YSjtQcB@;E@dh-?}N^GRUKsrDol8d`?(ac
z!*1^Srx7Gx)j!`q&Ib8~c1ktf^$IN}qHJj2Vk@FTEO9|{N%y}g9&~3OyRQZb
z47MWC^(8xsnLnpe1kZokJ(uU?+6oYq#Vp{fmU&o03^|Vky6Q3sN)~>ziVw4DJRPS;
zC6nW_SGWzvy`m?R@&^zvD2pJ)>h*4K;%5>$zyqOga5Dwc)`N6Nia+n>m}7Q7)we;JC5<-g_u$;0SzaoTA
zy-eOdY1qdNykw=p9AoWiw14T9jX18nLtwpl8VuyhBt_KUL9cIiM9@beg`L4m7_Pm>MFikS-kHI{`2WQnjj%f5+&0AfFZlJj>AgTxV
zdo-gnKFH$>EmFdW`y3}<;-r0+LV0d`o!>`SX6Z|Dsg$EW&}SW8;(bN{)`v)PVzzExKz
zr43~CBNl!!B3_|KK1^Udfs@Vw2cLCELbhNy;8!dRDEZ^np_qH9N?{600P7a?cX1mP
zvp*@>l;`?6{Ne{-2h{FyM;?y`VjDTPX!a48WPo-wkGIR(**sFW7&G+opx?zY9GeKK>i1|~+ipO)X0esq}d
zDIuip_xT=~B3~Nh-Jv+Q-H{B$MfFepSSX?FjrF`~ZiwgVyP#y{xmjMN=
zOGN@KA-5*C7)++oWCm$$Fq+dIP)Cg7p_7B6BS3v2UTde?d1P&moE5!5e5Oc{EDS^c
zDv0qYr5D?TcyT%1;di+aJe{4s6&CgfEkF24Qk$PlWm=jjsuSiytG<{XUSdp&h-hwI
zxyVrJsFiE5==gmIJdEc#3!$S$@H0^l$s-G}YH_yd;F4k7ZyVlvn$%>{<=#HYqIV^@
zmPKpNN?~It&SJJ^r&WF1<{t8uddnl`et)s`^PrdUkR}7h<2pce_ZuR=IgRge=U^SL
z01U*nu5IpabEeP_p214~R*eT;#v3lPXznfKkncRP)SwxqFyGfjH}~__yrSP?u;f1g
zA*Etyc3*aNHxcisjNq6MAJBENL+k}lri6n}WX
z`x|sOAJ32Z&`rxn#2Mr8IpZXrB*rlym>5=Ac~+a^(j%a6m*m}UDv7`FAu6!e
zN5x&HFaAm{aHhRL82#(v;rA&dVw=te2(x=nN%A@rWf7huiQR+`@bhn(<6OFQlsrF9
zdUY{wK`mT+msXt29fk>fv?F}1;wUDM${H#uWvmuY#?EKqmsn*L^3>=&;5aq;
zMuhNsz_{nFLz2RcA&Lo`e8$o?ZJZ|gI#5eeVS8JRP`y&8QKSKr3X@0Jo
ze_v$O5}{pCGOj8DLra=OMWu+xI*@Q?PXlB#>ymeG8wNls?Fg(#W6N-`=6)2#{Ak5Y
zDdZ$_--&Edo@CB*!eyxgYC@-wF5@YxyUs?9|%BJJ;d&p%>@2vn!^J
zR>`BIW>0O$4%FA<9P!#NBr`e)1td%df=Wy==j@)rFc>w!ZemA_PbOL(JgSk%KO;35WL|kX?*HV
z{hl_L+}q`<+(Aj0EP*C2TWvBkj2>3z$EztpQsqFhEfRX6{)9B}Ap~RKL=_2JjH|ll
z#cucKbox(f^%6taT~_A^fMxTG%8H5LoNKV?tYqGC<}rGJO^`GC}QZKoV*ivI=^AL|ZJQ><-c%>vXnX_~aVAOD^SaROT6&euq_hha_d^bEjpy
zJ)S#|$uf(2dVSTQIfuf%k!RFxk`{ptD>&avC!h#``=x^Lz-smi;5m7Lxc5b)6bjYI
zjfAOji*kpxwntM#ogmyIjYaR!Fmp^hGfx~6T#Q*pLY;Gu{4uW-b2Kk@2R|+O#>BGz
zIdvY+H=??cdrtL}Vxp_Yl)H^-8&sb)_7K#%#wfYjI!Oj`
z(H`@g#0mdDjKun1S(_2lDhT>-HSPK<7qIMypGSd_6
zVJ(n-x6$zDP={_f`*p&GQri->R6y_^dAN@$>S~E5Fbc>wN}7@!D?%dtPEQ(o-F1kl}TLGyv#a
zPkM^*K3-QvRt1vldVo^Fo15K$xVO_A60&Y>|ND@dk(q6pRJ&jwoMyEZI(wEPgsw2XYG5bgq;u2LwtHDyHAT|;N*Bit}>W*Ef^KMC@U0QgM!FLwz<`r
z1?_LblQlh-EY@i$w50yw4ZNE`97jTAlf7^n@$YfcYgd^KGxE3g7W`_{G=U6O<1?}c
z@~NZ6H5o_u7}OE=orxE7ZW~
z+z{BWz0tM7SbXlzCY-z)Fu2f?W^79|5Rg&2zcx@N0KBLAN{b==0}l`gPLT)_EI@`l
zVVVRY7VwKrLOmIR1tu)qQ${H%a6%zH14N~~aGCD6N>!nepH@YUrlf3yN_EYxiG7vd
zwvgTJ_3s_MO`jh=o8Gp1=Y3BZ8Pf_#R8NP8SAsruJ*yj+cegDbJAYm}&?Pvcl}{Ft
z-T<$E0Fpw)6Q^}}AB<_AO2n5(1!{cPMvE1ZI)?-~uGR*8xL2z`hYT}TIi9Suv&Q&N
z&(-_GlwJkI4iot>@5%*cPlE)Qj@Xabbq~pMkbdD1$FEom#4oMaEm08N9MMtnQVunL^k<3x!Q1H9`)+3HR7wg<-R5nD81agYlS=YMiBXjhs6piY=UY
zs3Ci|#5g5(n^AI=*fv(hT6WXbD935IF+kAWo)Udl1nsV2POk(1QoH=n=1`)6v5#{L
zZmiD2Xgh8j2a@Q)NLj|E5uGCr58O>=hbb~ZHoDx2s`M$)QEUW{tFD|>SmyPuOx`e3<`
z7Bm$d
zXe(25v12my#IYD|3@{L&59QPKDN8QXx7-r3C*b&%`EZV6=0pv4990B9DL@%%d
zNOmAOQy$!U69L0SCuzkhH33wOL;I2J$%C)})*0MIyd@}e@)#W(7n`2Xv>kj2>3#Ha>epEqHVyvgvjyZw%_QbT;vbF{O#qZ>VTEni3gY^@
zbyz3KvT5j{%4^S-Lo{2WS0-MnMq+I{M1E1&xJb4*;RVmQnUlXW4hK1Va+q^2N_F7s
zit&v_W~RO9*sq+_mV%`OM%Rt7CDy}s@0C$x5#=?%C$v(i){aiip;%FcZ|yfyRdL+!i-kT;&-81dQ2)1&BuTgnZ3yXr$3#VbO~#6{bZxDRp4%hx?$4`z;`b_bym
zdQ>y*H&pnzKA?k-&>|zo{^W)XnAr1q6#@j`Vv7s(d*J*b5jOiZXPKMq`>ED~Pzt^HN6%?Fdf!Gv)->wwt)<=~$
zhUg#Em%C*Nwmh_1ldA4Gi%<1%C7(T@e6NP9R}m{U0stmB+GM!iMSlc_)nNll@81O@Lx;bw+-;e2UV%^v~eSr#ofAxaNsO^3P1EbZ{
zXMjf#BDLlj#f|0kS2^noB%`d60^exOAjRl1Ampi53+1t4_PHEkFCDYMfHobmfPtA`
zG}sCvBe82ykHTV(%;>ZJ)kcADb2076gT~CIrwfQ8x3*QRMx>U`w9nMuiW!aefIPe&
z`P(~G$8&|>uDqLq>Km-H`IBij{96G(e4>E|4ap6gTYAum9Ve_X=h^~iGZHE*I%{*k
z!6HN(;f}UZy~$kam>R13+Bfz)_Y0zmnb<^Wi4T{Xp$0zKZV$d%PZVR;lpLZz5Vcz^
zECxWSk2Fh|WHo*o3@P5ovn@a9c(DDN6KMTZ)+db!9i{vr#p>bd$agkj(*BzL;W&Wt
z!jV?8zvp*SB>j4RPN>TK(S05_QWGbN4EyBN?U-LZ$5*bQBkVoT2ng;xu`Ixv6s*HV
zze<9$I^655cXiAVCIj#67s33q$5C7S=n5z)`r-(!I^$S<9gWYp9-;{e>y<oF6WDebIVaI$;_npTRqkYR^6t8;Q>34o`=K`UouvA(yALi~Wp~bUiN00R3
z#HN`_q_|F(=nl2JYes=Kmgu4Zi8kxF%~!V~NA|+Z^}*Jb_S&j8Sz)xCCQs@CIR)Ts
zskMH<82)Sedh;&F!b~h3N2h0PnqbsLWVI)4<_maztfxXy7WvzA1Tl75Me
zdd*k&r&lC9y?et}W>0D7BbWh!S6-~GOT
zy>ukR`y)~(Zqu`AWeF>+Mv6H(&ePEPjc;e@d(h{Frs5Uh!9g(uZNx@W7G(MM^AFY9
zE*$B;*O$rEF|SBmxTPuGn>7yRxryMw-=*4PjE_ewK&aq*Jg?Y#X~XM!|(k;$|bfI-}np3^=qpx*et>|
zb?@@-JVG&QCI9O90V;%aqf&cSm=3LMc4o|VZP<*id-{1t+;p*7e`a9P7fnWjRH`W5
zkkKK|218b395vb=lx4ib`=S8zI?chE5##cQC7x2u(hj%7kCH2WgkMgHS|7ocjLAMi
zDxW+Z^asy0^9LJow`sp@SvwYntaOCEGUJlk(wYtL1#57&qU+*Pt!QJ5UCxCb5wfyx
z%Ag}Ks-V%Uxq#9ON(`4J`t?g$-n6$k9rK)-liNKU2RL^ihhj2Rx?zA@u==VZSl3)M
zmQjl&6q259eR!iHQqH^b{$@rylBD~+~Q8ilUZFI334pACRf@1L|pD=HY
z*L8beLytHeAr+S-k8|Mo9>T}mztu$5el{Il6p57@@)e~?tv#!$sObdwSSyHo5yy_K
zj9q&?FYuI#`>J~Tm;M0Ezu+l{?k<
z&i=8YjQYkx7wisaX5lhg@;lvI1zd~;*6c;WPceb6@YzIno7lPA%L3l;^7&hN2kgpu
zP~6_kR?7JCcZ=|SBAemy-;7yDOcKH
zONQTz{QAItc_zn|^0uVv1>|~P+L~E7Zg##M0*#Msdq3N(mJ<&*^vdFJ9&fkNGI5_m5Q}hon1asoD+!8
z3Ws8L%-d9D{uXs`DAbaxTUyFr$`9^Y4bS-iUG+xmvwh|Dh7v?hdEl=fW|jR-;K7Ex
z-~;itzuFJzlXM`IPE|Vou0kvbo%Kx3Dy1TrmsrC!+!OE#>&+U-=fX;cTbI&7T51OZ
zW4G6dFGnB;&cq0*51pfPgka*tti5&GZ$g~(L74^FRnQUEpcsVT)12SiU2q`LvZ--R
zdymmUEGCELWwGW7KCx9fG2o7UGXM5LG`xjAVhJSDbv{~t
zHwS0sg$bf?Wi`mTa$rg4HoUV5K*hecuZJ}o$>7v{}|pEH#^Y0=m&gAAcP@zC~!
zh<0;j!H3Sk)vL%v#Prcj9GS4_L{6{TF?D=yL8Q4Y%8N8Jk*Hq#2b8|}Dd8+h|2A?D
zp+Z(r#u4zlQ0c3y-{+|b!YHrTUGJ6iih2eZFT$-E^M*`_$BY+i~I2vS}dv$k)6-Y1o3BJ3_^Z!yc!E{I>PI+lCM2ppW+V*aKE1hZQ1
zj1_D~#n(lVxv@lVRfU+})c5S@p;dc0`KEj)L`?(7>pGVRITmwm`G;<++XZ42f
z=9KBn=@tJ$F0fl>4bnx;`pzHn^t=p+TABY%Gq^3!X(pZw-G~V3kEg6^?SwGD4%Pl8
z)DzvnxmP|)Q4X{Y+(AAIhk4ca&+wNJmO^`Z)p>*Ea9ptKh2Geaa=5yJe5;USm(Aq0
zvKIuTEq;@a5kUjpfkz-;5L*g@;|U%w4BMPcf%E4Z->vdLp24+vD}4oH7vEta-xaeh>asgzU2Enw67+a
z7IUz_22`A9e^Go9e=GouS&_xru6~y-*Ocqdpu`FXyFY%z(7q88b68_n*Tmshy0fHt
zY_24QiVCqg8W(u{tfugbsK$yRkThtoi!{M5+>Yhf~;7
z9{kkd)`@lPhLacKF1#^U*hBOnX(b%~1l7#l>2e0LF&F$v)yxeS>_9mGNciJM7^Eqs
zi4YkM;;%CW8b+?_mp!n1Q2C!x<|gE!NtpkCeED$73@pP9C>lZzw425z_|KRe2nhdQ
z^U8m&Q1^d8zuobGThl0z|2^w-H;oB~@BPmM`WZ5a|C+4;s?1=4A^ZJv58#LYUtog&
zYqih3#nDPGr6B40-l5%H$RQP65YS^d&>C#
zUij}+{?&p0SM>Spf54&(qy+!fq}=}-(c!Pb!qI?YTt%9WxUh%^cGCxp?-c+hETI5-m+-&_
k2Y?z&=3ps9!0shhh<^xow}b_DG6aNOrh^h5{;!YxKjAK;s{jB1

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index ea720f986..16d28051c 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip

From 61079bb39a91ac4942bf5f5320422f19ac2dabd3 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Thu, 3 May 2018 15:16:52 -0600
Subject: [PATCH 161/229] Updating get job transfer methods to add version id
 when getting object

---
 .../ds3client/integration/Util.java           | 22 ++++++--
 .../integration/VersionedObject_Test.java     | 50 +++++++++++++++++++
 .../GetJobPartialBlobTransferMethod.java      |  1 +
 .../GetJobTransferMethod.java                 |  2 +
 4 files changed, 71 insertions(+), 4 deletions(-)

diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index 8f6f54212..70d8173a1 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -18,13 +18,11 @@
 import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.Ds3ClientBuilder;
-import com.spectralogic.ds3client.commands.spectrads3.CancelJobSpectraS3Request;
-import com.spectralogic.ds3client.commands.spectrads3.GetJobsSpectraS3Request;
-import com.spectralogic.ds3client.commands.spectrads3.GetJobsSpectraS3Response;
-import com.spectralogic.ds3client.commands.spectrads3.GetSystemInformationSpectraS3Request;
+import com.spectralogic.ds3client.commands.spectrads3.*;
 import com.spectralogic.ds3client.helpers.DeleteBucket;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.channelbuilders.PrefixAdderObjectChannelBuilder;
+import com.spectralogic.ds3client.models.Bucket;
 import com.spectralogic.ds3client.models.Job;
 import com.spectralogic.ds3client.models.JobStatus;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
@@ -65,6 +63,22 @@ public static Ds3Client insecureFromEnv() {
         return builder.build();
     }
 
+    /**
+     * Goes through all buckets on a BP and attempts to delete them and all their contents. It cancels all active jobs
+     * associated with this bucket. Created for cleaning up after cascade failures in functional tests.
+     *
+     * USE ONLY WHEN DELETING ALL BUCKETS AND ALL CONTENT IS YOUR GOAL.
+     */
+    public static void forceCleanupBucketsOnBP(final Ds3Client client) throws IOException {
+        final GetBucketsSpectraS3Request request = new GetBucketsSpectraS3Request();
+        final GetBucketsSpectraS3Response response = client.getBucketsSpectraS3(request);
+
+        for (Bucket bucket : response.getBucketListResult().getBuckets()) {
+            cancelAllJobsForBucket(client, bucket.getName());
+            client.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(bucket.getName()).withForce(true));
+        }
+    }
+
     public static void assumeVersion1_2(final Ds3Client client) throws IOException {
         final int majorVersion = Integer.parseInt(client.getSystemInformationSpectraS3(
                 new GetSystemInformationSpectraS3Request()).getSystemInformationResult().getBuildInformation().getVersion().split("\\.")[0]);
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
index e15c4ac85..b3abbdc33 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/VersionedObject_Test.java
@@ -22,19 +22,26 @@
 import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
 import com.spectralogic.ds3client.commands.spectrads3.StageObjectsJobSpectraS3Request;
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.helpers.FileObjectGetter;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.models.VersioningLevel;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.utils.ResourceUtils;
+import org.apache.commons.io.FileUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -42,6 +49,7 @@
 import static com.spectralogic.ds3client.integration.Util.*;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 public class VersionedObject_Test {
 
@@ -138,4 +146,46 @@ public void stageObjectsWithVersioning() throws IOException, URISyntaxException
             CLIENT.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(TEST_ENV_NAME).withForce(true));
         }
     }
+
+    @Test
+    public void getObjectsWithVersioningUsingHelpers() throws IOException, URISyntaxException, InterruptedException {
+        // Create temp directory for files we will retrieve from bp
+        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), null);
+
+        try {
+            HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
+            final String objectName = "object_with_versions";
+
+            // Put different content for object twice
+            loadTestBook(CLIENT, BOOKS[0], objectName, TEST_ENV_NAME); // putting beowulf as content
+            loadTestBook(CLIENT, BOOKS[1], objectName, TEST_ENV_NAME); // putting sherlock holmes as content
+
+            // Get the version of the objects
+            final GetBucketRequest getBucketRequest = new GetBucketRequest(TEST_ENV_NAME).withVersions(true);
+            final GetBucketResponse getBucketResponse = CLIENT.getBucket(getBucketRequest);
+
+            assertThat(getBucketResponse.getListBucketResult().getObjects().size(), is(0));
+            assertThat(getBucketResponse.getListBucketResult().getVersionedObjects().size(), is(2));
+
+            // Create bulk get job with the oldest version of the object
+            final List objects = getBucketResponse.getListBucketResult().getVersionedObjects().stream()
+                    .filter(obj -> !obj.getIsLatest()) // filters out the latest version of the object
+                    .map(obj -> new Ds3Object(obj.getKey(), obj.getVersionId()))
+                    .collect(Collectors.toList());
+
+            final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(TEST_ENV_NAME, objects);
+
+            readJob.transfer(new FileObjectGetter(tempDirectory));
+
+            // Check content is beowulf, which was the first version of content
+            final File originalFile = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + BOOKS[0]).toFile();
+            final File fileCopiedFromBP = Paths.get(tempDirectory.toString(), objectName).toFile();
+            assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP));
+
+        } finally {
+            cancelAllJobsForBucket(CLIENT, TEST_ENV_NAME);
+            deleteAllContents(CLIENT, TEST_ENV_NAME);
+            FileUtils.deleteDirectory(tempDirectory.toFile());
+        }
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
index dc8bc3b56..e4269d24f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
@@ -92,6 +92,7 @@ private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekable
                 jobId,
                 blob.getOffset());
 
+        getObjectRequest.withVersionId(blob.getVersionId());
         getObjectRequest.withByteRanges(blobRanges);
 
         return getObjectRequest;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
index 3a174145d..bf500a106 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
@@ -86,6 +86,8 @@ private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekable
                 jobId,
                 blob.getOffset());
 
+        getObjectRequest.withVersionId(jobPart.getBlob().getVersionId());
+
         final ImmutableCollection rangesForBlob = StrategyUtils.getRangesForBlob(rangesForBlobs, blob);
 
         if (rangesForBlob != null) {

From 0edafb60ec1297d41008708275a1ad9751a7e371 Mon Sep 17 00:00:00 2001
From: Rachel Tucker 
Date: Fri, 4 May 2018 14:03:28 -0600
Subject: [PATCH 162/229] Checking if version id is null

---
 .../java/com/spectralogic/ds3client/integration/Util.java     | 2 +-
 .../transferstrategy/GetJobPartialBlobTransferMethod.java     | 4 +++-
 .../strategy/transferstrategy/GetJobTransferMethod.java       | 4 +++-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index 70d8173a1..540a06c86 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -73,7 +73,7 @@ public static void forceCleanupBucketsOnBP(final Ds3Client client) throws IOExce
         final GetBucketsSpectraS3Request request = new GetBucketsSpectraS3Request();
         final GetBucketsSpectraS3Response response = client.getBucketsSpectraS3(request);
 
-        for (Bucket bucket : response.getBucketListResult().getBuckets()) {
+        for (final Bucket bucket : response.getBucketListResult().getBuckets()) {
             cancelAllJobsForBucket(client, bucket.getName());
             client.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(bucket.getName()).withForce(true));
         }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
index e4269d24f..3fa3c9eab 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobPartialBlobTransferMethod.java
@@ -92,7 +92,9 @@ private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekable
                 jobId,
                 blob.getOffset());
 
-        getObjectRequest.withVersionId(blob.getVersionId());
+        if (blob.getVersionId() != null) {
+            getObjectRequest.withVersionId(blob.getVersionId());
+        }
         getObjectRequest.withByteRanges(blobRanges);
 
         return getObjectRequest;
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
index bf500a106..3a7c8c1d5 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/GetJobTransferMethod.java
@@ -86,7 +86,9 @@ private GetObjectRequest makeGetObjectRequest(final SeekableByteChannel seekable
                 jobId,
                 blob.getOffset());
 
-        getObjectRequest.withVersionId(jobPart.getBlob().getVersionId());
+        if (jobPart.getBlob().getVersionId() != null) {
+            getObjectRequest.withVersionId(jobPart.getBlob().getVersionId());
+        }
 
         final ImmutableCollection rangesForBlob = StrategyUtils.getRangesForBlob(rangesForBlobs, blob);
 

From 3cc8950d58a14ea9e83b1ec28f7a9b8bbdfd95b3 Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Tue, 8 May 2018 13:21:31 -0600
Subject: [PATCH 163/229] Updating version to 5.0.3

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 6c7b9012c..218f06963 100644
--- a/build.gradle
+++ b/build.gradle
@@ -36,7 +36,7 @@ plugins {
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '5.0.2'
+    version = '5.0.3'
 }
 
 subprojects {

From 36384175d7fadaabd92a1ac894a71b793603002c Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Tue, 8 May 2018 13:44:36 -0600
Subject: [PATCH 164/229] README Update with latest version badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9b2d09b9d..81c3d8c28 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@ ds3_java_sdk
   Coverity Scan Build Status
 
-[![Download](https://api.bintray.com/packages/spectralogic/ds3/ds3_java_sdk/images/download.svg) ](https://bintray.com/spectralogic/ds3/ds3_java_sdk/_latestVersion) 
+[ ![Download](https://api.bintray.com/packages/spectralogic/ds3/ds3-sdk/images/download.svg) ](https://bintray.com/spectralogic/ds3/ds3-sdk/_latestVersion)
 [![Apache V2 License](http://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/SpectraLogic/ds3_java_sdk/blob/master/LICENSE.md) [![Open Hub project report for ds3_java_sdk](https://www.openhub.net/p/ds3_java_sdk/widgets/project_thin_badge.gif)](https://www.openhub.net/p/ds3_java_sdk?ref=sample)
 
 ## Contact Us

From b93ff4f059ef4daaa0a02bec8a25acb3d13353d4 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Mon, 12 Feb 2018 19:16:24 -0700
Subject: [PATCH 165/229] Implementing job cancel

---
 .../integration/PutJobManagement_Test.java    | 163 ++++++++++++++----
 .../ds3client/helpers/Ds3ClientHelpers.java   |  20 +++
 .../ds3client/helpers/JobImpl.java            |  22 ++-
 .../ds3client/helpers/ReadJobImpl.java        |   5 +
 .../ds3client/helpers/WriteJobImpl.java       |   5 +
 .../AbstractTransferStrategy.java             |  89 +++++++---
 .../CanceledEventObserver.java                |  25 +++
 .../transferstrategy/EventDispatcher.java     |   6 +
 .../transferstrategy/EventDispatcherImpl.java |  26 ++-
 .../MultiThreadedTransferStrategy.java        |   3 +
 .../SingleThreadedTransferStrategy.java       |   4 +
 .../transferstrategy/TransferStrategy.java    |   7 +
 .../TransferStrategyBuilder.java              |   4 +
 .../TransferStrategy_Test.java                | 154 +++++++++++++++--
 14 files changed, 462 insertions(+), 71 deletions(-)
 create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/CanceledEventObserver.java

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
index e2d283cc7..ba99e0f30 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
@@ -17,6 +17,7 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.Ds3ClientBuilder;
@@ -29,6 +30,7 @@
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
 import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
+import com.spectralogic.ds3client.helpers.pagination.GetBucketKeyLoaderFactory;
 import com.spectralogic.ds3client.helpers.strategy.blobstrategy.*;
 import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy;
 import com.spectralogic.ds3client.helpers.strategy.channelstrategy.SequentialFileReaderChannelStrategy;
@@ -43,6 +45,7 @@
 import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
 import com.spectralogic.ds3client.utils.Platform;
 import com.spectralogic.ds3client.utils.ResourceUtils;
+import com.spectralogic.ds3client.utils.collections.LazyIterable;
 import com.spectralogic.ds3client.utils.hashing.ChecksumUtils;
 import com.spectralogic.ds3client.utils.hashing.Hasher;
 import org.apache.commons.io.FileUtils;
@@ -1940,37 +1943,18 @@ public void onFailure(final FailureEvent failureEvent) {
         }
     }
 
+    private static final String SOURCE_DIRECTORY = "little_files/";
+    private static final String SOURCE_FILE_BASE_NAME = "tape";
+    private static final String SOURCE_FILE_EXTENSION = "png";
+    private static final String SOURCE_FILE_NAME = SOURCE_FILE_BASE_NAME + "." + SOURCE_FILE_EXTENSION;
+
     @Test
     public void testPutting15000Files() throws IOException, URISyntaxException {
         final String tempPathPrefix = null;
         Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
 
         try {
-            final String sourceDirectory = "little_files/";
-            final String sourceFileName = "tape.png";
-
-            final Path sourceFilePath = ResourceUtils.loadFileResource(sourceDirectory + sourceFileName);
-
-            Files.copy(sourceFilePath, Paths.get(tempDirectory.toString(), sourceFileName));
-
-            final List fileNames = new ArrayList<>(15001);
-            fileNames.add(sourceFileName);
-
-            for (int i = 1; i <= 15000; ++i) {
-                final String destinationFileName = "tape" + i + ".png";
-                fileNames.add(destinationFileName);
-                Files.copy(sourceFilePath, Paths.get(tempDirectory.toString(), destinationFileName));
-            }
-
-            final List ds3Objects = new ArrayList<>();
-
-            for (final String fileName : fileNames) {
-                final Path filePath = Paths.get(tempDirectory.toString(), fileName);
-                final long fileSize = Files.size(filePath);
-                final Ds3Object ds3Object = new Ds3Object(fileName, fileSize);
-
-                ds3Objects.add(ds3Object);
-            }
+            final ImmutableList ds3Objects = copyFilesAndGenerateDsObjects(tempDirectory);
 
             final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client);
             final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, ds3Objects);
@@ -1982,7 +1966,7 @@ public void testPutting15000Files() throws IOException, URISyntaxException {
             tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
 
             final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Lists.newArrayList(
-                    new Ds3Object(sourceFileName, Files.size(sourceFilePath))));
+                    new Ds3Object(SOURCE_FILE_NAME, Files.size(sourceFilePath()))));
 
             final GetJobSpectraS3Response jobSpectraS3Response = client.getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));
 
@@ -1990,11 +1974,11 @@ public void testPutting15000Files() throws IOException, URISyntaxException {
 
             readJob.transfer(new FileObjectGetter(tempDirectory));
 
-            final File originalFile = ResourceUtils.loadFileResource(sourceDirectory + sourceFileName).toFile();
-            final File fileCopiedFromBP = Paths.get(tempDirectory.toString(), sourceFileName).toFile();
+            final File originalFile = ResourceUtils.loadFileResource(SOURCE_DIRECTORY + SOURCE_FILE_NAME).toFile();
+            final File fileCopiedFromBP = Paths.get(tempDirectory.toString(), SOURCE_FILE_NAME).toFile();
             assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP));
         } catch (final org.apache.http.client.ClientProtocolException e) {
-            fail("This test makes sure that we don't run out of connections when transferring lots of small files.  Oops");
+            fail("This test makes sure that we don't run out of connections when transferring lots of small files.  Oops.");
         } finally {
             FileUtils.deleteDirectory(tempDirectory.toFile());
             cancelAllJobsForBucket(client, BUCKET_NAME);
@@ -2002,6 +1986,127 @@ public void testPutting15000Files() throws IOException, URISyntaxException {
         }
     }
 
+    private Path sourceFilePath() throws IOException, URISyntaxException {
+        return ResourceUtils.loadFileResource(SOURCE_DIRECTORY + SOURCE_FILE_NAME);
+    }
+
+    private ImmutableList copyFilesAndGenerateDsObjects(final Path destinationDirectory) throws IOException, URISyntaxException {
+        final Path sourceFilePath = ResourceUtils.loadFileResource(SOURCE_DIRECTORY + SOURCE_FILE_NAME);
+
+        final ImmutableList fileNames = generateFileNames(SOURCE_FILE_BASE_NAME, SOURCE_FILE_EXTENSION, 15000);
+        copyFiles(sourceFilePath, destinationDirectory, fileNames);
+
+        return generateDs3Objects(destinationDirectory, fileNames);
+    }
+
+    private ImmutableList generateFileNames(final String baseFileName, final String baseFileExtension, final int numFileNames) {
+        final ImmutableList.Builder listBuilder = ImmutableList.builder();
+
+        listBuilder.add(baseFileName + "." + baseFileExtension);
+
+        for (int i = 1; i < numFileNames; ++i) {
+            listBuilder.add(baseFileName + i + "." + baseFileExtension);
+        }
+
+        return listBuilder.build();
+    }
+
+    private void copyFiles(final Path sourceFilePath, final Path destinationDirectory, final ImmutableList fileNames) throws IOException {
+        for (final String fileName : fileNames) {
+            Files.copy(sourceFilePath, Paths.get(destinationDirectory.toString(), fileName));
+        }
+    }
+
+    private ImmutableList generateDs3Objects(final Path destinationDirectory, final ImmutableList fileNames) throws IOException {
+        final ImmutableList.Builder listBuilder = ImmutableList.builder();
+
+        for (final String fileName : fileNames) {
+            final Path filePath = Paths.get(destinationDirectory.toString(), fileName);
+            final long fileSize = Files.size(filePath);
+            final Ds3Object ds3Object = new Ds3Object(fileName, fileSize);
+
+            listBuilder.add(ds3Object);
+        }
+
+        return listBuilder.build();
+    }
+
+    @Test
+    public void testCancelingJob() throws URISyntaxException, InterruptedException {
+        final String tempPathPrefix = null;
+        Path tempDirectory;
+
+        try {
+            tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
+        } catch (final IOException e) {
+            fail("Could not create temp folder.");
+            return;
+        }
+
+        try {
+            final ImmutableList ds3Objects = copyFilesAndGenerateDsObjects(tempDirectory);
+
+            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client);
+
+            final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, ds3Objects);
+
+            final AtomicBoolean caughtExceptionWhileCanceling = new AtomicBoolean(false);
+            final AtomicBoolean cancelHandlerCalled = new AtomicBoolean(false);
+            final CountDownLatch countDownLatch = new CountDownLatch(1);
+
+            writeJob.attachObjectCompletedListener(name -> {
+                try {
+                    writeJob.cancel();
+                } catch (final Throwable t) {
+                    if ( ! (t instanceof FailedRequestException)) {
+                        caughtExceptionWhileCanceling.set(true);
+                    }
+                } finally {
+                    countDownLatch.countDown();
+                }
+            });
+
+            writeJob.attachCanceledEventObserver(new CanceledEventObserver(eventData -> {
+                cancelHandlerCalled.set(true);
+                assertEquals(writeJob.getJobId(), eventData);
+            }));
+
+            writeJob.transfer(new FileObjectPutter(tempDirectory));
+
+            countDownLatch.await();
+
+            assertFalse(caughtExceptionWhileCanceling.get());
+            assertTrue(cancelHandlerCalled.get());
+
+            final String prefix = "";
+            final String nextMarker = null;
+            final int maxKeys = 15000;
+            final String delimiter = null;
+            final int numRetries = 5;
+
+            final GetBucketKeyLoaderFactory getBucketKeyLoaderFactory = new GetBucketKeyLoaderFactory<>(client, BUCKET_NAME, prefix, delimiter, nextMarker, maxKeys, numRetries, GetBucketKeyLoaderFactory.contentsFunction);
+            final LazyIterable iterable = new LazyIterable<>(getBucketKeyLoaderFactory);
+            final int numThingsInBucket = Iterables.size(iterable);
+
+            assertTrue(numThingsInBucket < maxKeys);
+
+            final GetCanceledJobSpectraS3Response getCanceledJobSpectraS3Response = client.getCanceledJobSpectraS3(new GetCanceledJobSpectraS3Request(writeJob.getJobId().toString()));
+            final CanceledJob canceledJob = getCanceledJobSpectraS3Response.getCanceledJobResult();
+            assertEquals(writeJob.getJobId(), canceledJob.getId());
+        } catch (final org.apache.http.client.ClientProtocolException e) {
+            fail("This test makes sure that we don't run out of connections when transferring lots of small files.  Oops.");
+        } catch (final IOException e) {
+            fail("IOException from something other than job cancelation.");
+        } finally {
+            try {
+                FileUtils.deleteDirectory(tempDirectory.toFile());
+                deleteAllContents(client, BUCKET_NAME);
+            } catch (final IOException e) {
+                LOG.error("Failure cleaning up.", e);
+            }
+        }
+    }
+
     @Test
     public void testThatFifoIsNotProcessed() throws IOException, InterruptedException {
         Assume.assumeFalse(Platform.isWindows());
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
index 4ca191438..136d627e3 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
@@ -21,6 +21,7 @@
 import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.helpers.pagination.FileSystemKey;
+import com.spectralogic.ds3client.helpers.strategy.transferstrategy.CanceledEventObserver;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.models.Contents;
 import com.spectralogic.ds3client.models.bulk.Ds3Object;
@@ -127,6 +128,25 @@ void transfer(final ObjectChannelBuilder channelBuilder)
          * @throws IOException
          */
         void transfer() throws IOException;
+
+        /**
+         * Cancels a transfer in progress in the current job.
+         * @throws IOException
+         */
+        void cancel() throws IOException;
+
+        /**
+         * Attaches a handler that fires when a job transfer in progress is canceled.
+         * @param canceledEventObserver An event notification object that carries the job UUID.
+         * @return An observer instance that can be later used for de-registration.
+         */
+        CanceledEventObserver attachCanceledEventObserver(final CanceledEventObserver canceledEventObserver);
+
+        /**
+         * Remove an event handler previously registered with a call to {@link #attachCanceledEventObserver(CanceledEventObserver)}
+         * @param canceledEventObserver An instance of {@link CanceledEventObserver} returned from a call to {@link #attachCanceledEventObserver(CanceledEventObserver)}
+         */
+        void removeCanceledEventObserver(final CanceledEventObserver canceledEventObserver);
     }
 
     /**
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java
index eb94742db..04829877b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java
@@ -16,7 +16,9 @@
 package com.spectralogic.ds3client.helpers;
 
 import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.Job;
+import com.spectralogic.ds3client.helpers.strategy.transferstrategy.CanceledEventObserver;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher;
+import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
 import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;
 
 import org.slf4j.Logger;
@@ -130,16 +132,34 @@ public void removeObjectCompletedListener(final ObjectCompletedListener listener
         eventDispatcher().removeObjectCompletedListener(listener);
     }
 
+    @Override
+    public CanceledEventObserver attachCanceledEventObserver(final CanceledEventObserver canceledEventObserver) {
+        checkRunning();
+        return eventDispatcher().attachCanceledEventObserver(canceledEventObserver);
+    }
+
+    @Override
+    public void removeCanceledEventObserver(final CanceledEventObserver canceledEventObserver) {
+        checkRunning();
+        eventDispatcher().removeCanceledEventObserver(canceledEventObserver);
+    }
+
     @Override
     public void transfer(final Ds3ClientHelpers.ObjectChannelBuilder channelBuilder) throws IOException {
         transferStrategyBuilder.withChannelBuilder(channelBuilder);
     }
 
-    protected TransferStrategyBuilder transferStrategyBuilder() {
+    TransferStrategyBuilder transferStrategyBuilder() {
         return transferStrategyBuilder;
     }
 
     protected EventDispatcher eventDispatcher() {
         return transferStrategyBuilder.eventDispatcher();
     }
+
+    void cancel(final TransferStrategy transferStrategy) throws IOException {
+        if (transferStrategy != null) {
+            transferStrategy.cancel();
+        }
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java
index 394e43369..7a93dfc19 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java
@@ -77,4 +77,9 @@ public void transfer() throws IOException {
         running = true;
         transferStrategy.transfer();
     }
+
+    @Override
+    public void cancel() throws IOException {
+        cancel(transferStrategy);
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java
index e1b394162..c19846669 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java
@@ -81,5 +81,10 @@ public void transfer() throws IOException {
         running = true;
         transferStrategy.transfer();
     }
+
+    @Override
+    public void cancel() throws IOException {
+        cancel(transferStrategy);
+    }
 }
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index 097a2200a..4720991e2 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -18,6 +18,7 @@
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Iterables;
 import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.spectrads3.CancelJobSpectraS3Request;
 import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
 import com.spectralogic.ds3client.commands.spectrads3.PutBulkJobSpectraS3Request;
 import com.spectralogic.ds3client.exceptions.Ds3NoMoreRetriesException;
@@ -30,6 +31,8 @@
 
 import java.io.IOException;
 import java.util.NoSuchElementException;
+import java.util.UUID;
+import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -39,6 +42,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 /**
  * An implementation of {@link TransferStrategy} that provides a default implementation {@code transfer}
  * implementation.
@@ -51,9 +56,11 @@ abstract class AbstractTransferStrategy implements TransferStrategy {
     private final ExecutorService executorService;
     private final EventDispatcher eventDispatcher;
     private final MasterObjectList masterObjectList;
+    private final Ds3Client ds3Client;
     private final FailureEvent.FailureActivity failureActivity;
 
     private final AtomicReference cachedException = new AtomicReference<>();
+    private volatile boolean canceled = false;
 
     private TransferMethod transferMethod;
 
@@ -76,6 +83,7 @@ public AbstractTransferStrategy(final BlobStrategy blobStrategy,
                                     final ExecutorService executorService,
                                     final EventDispatcher eventDispatcher,
                                     final MasterObjectList masterObjectList,
+                                    final Ds3Client ds3Client,
                                     final FailureEvent.FailureActivity failureActivity)
     {
         this.blobStrategy = blobStrategy;
@@ -83,6 +91,7 @@ public AbstractTransferStrategy(final BlobStrategy blobStrategy,
         this.executorService = executorService;
         this.eventDispatcher = eventDispatcher;
         this.masterObjectList = masterObjectList;
+        this.ds3Client = ds3Client;
         this.failureActivity = failureActivity;
     }
 
@@ -104,6 +113,7 @@ public AbstractTransferStrategy withTransferMethod(final TransferMethod transfer
     @Override
     public void transfer() throws IOException {
         cachedException.set(null);
+        // Keep going, unless we've been canceled
 
         try {
             transferAllJobBlobs();
@@ -117,11 +127,12 @@ public void transfer() throws IOException {
     }
 
     private void transferAllJobBlobs() throws IOException {
+        final Predicate transferPredicate = () -> ! canceled;
         final AtomicInteger numBlobsRemaining = new AtomicInteger(jobState.numBlobsInJob());
 
-        while (!Thread.currentThread().isInterrupted() && numBlobsRemaining.get() > 0) {
+        while ( ! Thread.currentThread().isInterrupted() && transferPredicate.test() && numBlobsRemaining.get() > 0) {
             try {
-                final Iterable jobParts = jobPartsNotAlreadyTransferred();
+                final Iterable jobParts = jobPartsNotAlreadyTransferred(transferPredicate);
 
                 final int numJobParts = Iterables.size(jobParts);
 
@@ -129,8 +140,8 @@ private void transferAllJobBlobs() throws IOException {
                     break;
                 }
 
-                final CountDownLatch countDownLatch = new CountDownLatch(numJobParts);
-                transferJobParts(jobParts, countDownLatch, numBlobsRemaining);
+                final CountDownLatch countDownLatch = new CountDownLatch(1);
+                transferJobParts(jobParts, countDownLatch, numBlobsRemaining, transferPredicate);
                 countDownLatch.await();
             } catch (final Ds3NoMoreRetriesException | FailedRequestException e) {
                 emitFailureEvent(makeFailureEvent(failureActivity, e, firstChunk()));
@@ -144,10 +155,15 @@ private void transferAllJobBlobs() throws IOException {
     }
 
 
-    private Iterable jobPartsNotAlreadyTransferred() throws IOException, InterruptedException {
+    private Iterable jobPartsNotAlreadyTransferred(final Predicate transferPredicate) throws IOException, InterruptedException {
+        // If we've been canceled, bail
+        if ( ! transferPredicate.test()) {
+            return FluentIterable.of();
+        }
+
         return FluentIterable
                 .from(blobStrategy.getWork())
-                .filter(jobPart -> jobState.contains(jobPart.getBlob()));
+                .filter(jobPart -> jobState.contains(jobPart.getBlob()) && transferPredicate.test());
     }
 
     private void emitFailureAndSetCachedException(final Throwable t) {
@@ -167,24 +183,44 @@ private synchronized void maybeSetCachedException(final Throwable t) {
 
     private void transferJobParts(final Iterable jobParts,
                                   final CountDownLatch countDownLatch,
-                                  final AtomicInteger numBlobsTransferred) throws IOException {
+                                  final AtomicInteger numBlobsTransferred,
+                                  final Predicate transferPredicate) throws IOException
+    {
+        final AtomicInteger numJobPartsToTransfer = new AtomicInteger(Iterables.size(jobParts));
+
         for (final JobPart jobPart : jobParts) {
-            executorService.execute(() -> {
-                try {
+            if (executorService.isShutdown()) {
+                countDownLatch.countDown();
+            } else {
+                maybeSubmitJobPartTransfer(countDownLatch, numBlobsTransferred, transferPredicate, numJobPartsToTransfer, jobPart);
+            }
+        }
+    }
+
+    private void maybeSubmitJobPartTransfer(final CountDownLatch countDownLatch, final AtomicInteger numBlobsTransferred, final Predicate transferPredicate, final AtomicInteger numJobPartsToTransfer, final JobPart jobPart) {
+        executorService.submit((Callable) () -> {
+            try {
+                if ( ! transferPredicate.test()) {
+                    countDownLatch.countDown();
+                } else {
                     transferMethod.transferJobPart(jobPart);
-                } catch (final RuntimeException e) {
-                    emitFailureAndSetCachedException(e);
-                    throw e;
-                } catch (final Exception e) {
-                    emitFailureAndSetCachedException(e);
-                    throw new RuntimeException(e);
-                } finally {
-                    jobState.blobTransferredOrFailed(jobPart.getBlob());
-                    numBlobsTransferred.decrementAndGet();
+                }
+            } catch (final RuntimeException e) {
+                emitFailureAndSetCachedException(e);
+                throw e;
+            } catch (final Exception e) {
+                emitFailureAndSetCachedException(e);
+                throw new RuntimeException(e);
+            } finally {
+                jobState.blobTransferredOrFailed(jobPart.getBlob());
+                numBlobsTransferred.decrementAndGet();
+                if (numJobPartsToTransfer.decrementAndGet() <= 0) {
                     countDownLatch.countDown();
                 }
-            });
-        }
+            }
+
+            return null;
+        });
     }
 
     private void emitFailureEvent(final FailureEvent failureEvent) {
@@ -219,6 +255,19 @@ private Objects firstChunk() {
 
     @Override
     public void close() throws IOException {
+        canceled = true;
         executorService.shutdown();
     }
+
+    @Override
+    public void cancel() throws IOException {
+        close();
+        final UUID jobId = masterObjectList.getJobId();
+        ds3Client.cancelJobSpectraS3(new CancelJobSpectraS3Request(jobId));
+        eventDispatcher.emitCanceledEvent(jobId);
+    }
+
+    private interface Predicate {
+        boolean test();
+    }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/CanceledEventObserver.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/CanceledEventObserver.java
new file mode 100644
index 000000000..64f563960
--- /dev/null
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/CanceledEventObserver.java
@@ -0,0 +1,25 @@
+/*
+ * ****************************************************************************
+ *    Copyright 2014-2018 Spectra Logic Corporation. All Rights Reserved.
+ *    Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *    this file except in compliance with the License. A copy of the License is located at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file.
+ *    This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *    CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *    specific language governing permissions and limitations under the License.
+ *  ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
+
+import java.util.UUID;
+
+public class CanceledEventObserver extends AbstractObserver {
+    public CanceledEventObserver(final UpdateStrategy updateStrategy) {
+        super(updateStrategy);
+    }
+}
+
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java
index 0810ac7bb..0e5e07187 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcher.java
@@ -26,6 +26,8 @@
 import com.spectralogic.ds3client.models.ChecksumType;
 import com.spectralogic.ds3client.networking.Metadata;
 
+import java.util.UUID;
+
 /**
  * An interface to provide the ability to register for and emit events.  In prior versions of this SDK,
  * clients needed to hold an instance of a listener implementation to later unregister its receipt of events.
@@ -118,6 +120,9 @@ public interface EventDispatcher {
     void attachFailureEventListener(final FailureEventListener listener);
     void removeFailureEventListener(final FailureEventListener listener);
 
+    CanceledEventObserver attachCanceledEventObserver(final CanceledEventObserver canceledEventObserver);
+    void removeCanceledEventObserver(final CanceledEventObserver canceledEventObserver);
+
     void emitFailureEvent(final FailureEvent failureEvent);
     void emitWaitingForChunksEvents(final int secondsToDelay);
     void emitChecksumEvent(final BulkObject blob, final ChecksumType.Type type, final String checksum);
@@ -126,6 +131,7 @@ public interface EventDispatcher {
     void emitObjectCompletedEvent(final String blobName);
     void emitMetaDataReceivedEvent(final String objectName, final Metadata metadata);
     void emitBlobTransferredEvent(final BulkObject blob);
+    void emitCanceledEvent(final UUID jobId);
 
     /**
      * Emit an event when a get transfers less content than was intended.  This event is triggered by catching
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java
index 946648f78..3e7f71f89 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/EventDispatcherImpl.java
@@ -33,6 +33,7 @@
 
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 
 public class EventDispatcherImpl implements EventDispatcher {
     private final EventRunner eventRunner;
@@ -44,6 +45,7 @@ public class EventDispatcherImpl implements EventDispatcher {
     private final Set failureEventObservers = Sets.newIdentityHashSet();
     private final Set metaDataReceivedObservers = Sets.newConcurrentHashSet();
     private final Set blobTransferredEventObservers = Sets.newIdentityHashSet();
+    private final Set canceledEventObservers = Sets.newIdentityHashSet();
 
     private final Map dataTransferredListeners = Maps.newConcurrentMap();
     private final Map objectCompletedListeners = Maps.newIdentityHashMap();
@@ -255,19 +257,26 @@ public void removeFailureEventListener(final FailureEventListener listener) {
         }
     }
 
+    @Override
+    public CanceledEventObserver attachCanceledEventObserver(final CanceledEventObserver canceledEventObserver) {
+        canceledEventObservers.add(canceledEventObserver);
+        return canceledEventObserver;
+    }
+
+    @Override
+    public void removeCanceledEventObserver(final CanceledEventObserver canceledEventObserver) {
+        canceledEventObservers.remove(canceledEventObserver);
+    }
+
     @Override
     public void emitFailureEvent(final FailureEvent failureEvent) {
         emitEvents(failureEventObservers, failureEvent);
     }
 
+    @SuppressWarnings("unchecked")
     private  void emitEvents(final Set eventObservers, final T eventData) {
         for (final Object eventObserver : eventObservers) {
-            eventRunner.emitEvent(new Runnable() {
-                @Override
-                public void run() {
-                    ((Observer)eventObserver).update(eventData);
-                }
-            });
+            eventRunner.emitEvent(() -> ((Observer)eventObserver).update(eventData));
         }
     }
 
@@ -306,6 +315,11 @@ public void emitBlobTransferredEvent(final BulkObject blob) {
         emitEvents(blobTransferredEventObservers, blob);
     }
 
+    @Override
+    public void emitCanceledEvent(final UUID jobId) {
+        emitEvents(canceledEventObservers, jobId);
+    }
+
     @Override
     public void emitContentLengthMismatchFailureEvent(final BulkObject ds3Object, final String endpoint, final Throwable t) {
         final FailureEvent failureEvent = FailureEvent.builder()
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java
index bbf0197ef..6855baf6a 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/MultiThreadedTransferStrategy.java
@@ -15,6 +15,7 @@
 
 package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
 
+import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.helpers.JobState;
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
 import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy;
@@ -31,6 +32,7 @@ public MultiThreadedTransferStrategy(final BlobStrategy blobStrategy,
                                          final int numConcurrentTransferThreads,
                                          final EventDispatcher eventDispatcher,
                                          final MasterObjectList masterObjectList,
+                                         final Ds3Client ds3Client,
                                          final FailureEvent.FailureActivity failureActivity)
     {
         super(blobStrategy,
@@ -38,6 +40,7 @@ public MultiThreadedTransferStrategy(final BlobStrategy blobStrategy,
               Executors.newFixedThreadPool(numConcurrentTransferThreads),
               eventDispatcher,
               masterObjectList,
+              ds3Client,
               failureActivity);
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java
index 7991fc2b7..773573c87 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/SingleThreadedTransferStrategy.java
@@ -15,6 +15,8 @@
 
 package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
 
+import com.google.common.util.concurrent.MoreExecutors;
+import com.spectralogic.ds3client.Ds3Client;
 import com.spectralogic.ds3client.helpers.JobState;
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
 import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy;
@@ -30,6 +32,7 @@ public SingleThreadedTransferStrategy(final BlobStrategy blobStrategy,
                                           final JobState jobState,
                                           final EventDispatcher eventDispatcher,
                                           final MasterObjectList masterObjectList,
+                                          final Ds3Client ds3Client,
                                           final FailureEvent.FailureActivity failureActivity)
     {
         super(blobStrategy,
@@ -37,6 +40,7 @@ public SingleThreadedTransferStrategy(final BlobStrategy blobStrategy,
               Executors.newSingleThreadExecutor(),
               eventDispatcher,
               masterObjectList,
+              ds3Client,
               failureActivity);
     }
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java
index f67ccf349..e85c0a66b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy.java
@@ -27,4 +27,11 @@ public interface TransferStrategy extends Closeable {
      * @throws IOException
      */
     void transfer() throws IOException;
+
+    /**
+     * Cancel a transfer in progress.  This is a cooperative activity, meaning that transfers
+     * in progress are allowed to complete and are not forcefully canceled.
+     * @throws IOException
+     */
+    void cancel() throws IOException;
 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java
index a6c2678dc..58bd8c018 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategyBuilder.java
@@ -558,6 +558,7 @@ private TransferStrategy makeTransferStrategy(final TransferMethod transferMetho
                         jobState,
                         eventDispatcher,
                         masterObjectList,
+                        ds3Client,
                         failureActivity)
                         .withTransferMethod(transferMethod);
 
@@ -567,6 +568,7 @@ private TransferStrategy makeTransferStrategy(final TransferMethod transferMetho
                         numConcurrentTransferThreads,
                         eventDispatcher,
                         masterObjectList,
+                        ds3Client,
                         failureActivity)
                         .withTransferMethod(transferMethod);
 
@@ -694,6 +696,7 @@ private TransferStrategy makeOriginalSdkSemanticsTransferStrategy(final Transfer
                     numConcurrentTransferThreads,
                     eventDispatcher,
                     masterObjectList,
+                    ds3Client,
                     failureActivity)
                     .withTransferMethod(transferMethod);
         } else {
@@ -701,6 +704,7 @@ private TransferStrategy makeOriginalSdkSemanticsTransferStrategy(final Transfer
                     jobState,
                     eventDispatcher,
                     masterObjectList,
+                    ds3Client,
                     failureActivity)
                     .withTransferMethod(transferMethod);
         }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java
index 4580f9bac..482699770 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java
@@ -16,6 +16,8 @@
 package com.spectralogic.ds3client.helpers.strategy.transferstrategy;
 
 import com.google.common.collect.Sets;
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
 import com.spectralogic.ds3client.helpers.JobPart;
 import com.spectralogic.ds3client.helpers.JobState;
 import com.spectralogic.ds3client.helpers.events.FailureEvent;
@@ -26,13 +28,18 @@
 import com.spectralogic.ds3client.models.Objects;
 
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import java.io.IOException;
+import java.nio.channels.SeekableByteChannel;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -49,11 +56,14 @@ public void testContinuallyGettingBlobThatFailsToTransfer() throws IOException {
         final Set blobsThatFailed = Sets.newConcurrentHashSet();
         final AtomicInteger numTimesTransferCalled = new AtomicInteger(0);
 
+        final Ds3Client ds3Client = Mockito.mock(Ds3Client.class);
+
         new MultiThreadedTransferStrategy(new ContinuallyReturnFailingBlobStrategy(),
                 new JobState(chunksWithBlobs()),
                 10,
                 new EventDispatcherImpl(new SameThreadEventRunner()),
                 masterObjectList(),
+                ds3Client,
                 FailureEvent.FailureActivity.GettingObject)
                 .withTransferMethod(transferMethod(blobsTransferredSuccessfully, blobsThatFailed, numTimesTransferCalled))
                 .transfer();
@@ -103,10 +113,10 @@ private JobPart[] remainingChunkWithFailingBlob() {
     }
 
     private List chunksWithBlobs() {
-        final BulkObject blob1 = new BulkObject(); blob1.setName(BLOB1);
-        final BulkObject blob2 = new BulkObject(); blob2.setName(BLOB2);
-        final BulkObject blob3 = new BulkObject(); blob3.setName(BLOB3);
-        final BulkObject blob4 = new BulkObject(); blob4.setName(BLOB4);
+        final BulkObject blob1 = new BulkObject(); blob1.setName(BLOB1); blob1.setInCache(false);
+        final BulkObject blob2 = new BulkObject(); blob2.setName(BLOB2); blob2.setInCache(false);
+        final BulkObject blob3 = new BulkObject(); blob3.setName(BLOB3); blob3.setInCache(false);
+        final BulkObject blob4 = new BulkObject(); blob4.setName(BLOB4); blob4.setInCache(false);
 
         final Objects chunk1 = new Objects();
         chunk1.setObjects(Arrays.asList(blob1, blob2));
@@ -120,6 +130,8 @@ private List chunksWithBlobs() {
     private MasterObjectList masterObjectList() {
         final MasterObjectList masterObjectList = new MasterObjectList();
         masterObjectList.setObjects(chunksWithBlobs());
+        masterObjectList.setJobId(UUID.randomUUID());
+        masterObjectList.setBucketName("bucket");
 
         return masterObjectList;
     }
@@ -128,18 +140,130 @@ private TransferMethod transferMethod(final Set blobsTransferredSuccessf
                                           final Set blobsThatFailed,
                                           final AtomicInteger numTimesTransferCalled)
     {
-        return new TransferMethod() {
-            @Override
-            public void transferJobPart(final JobPart jobPart) throws IOException {
-                numTimesTransferCalled.incrementAndGet();
-
-                if (jobPart.getBlob().getName().equals(BLOB2)) {
-                    blobsThatFailed.add(jobPart.getBlob().getName());
-                    throw new IOException(BLOB2);
-                } else {
-                    blobsTransferredSuccessfully.add(jobPart.getBlob().getName());
-                }
+        return jobPart -> {
+            numTimesTransferCalled.incrementAndGet();
+
+            if (jobPart.getBlob().getName().equals(BLOB2)) {
+                blobsThatFailed.add(jobPart.getBlob().getName());
+                throw new IOException(BLOB2);
+            } else {
+                blobsTransferredSuccessfully.add(jobPart.getBlob().getName());
             }
         };
     }
+
+    @Test
+    public void testCanceledTransferDoesNotTryToGetBlobs() throws IOException {
+        final Ds3Client ds3Client = Mockito.mock(Ds3Client.class);
+
+        final CanceledBlobStrategy blobStrategy = new CanceledBlobStrategy();
+
+        final TransferStrategy transferStrategy = new TransferStrategyBuilder()
+                .withMasterObjectList(masterObjectList())
+                .withBlobStrategy(blobStrategy)
+                .withChannelBuilder(new AChannelBuilder())
+                .withDs3Client(ds3Client)
+                .makePutTransferStrategy();
+
+        transferStrategy.transfer();
+
+        assertEquals(4, blobStrategy.numAttemptsToUse());
+
+        blobStrategy.reset();
+
+        new Thread(() -> {
+            try {
+                transferStrategy.cancel();
+            } catch (final Throwable t) {
+                System.out.println();
+            }
+        }).start();
+
+        transferStrategy.cancel();
+
+        transferStrategy.transfer();
+
+        assertEquals(0, blobStrategy.numAttemptsToUse());
+    }
+
+    private interface CanceledBehavior {
+        int numAttemptsToUse();
+        void reset();
+        void transferStrategy(final TransferStrategy transferStrategy);
+        void onGetWork(final OnGetWork onGetWork);
+    }
+
+    private interface OnGetWork {
+        void onGetWork(final TransferStrategy transferStrategy) throws IOException;
+    }
+
+    private static class CanceledBlobStrategy implements BlobStrategy, CanceledBehavior {
+        private final Ds3Client ds3Client = Mockito.mock(Ds3Client.class);
+        private final AtomicInteger numAttemptsToUse = new AtomicInteger(0);
+        private TransferStrategy transferStrategy;
+        private final AtomicReference onGetWork = new AtomicReference<>((transferStrategy) -> { });
+
+        @Override
+        public Iterable getWork() throws IOException, InterruptedException {
+            final int numAttempts = numAttemptsToUse.incrementAndGet();
+            onGetWork.get().onGetWork(transferStrategy);
+            final BulkObject blob = new BulkObject(); blob.setName("blob" + numAttempts); blob.setInCache(false);
+            return Collections.singletonList(new JobPart(ds3Client, blob));
+        }
+
+        @Override
+        public int numAttemptsToUse() {
+            return numAttemptsToUse.get();
+        }
+
+        @Override
+        public void reset() {
+            numAttemptsToUse.set(0);
+        }
+
+        @Override
+        public void transferStrategy(final TransferStrategy transferStrategy) {
+            this.transferStrategy = transferStrategy;
+        }
+
+        @Override
+        public void onGetWork(final OnGetWork onGetWork) {
+            this.onGetWork.set(onGetWork);
+        }
+    }
+
+    private static class AChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
+        @Override
+        public SeekableByteChannel buildChannel(final String key) throws IOException {
+            return null;
+        }
+    }
+
+    @Test
+    public void testCancelingTransferWhileGettingBlobs() throws IOException {
+        final Ds3Client ds3Client = Mockito.mock(Ds3Client.class);
+
+        final CanceledBlobStrategy blobStrategy = new CanceledBlobStrategy();
+
+        final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
+                .withMasterObjectList(masterObjectList())
+                .withBlobStrategy(blobStrategy)
+                .withChannelBuilder(new AChannelBuilder())
+                .withDs3Client(ds3Client);
+
+        final AtomicInteger canceledCount = new AtomicInteger(0);
+
+        transferStrategyBuilder.eventDispatcher()
+                .attachCanceledEventObserver(new CanceledEventObserver(eventData -> canceledCount.incrementAndGet()));
+
+        final TransferStrategy transferStrategy = transferStrategyBuilder.makePutTransferStrategy();
+
+        blobStrategy.transferStrategy(transferStrategy);
+        blobStrategy.onGetWork(TransferStrategy::cancel);
+
+        transferStrategy.transfer();
+
+        assertEquals(1, blobStrategy.numAttemptsToUse());
+        assertEquals(1, canceledCount.get());
+    }
 }

From 53cda89faa827497d566f21a987323e204dee581 Mon Sep 17 00:00:00 2001
From: GraciesPadre 
Date: Mon, 12 Feb 2018 21:01:10 -0700
Subject: [PATCH 166/229] codacity whines

---
 .../integration/PutJobManagement_Test.java       | 16 ++++++++--------
 .../spectralogic/ds3client/helpers/JobImpl.java  |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
index ba99e0f30..1dd390129 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
@@ -94,6 +94,11 @@ public class PutJobManagement_Test {
     private static TempStorageIds envStorageIds;
     private static UUID envDataPolicyId;
 
+    private static final String SOURCE_DIRECTORY = "little_files/";
+    private static final String SOURCE_FILE_BASE_NAME = "tape";
+    private static final String SOURCE_FILE_EXTENSION = "png";
+    private static final String SOURCE_FILE_NAME = SOURCE_FILE_BASE_NAME + "." + SOURCE_FILE_EXTENSION;
+
     @BeforeClass
     public static void startup() throws IOException {
         envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, client);
@@ -1943,11 +1948,6 @@ public void onFailure(final FailureEvent failureEvent) {
         }
     }
 
-    private static final String SOURCE_DIRECTORY = "little_files/";
-    private static final String SOURCE_FILE_BASE_NAME = "tape";
-    private static final String SOURCE_FILE_EXTENSION = "png";
-    private static final String SOURCE_FILE_NAME = SOURCE_FILE_BASE_NAME + "." + SOURCE_FILE_EXTENSION;
-
     @Test
     public void testPutting15000Files() throws IOException, URISyntaxException {
         final String tempPathPrefix = null;
@@ -2057,10 +2057,10 @@ public void testCancelingJob() throws URISyntaxException, InterruptedException {
             writeJob.attachObjectCompletedListener(name -> {
                 try {
                     writeJob.cancel();
+                } catch (final FailedRequestException failedRequestException) {
+                    caughtExceptionWhileCanceling.set(false);
                 } catch (final Throwable t) {
-                    if ( ! (t instanceof FailedRequestException)) {
-                        caughtExceptionWhileCanceling.set(true);
-                    }
+                    caughtExceptionWhileCanceling.set(true);
                 } finally {
                     countDownLatch.countDown();
                 }
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java
index 04829877b..bebd93d67 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java
@@ -149,7 +149,7 @@ public void transfer(final Ds3ClientHelpers.ObjectChannelBuilder channelBuilder)
         transferStrategyBuilder.withChannelBuilder(channelBuilder);
     }
 
-    TransferStrategyBuilder transferStrategyBuilder() {
+    protected TransferStrategyBuilder transferStrategyBuilder() {
         return transferStrategyBuilder;
     }
 
@@ -157,7 +157,7 @@ protected EventDispatcher eventDispatcher() {
         return transferStrategyBuilder.eventDispatcher();
     }
 
-    void cancel(final TransferStrategy transferStrategy) throws IOException {
+    protected void cancel(final TransferStrategy transferStrategy) throws IOException {
         if (transferStrategy != null) {
             transferStrategy.cancel();
         }

From 62ad828bf7eb60424784bf52a262428a5c40cd0b Mon Sep 17 00:00:00 2001
From: scribe 
Date: Mon, 10 Sep 2018 16:41:21 -0600
Subject: [PATCH 167/229] Changes based on feedback

---
 .../AbstractTransferStrategy.java             | 37 ++++++++++---------
 .../TransferStrategy_Test.java                |  5 ++-
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index 4720991e2..d377b5a2b 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -37,12 +37,12 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
 
 import com.spectralogic.ds3client.networking.FailedRequestException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
 
 /**
  * An implementation of {@link TransferStrategy} that provides a default implementation {@code transfer}
@@ -127,22 +127,25 @@ public void transfer() throws IOException {
     }
 
     private void transferAllJobBlobs() throws IOException {
-        final Predicate transferPredicate = () -> ! canceled;
+        final BooleanSupplier cancelationCheck = () -> ! canceled;
         final AtomicInteger numBlobsRemaining = new AtomicInteger(jobState.numBlobsInJob());
 
-        while ( ! Thread.currentThread().isInterrupted() && transferPredicate.test() && numBlobsRemaining.get() > 0) {
+        while ( ! Thread.currentThread().isInterrupted() && cancelationCheck.getAsBoolean() && numBlobsRemaining.get() > 0) {
             try {
-                final Iterable jobParts = jobPartsNotAlreadyTransferred(transferPredicate);
+                final Iterable jobParts = jobPartsNotAlreadyTransferred(cancelationCheck);
 
                 final int numJobParts = Iterables.size(jobParts);
 
-                if (numJobParts <= 0) {
+                if (numJobParts == 0) {
                     break;
+                } else if (numJobParts < 0) {
+                    LOG.error("Had negative number of job parts");
+                    return;
                 }
 
-                final CountDownLatch countDownLatch = new CountDownLatch(1);
-                transferJobParts(jobParts, countDownLatch, numBlobsRemaining, transferPredicate);
-                countDownLatch.await();
+                final CountDownLatch jobCompletionWait = new CountDownLatch(1);
+                transferJobParts(jobParts, jobCompletionWait, numBlobsRemaining, cancelationCheck);
+                jobCompletionWait.await();
             } catch (final Ds3NoMoreRetriesException | FailedRequestException e) {
                 emitFailureEvent(makeFailureEvent(failureActivity, e, firstChunk()));
                 throw e;
@@ -155,15 +158,15 @@ private void transferAllJobBlobs() throws IOException {
     }
 
 
-    private Iterable jobPartsNotAlreadyTransferred(final Predicate transferPredicate) throws IOException, InterruptedException {
+    private Iterable jobPartsNotAlreadyTransferred(final BooleanSupplier transferPredicate) throws IOException, InterruptedException {
         // If we've been canceled, bail
-        if ( ! transferPredicate.test()) {
+        if ( ! transferPredicate.getAsBoolean()) {
             return FluentIterable.of();
         }
 
         return FluentIterable
                 .from(blobStrategy.getWork())
-                .filter(jobPart -> jobState.contains(jobPart.getBlob()) && transferPredicate.test());
+                .filter(jobPart -> jobState.contains(jobPart.getBlob()) && transferPredicate.getAsBoolean());
     }
 
     private void emitFailureAndSetCachedException(final Throwable t) {
@@ -184,12 +187,13 @@ private synchronized void maybeSetCachedException(final Throwable t) {
     private void transferJobParts(final Iterable jobParts,
                                   final CountDownLatch countDownLatch,
                                   final AtomicInteger numBlobsTransferred,
-                                  final Predicate transferPredicate) throws IOException
+                                  final BooleanSupplier transferPredicate)
     {
         final AtomicInteger numJobPartsToTransfer = new AtomicInteger(Iterables.size(jobParts));
 
         for (final JobPart jobPart : jobParts) {
             if (executorService.isShutdown()) {
+                LOG.debug("Executor service is shut down, decrementing countdown latch");
                 countDownLatch.countDown();
             } else {
                 maybeSubmitJobPartTransfer(countDownLatch, numBlobsTransferred, transferPredicate, numJobPartsToTransfer, jobPart);
@@ -197,10 +201,10 @@ private void transferJobParts(final Iterable jobParts,
         }
     }
 
-    private void maybeSubmitJobPartTransfer(final CountDownLatch countDownLatch, final AtomicInteger numBlobsTransferred, final Predicate transferPredicate, final AtomicInteger numJobPartsToTransfer, final JobPart jobPart) {
+    private void maybeSubmitJobPartTransfer(final CountDownLatch countDownLatch, final AtomicInteger numBlobsTransferred, final BooleanSupplier transferPredicate, final AtomicInteger numJobPartsToTransfer, final JobPart jobPart) {
         executorService.submit((Callable) () -> {
             try {
-                if ( ! transferPredicate.test()) {
+                if ( ! transferPredicate.getAsBoolean()) {
                     countDownLatch.countDown();
                 } else {
                     transferMethod.transferJobPart(jobPart);
@@ -254,7 +258,7 @@ private Objects firstChunk() {
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() {
         canceled = true;
         executorService.shutdown();
     }
@@ -267,7 +271,4 @@ public void cancel() throws IOException {
         eventDispatcher.emitCanceledEvent(jobId);
     }
 
-    private interface Predicate {
-        boolean test();
-    }
 }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java
index 482699770..0939b3258 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/TransferStrategy_Test.java
@@ -174,8 +174,9 @@ public void testCanceledTransferDoesNotTryToGetBlobs() throws IOException {
         new Thread(() -> {
             try {
                 transferStrategy.cancel();
-            } catch (final Throwable t) {
-                System.out.println();
+            } catch (final IOException e) {
+                System.out.println(e.getMessage());
+                assert(false);
             }
         }).start();
 

From 3af9f427100847a121ce03e1802763fdd9a25e7c Mon Sep 17 00:00:00 2001
From: scribe 
Date: Thu, 13 Sep 2018 09:55:43 -0600
Subject: [PATCH 168/229] Working version

---
 build.gradle                                  |  4 +-
 .../AbstractTransferStrategy.java             | 78 ++++++++-----------
 2 files changed, 35 insertions(+), 47 deletions(-)

diff --git a/build.gradle b/build.gradle
index 218f06963..9ac650061 100644
--- a/build.gradle
+++ b/build.gradle
@@ -14,7 +14,7 @@
  */
 
 buildscript {
-    ext.kotlin_version = '1.1.51'
+    ext.kotlin_version = '1.2.61'
 
     repositories {
         mavenCentral()
@@ -36,7 +36,7 @@ plugins {
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '5.0.3'
+    version = '5.0.4.5-debug'
 }
 
 subprojects {
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index d377b5a2b..1c799565f 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -32,9 +32,9 @@
 import java.io.IOException;
 import java.util.NoSuchElementException;
 import java.util.UUID;
-import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BooleanSupplier;
@@ -43,7 +43,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
 /**
  * An implementation of {@link TransferStrategy} that provides a default implementation {@code transfer}
  * implementation.
@@ -127,25 +126,22 @@ public void transfer() throws IOException {
     }
 
     private void transferAllJobBlobs() throws IOException {
-        final BooleanSupplier cancelationCheck = () -> ! canceled;
+        final BooleanSupplier cancellationCheck = () -> !canceled;
         final AtomicInteger numBlobsRemaining = new AtomicInteger(jobState.numBlobsInJob());
 
-        while ( ! Thread.currentThread().isInterrupted() && cancelationCheck.getAsBoolean() && numBlobsRemaining.get() > 0) {
+        while (!Thread.currentThread().isInterrupted() && cancellationCheck.getAsBoolean() && numBlobsRemaining.get() > 0) {
             try {
-                final Iterable jobParts = jobPartsNotAlreadyTransferred(cancelationCheck);
+                final Iterable jobParts = jobPartsNotAlreadyTransferred(cancellationCheck);
 
                 final int numJobParts = Iterables.size(jobParts);
 
                 if (numJobParts == 0) {
                     break;
-                } else if (numJobParts < 0) {
-                    LOG.error("Had negative number of job parts");
-                    return;
                 }
 
-                final CountDownLatch jobCompletionWait = new CountDownLatch(1);
-                transferJobParts(jobParts, jobCompletionWait, numBlobsRemaining, cancelationCheck);
-                jobCompletionWait.await();
+                final CountDownLatch countDownLatch = new CountDownLatch(numJobParts);
+                transferJobParts(jobParts, countDownLatch, numBlobsRemaining, cancellationCheck);
+                countDownLatch.await();
             } catch (final Ds3NoMoreRetriesException | FailedRequestException e) {
                 emitFailureEvent(makeFailureEvent(failureActivity, e, firstChunk()));
                 throw e;
@@ -158,15 +154,15 @@ private void transferAllJobBlobs() throws IOException {
     }
 
 
-    private Iterable jobPartsNotAlreadyTransferred(final BooleanSupplier transferPredicate) throws IOException, InterruptedException {
+    private Iterable jobPartsNotAlreadyTransferred(final BooleanSupplier cancellationCheck) throws IOException, InterruptedException {
         // If we've been canceled, bail
-        if ( ! transferPredicate.getAsBoolean()) {
+        if ( ! cancellationCheck.getAsBoolean()) {
             return FluentIterable.of();
         }
 
         return FluentIterable
                 .from(blobStrategy.getWork())
-                .filter(jobPart -> jobState.contains(jobPart.getBlob()) && transferPredicate.getAsBoolean());
+                .filter(jobPart -> jobState.contains(jobPart.getBlob()) && cancellationCheck.getAsBoolean());
     }
 
     private void emitFailureAndSetCachedException(final Throwable t) {
@@ -187,46 +183,33 @@ private synchronized void maybeSetCachedException(final Throwable t) {
     private void transferJobParts(final Iterable jobParts,
                                   final CountDownLatch countDownLatch,
                                   final AtomicInteger numBlobsTransferred,
-                                  final BooleanSupplier transferPredicate)
-    {
-        final AtomicInteger numJobPartsToTransfer = new AtomicInteger(Iterables.size(jobParts));
-
+                                  final BooleanSupplier cancellationCheck) {
         for (final JobPart jobPart : jobParts) {
             if (executorService.isShutdown()) {
                 LOG.debug("Executor service is shut down, decrementing countdown latch");
                 countDownLatch.countDown();
             } else {
-                maybeSubmitJobPartTransfer(countDownLatch, numBlobsTransferred, transferPredicate, numJobPartsToTransfer, jobPart);
+                executorService.execute(() -> {
+                    try {
+                        if (cancellationCheck.getAsBoolean()) {
+                            transferMethod.transferJobPart(jobPart);
+                        }
+                    } catch (final RuntimeException e) {
+                        emitFailureAndSetCachedException(e);
+                        throw e;
+                    } catch (final Exception e) {
+                        emitFailureAndSetCachedException(e);
+                        throw new RuntimeException(e);
+                    } finally {
+                        jobState.blobTransferredOrFailed(jobPart.getBlob());
+                        numBlobsTransferred.decrementAndGet();
+                        countDownLatch.countDown();
+                    }
+                });
             }
         }
     }
 
-    private void maybeSubmitJobPartTransfer(final CountDownLatch countDownLatch, final AtomicInteger numBlobsTransferred, final BooleanSupplier transferPredicate, final AtomicInteger numJobPartsToTransfer, final JobPart jobPart) {
-        executorService.submit((Callable) () -> {
-            try {
-                if ( ! transferPredicate.getAsBoolean()) {
-                    countDownLatch.countDown();
-                } else {
-                    transferMethod.transferJobPart(jobPart);
-                }
-            } catch (final RuntimeException e) {
-                emitFailureAndSetCachedException(e);
-                throw e;
-            } catch (final Exception e) {
-                emitFailureAndSetCachedException(e);
-                throw new RuntimeException(e);
-            } finally {
-                jobState.blobTransferredOrFailed(jobPart.getBlob());
-                numBlobsTransferred.decrementAndGet();
-                if (numJobPartsToTransfer.decrementAndGet() <= 0) {
-                    countDownLatch.countDown();
-                }
-            }
-
-            return null;
-        });
-    }
-
     private void emitFailureEvent(final FailureEvent failureEvent) {
         eventDispatcher.emitFailureEvent(failureEvent);
     }
@@ -261,6 +244,11 @@ private Objects firstChunk() {
     public void close() {
         canceled = true;
         executorService.shutdown();
+        try {
+            executorService.awaitTermination(10000, TimeUnit.MILLISECONDS);
+        } catch (final InterruptedException e) {
+            executorService.shutdownNow();
+        }
     }
 
     @Override

From a42c6d63a5793ec29c9ce28ee2679be934695414 Mon Sep 17 00:00:00 2001
From: scribe 
Date: Thu, 13 Sep 2018 14:15:45 -0600
Subject: [PATCH 169/229] Redoing changes

---
 build.gradle                                  |  4 ++--
 .../integration/PutJobManagement_Test.java    | 21 +++++++++++--------
 .../AbstractTransferStrategy.java             | 19 ++++++++++-------
 3 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/build.gradle b/build.gradle
index 9ac650061..a89a6debf 100644
--- a/build.gradle
+++ b/build.gradle
@@ -14,7 +14,7 @@
  */
 
 buildscript {
-    ext.kotlin_version = '1.2.61'
+    ext.kotlin_version = '1.2.70'
 
     repositories {
         mavenCentral()
@@ -36,7 +36,7 @@ plugins {
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '5.0.4.5-debug'
+    version = '5.0.4'
 }
 
 subprojects {
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
index 1dd390129..da3354a0b 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java
@@ -2054,15 +2054,18 @@ public void testCancelingJob() throws URISyntaxException, InterruptedException {
             final AtomicBoolean cancelHandlerCalled = new AtomicBoolean(false);
             final CountDownLatch countDownLatch = new CountDownLatch(1);
 
-            writeJob.attachObjectCompletedListener(name -> {
-                try {
-                    writeJob.cancel();
-                } catch (final FailedRequestException failedRequestException) {
-                    caughtExceptionWhileCanceling.set(false);
-                } catch (final Throwable t) {
-                    caughtExceptionWhileCanceling.set(true);
-                } finally {
-                    countDownLatch.countDown();
+            final AtomicBoolean hasBeenCancelled = new AtomicBoolean(false);
+            writeJob.attachDataTransferredListener(dataTransferred -> {
+                if(hasBeenCancelled.compareAndSet(false, true)) {
+                    new Thread(() -> {
+                        try {
+                            writeJob.cancel();
+                        } catch (IOException e) {
+                            fail();
+                        } finally {
+                            countDownLatch.countDown();
+                        }
+                    }).start();
                 }
             });
 
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
index 1c799565f..3fde91ccf 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/transferstrategy/AbstractTransferStrategy.java
@@ -135,17 +135,18 @@ private void transferAllJobBlobs() throws IOException {
 
                 final int numJobParts = Iterables.size(jobParts);
 
-                if (numJobParts == 0) {
+                if (numJobParts <= 0) {
                     break;
                 }
 
-                final CountDownLatch countDownLatch = new CountDownLatch(numJobParts);
-                transferJobParts(jobParts, countDownLatch, numBlobsRemaining, cancellationCheck);
-                countDownLatch.await();
+                final CountDownLatch jobPartsCompleteLatch = new CountDownLatch(numJobParts);
+                transferJobParts(jobParts, jobPartsCompleteLatch, numBlobsRemaining, cancellationCheck);
+                jobPartsCompleteLatch.await();
             } catch (final Ds3NoMoreRetriesException | FailedRequestException e) {
                 emitFailureEvent(makeFailureEvent(failureActivity, e, firstChunk()));
                 throw e;
             } catch (final InterruptedException | NoSuchElementException e) {
+                LOG.warn("Thread was interrupted");
                 Thread.currentThread().interrupt();
             } catch (final Throwable t) {
                 emitFailureAndSetCachedException(t);
@@ -181,13 +182,13 @@ private synchronized void maybeSetCachedException(final Throwable t) {
     }
 
     private void transferJobParts(final Iterable jobParts,
-                                  final CountDownLatch countDownLatch,
+                                  final CountDownLatch jobPartsCompleteLatch,
                                   final AtomicInteger numBlobsTransferred,
                                   final BooleanSupplier cancellationCheck) {
         for (final JobPart jobPart : jobParts) {
             if (executorService.isShutdown()) {
                 LOG.debug("Executor service is shut down, decrementing countdown latch");
-                countDownLatch.countDown();
+                jobPartsCompleteLatch.countDown();
             } else {
                 executorService.execute(() -> {
                     try {
@@ -203,7 +204,7 @@ private void transferJobParts(final Iterable jobParts,
                     } finally {
                         jobState.blobTransferredOrFailed(jobPart.getBlob());
                         numBlobsTransferred.decrementAndGet();
-                        countDownLatch.countDown();
+                        jobPartsCompleteLatch.countDown();
                     }
                 });
             }
@@ -245,7 +246,9 @@ public void close() {
         canceled = true;
         executorService.shutdown();
         try {
-            executorService.awaitTermination(10000, TimeUnit.MILLISECONDS);
+            if(!executorService.awaitTermination(10, TimeUnit.SECONDS)) {
+                executorService.shutdownNow();
+            };
         } catch (final InterruptedException e) {
             executorService.shutdownNow();
         }

From 9f8a30535a8f5f50289881b4fa74ab012aa6572f Mon Sep 17 00:00:00 2001
From: Ryan Moore 
Date: Fri, 14 Sep 2018 10:11:22 -0600
Subject: [PATCH 170/229] Update README.md

Updating the maven and gradle examples to use the latest version number.
---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 81c3d8c28..1f6b11291 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ The SDK can also be included directly into a Maven or Gradle build. There is als
       
         com.spectralogic.ds3
         ds3-sdk
-        3.5.2
+        5.0.4
         
       
     ...
@@ -64,8 +64,8 @@ repositories {
 
 dependencies {
     ...
-    compile 'com.spectralogic.ds3:ds3-sdk:3.5.2'
-    // compile 'com.spectralogic.ds3:ds3-sdk:3.5.2:all'
+    compile 'com.spectralogic.ds3:ds3-sdk:5.0.4'
+    // compile 'com.spectralogic.ds3:ds3-sdk:5.0.4:all'
     ...
 }
 

From eaa17329dd561a04389de3c78d1f03e68e166a01 Mon Sep 17 00:00:00 2001
From: Sharon Shabtai 
Date: Wed, 17 Apr 2019 20:33:14 -0600
Subject: [PATCH 171/229] Fix metadata header escape, split between key and
 value

---
 .../ds3client/integration/Metadata_Test.java  |   4 +-
 .../interfaces/RequestHeadersImpl.java        |  11 +-
 .../commands/interfaces/MetadataImpl.java     |   4 +-
 .../utils/MetadataStringManipulation.java     |  44 +++--
 .../MetadataStringManipulation_Test.java      | 186 ++++++++++++------
 5 files changed, 169 insertions(+), 80 deletions(-)

diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Metadata_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Metadata_Test.java
index 00733373e..96a1c5c23 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Metadata_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Metadata_Test.java
@@ -125,7 +125,7 @@ public void multipleMetadataFields() throws IOException {
     @Test
     public void escapedMetadataWithSpaces() throws IOException {
         final String stringWithSpaces = "percent encoded space";
-        final String escapedWithSpaces = MetadataStringManipulation.toEncodedString(stringWithSpaces);
+        final String escapedWithSpaces = MetadataStringManipulation.toEncodedKeyString(stringWithSpaces);
 
         final Metadata metadata = processMetadataRequest(
                 "metadataBucket",
@@ -145,7 +145,7 @@ public void escapedMetadataWithSpaces() throws IOException {
 
     @Test
     public void escapedMetadataWithSymbols() throws IOException {
-        final String escapedWithSymbols = MetadataStringManipulation.toEncodedString(STRING_WITH_SYMBOLS);
+        final String escapedWithSymbols = MetadataStringManipulation.toEncodedKeyString(STRING_WITH_SYMBOLS);
 
         final Metadata metadata = processMetadataRequest(
                 "metadataBucket",
diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/RequestHeadersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/RequestHeadersImpl.java
index cb6e7f5fb..49128284d 100644
--- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/RequestHeadersImpl.java
+++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/RequestHeadersImpl.java
@@ -23,7 +23,8 @@
 import java.util.Set;
 
 import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toDecodedString;
-import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toEncodedString;
+import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toEncodedKeyString;
+import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toEncodedValueString;
 
 /**
  * Implementation of request headers which percent encodes key and values.
@@ -37,7 +38,7 @@ public class RequestHeadersImpl implements RequestHeaders {
      */
     @Override
     public void put(final String key, final String value) {
-        headers.put(toEncodedString(key), toEncodedString(value));
+        headers.put(toEncodedKeyString(key), toEncodedValueString(value));
     }
 
     /**
@@ -47,7 +48,7 @@ public void put(final String key, final String value) {
     @Override
     public Collection get(final String key) {
         final ImmutableList.Builder builder = ImmutableList.builder();
-        for (final String value : headers.get(toEncodedString(key))) {
+        for (final String value : headers.get(toEncodedKeyString(key))) {
             builder.add(toDecodedString(value));
         }
         return builder.build();
@@ -67,7 +68,7 @@ public int size() {
      */
     @Override
     public boolean containsKey(final String key) {
-        return headers.containsKey(toEncodedString(key));
+        return headers.containsKey(toEncodedKeyString(key));
     }
 
     /**
@@ -76,7 +77,7 @@ public boolean containsKey(final String key) {
      */
     @Override
     public Collection removeAll(final String key) {
-        return headers.removeAll(toEncodedString(key));
+        return headers.removeAll(toEncodedKeyString(key));
     }
 
     /**
diff --git a/ds3-utils/src/main/java/com/spectralogic/ds3client/commands/interfaces/MetadataImpl.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/commands/interfaces/MetadataImpl.java
index 6fef3de88..b3acbe241 100644
--- a/ds3-utils/src/main/java/com/spectralogic/ds3client/commands/interfaces/MetadataImpl.java
+++ b/ds3-utils/src/main/java/com/spectralogic/ds3client/commands/interfaces/MetadataImpl.java
@@ -25,7 +25,7 @@
 import java.util.Set;
 
 import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toDecodedString;
-import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toEncodedString;
+import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toEncodedKeyString;
 
 public class MetadataImpl implements Metadata {
 
@@ -85,7 +85,7 @@ private static List getValues(final Headers headers, final String key) {
     @Override
     public List get(final String name) {
         //Only ASCII chars are lower cased, do not search for key with non-ASCII symbols lower cased
-        final String lowerCasedName = toDecodedString(toEncodedString(name).toLowerCase());
+        final String lowerCasedName = toDecodedString(toEncodedKeyString(name).toLowerCase());
         return metadata.get(lowerCasedName).asList();
     }
 
diff --git a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/MetadataStringManipulation.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/MetadataStringManipulation.java
index 24dbbf091..adca9706d 100644
--- a/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/MetadataStringManipulation.java
+++ b/ds3-utils/src/main/java/com/spectralogic/ds3client/utils/MetadataStringManipulation.java
@@ -31,15 +31,17 @@ public final class MetadataStringManipulation {
 
     /**
      * List of printable US-ASCII characters that do not need percent encoding.
-     * Includes separators equals "=" and comma ",": not encoded for use in creating Range header
-     * Excludes separators: "(" | ")" | "<" | ">" | "@" | ";" | ":" | "\" | <"> | "/" | "[" | "]" | "?" | "{" | "}"
-     * Excludes Percent "%": not considered safe because it is used in percent encoding
-     * Excludes Plus "+": not considered safe because its interpreted as space during decoding
+     * Following the spec https://tools.ietf.org/html/rfc2616#page-31 for Message Headers
      */
-    private static final String HTTP_HEADER_SAFE_CHARS = "!#$&'*-.~^_`|,=";
+    private static final String HTTP_HEADER_KEY_SAFE_CHARS = "!#$&'*-.~^_`|";
+    private static final String HTTP_HEADER_VALUE_SAFE_CHARS = "()<>@,;:\"/\\[]?={}" + HTTP_HEADER_KEY_SAFE_CHARS;
 
-    private static final Escaper HTTP_HEADER_ESCAPER =
-            new PercentEscaper(HTTP_HEADER_SAFE_CHARS, false);
+
+    private static final Escaper HTTP_HEADER_KEY_ESCAPER =
+            new PercentEscaper(HTTP_HEADER_KEY_SAFE_CHARS, false);
+
+    private static final Escaper HTTP_HEADER_VALUE_ESCAPER =
+            new PercentEscaper(HTTP_HEADER_VALUE_SAFE_CHARS, false);
 
     private MetadataStringManipulation() {
         //pass
@@ -47,15 +49,29 @@ private MetadataStringManipulation() {
 
     /**
      * Percent encodes non-alpha-numeric characters within the specified string
-     * excluding the symbols listed in {@link #HTTP_HEADER_SAFE_CHARS} using UTF-8
+     * excluding the symbols listed in {@link #HTTP_HEADER_KEY_SAFE_CHARS} using UTF-8
      */
-    public static String toEncodedString(final String str) {
+    public static String toEncodedKeyString(final String str) {
         if (str == null) {
             return null;
         }
 
         final String strUtf8 = new String(str.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8);
-        return getMetadataEscaper().escape(strUtf8);
+        return getMetadataKeyEscaper().escape(strUtf8);
+    }
+
+
+    /**
+     * Percent encodes non-alpha-numeric characters within the specified string
+     * excluding the symbols listed in {@link #HTTP_HEADER_VALUE_SAFE_CHARS} using UTF-8
+     */
+    public static String toEncodedValueString(final String str) {
+        if (str == null) {
+            return null;
+        }
+
+        final String strUtf8 = new String(str.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8);
+        return getMetadataValueEscaper().escape(strUtf8);
     }
 
     /**
@@ -73,7 +89,11 @@ public static String toDecodedString(final String str) {
         }
     }
 
-    private static Escaper getMetadataEscaper() {
-        return HTTP_HEADER_ESCAPER;
+    private static Escaper getMetadataKeyEscaper() {
+        return HTTP_HEADER_KEY_ESCAPER;
+    }
+
+    private static Escaper getMetadataValueEscaper() {
+        return HTTP_HEADER_VALUE_ESCAPER;
     }
 }
diff --git a/ds3-utils/src/test/java/com/spectralogic/ds3client/utils/MetadataStringManipulation_Test.java b/ds3-utils/src/test/java/com/spectralogic/ds3client/utils/MetadataStringManipulation_Test.java
index 01af142ae..5cace80ea 100644
--- a/ds3-utils/src/test/java/com/spectralogic/ds3client/utils/MetadataStringManipulation_Test.java
+++ b/ds3-utils/src/test/java/com/spectralogic/ds3client/utils/MetadataStringManipulation_Test.java
@@ -20,7 +20,8 @@
 import java.util.regex.Pattern;
 
 import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toDecodedString;
-import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toEncodedString;
+import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toEncodedKeyString;
+import static com.spectralogic.ds3client.utils.MetadataStringManipulation.toEncodedValueString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.junit.Assert.assertFalse;
@@ -33,107 +34,174 @@ public class MetadataStringManipulation_Test {
     private static final String STRING_WITH_SYMBOLS = "1234567890-!@#$%^&*()_+`~[]\\{}|;':\"./<>?∞πϊφϠ";
     private static final String STRING_WITH_SYMBOLS_UNICODE = "1234567890-!@#$%^&*()_+`~[]\\{}|;':\"./<>?\u03C0\u221E\u03CA\u03D5\u03E0";
 
-    private static final Pattern ENCODED_PATTERN = Pattern.compile("[a-zA-Z0-9!#$&'*\\-.~\\^_`|,=%]*");
+    private static final Pattern KEY_ENCODED_PATTERN = Pattern.compile("[a-zA-Z0-9!#$&'*-.~^_`|%]*");
+
+    private static final Pattern VALUE_ENCODED_PATTERN = Pattern.compile("[a-zA-Z0-9!#$&'*-.~^_`|%()<>@,;:\"/\\\\\\[\\]?={}]*");
 
     /*
-     * Verifies that an encoded string only contains header safe characters
+     * Verifies that an encoded string only contains key header safe characters
      */
-    private static boolean isEncodedSafeChars(final String str) {
-        return ENCODED_PATTERN.matcher(str).matches();
+    private static boolean isEncodedSafeKeyChars(final String str) {
+        return KEY_ENCODED_PATTERN.matcher(str).matches();
+    }
+
+    /*
+     * Verifies that an encoded string only contains value header safe characters
+     */
+    private static boolean isEncodedSafeValueChars(final String str) {
+        return VALUE_ENCODED_PATTERN.matcher(str).matches();
     }
 
     @Test
-    public void encodedPattern_Test() {
-        assertTrue(isEncodedSafeChars(""));
-        assertTrue(isEncodedSafeChars("abc123"));
-        assertTrue(isEncodedSafeChars("!#$&'*-.~^_`|,=%"));
-
-        assertFalse(isEncodedSafeChars(" "));
-        assertFalse(isEncodedSafeChars("{"));
-        assertFalse(isEncodedSafeChars("}"));
-        assertFalse(isEncodedSafeChars("["));
-        assertFalse(isEncodedSafeChars("]"));
-        assertFalse(isEncodedSafeChars("<"));
-        assertFalse(isEncodedSafeChars(">"));
-        assertFalse(isEncodedSafeChars("@"));
-        assertFalse(isEncodedSafeChars(";"));
-        assertFalse(isEncodedSafeChars(":"));
-        assertFalse(isEncodedSafeChars("\\"));
-        assertFalse(isEncodedSafeChars("\""));
-        assertFalse(isEncodedSafeChars("/"));
-        assertFalse(isEncodedSafeChars("?"));
-        assertFalse(isEncodedSafeChars("+"));
-
-        assertFalse(isEncodedSafeChars("\u221E"));
-        assertFalse(isEncodedSafeChars("\u03C0"));
-        assertFalse(isEncodedSafeChars("\u03CA"));
-        assertFalse(isEncodedSafeChars("\u03D5"));
-        assertFalse(isEncodedSafeChars("\u03E0"));
+    public void encodedKeyPattern_Test() {
+        assertTrue(isEncodedSafeKeyChars(""));
+        assertTrue(isEncodedSafeKeyChars("abc123"));
+        assertTrue(isEncodedSafeKeyChars("!#$&'*-.~^_`|%"));
+
+        assertFalse(isEncodedSafeKeyChars(" "));
+        assertFalse(isEncodedSafeKeyChars("{"));
+        assertFalse(isEncodedSafeKeyChars("}"));
+        assertFalse(isEncodedSafeKeyChars("["));
+        assertFalse(isEncodedSafeKeyChars("]"));
+        assertFalse(isEncodedSafeKeyChars("<"));
+        assertFalse(isEncodedSafeKeyChars(">"));
+        assertFalse(isEncodedSafeKeyChars("@"));
+        assertFalse(isEncodedSafeKeyChars(";"));
+        assertFalse(isEncodedSafeKeyChars(":"));
+        assertFalse(isEncodedSafeKeyChars("\\"));
+        assertFalse(isEncodedSafeKeyChars("\""));
+        assertFalse(isEncodedSafeKeyChars("/"));
+        assertFalse(isEncodedSafeKeyChars("?"));
+
+        assertFalse(isEncodedSafeKeyChars("\u221E"));
+        assertFalse(isEncodedSafeKeyChars("\u03C0"));
+        assertFalse(isEncodedSafeKeyChars("\u03CA"));
+        assertFalse(isEncodedSafeKeyChars("\u03D5"));
+        assertFalse(isEncodedSafeKeyChars("\u03E0"));
+    }
 
+    @Test
+    public void encodedValuePattern_Test() {
+        assertTrue(isEncodedSafeValueChars(""));
+        assertTrue(isEncodedSafeValueChars("abc123"));
+        assertTrue(isEncodedSafeValueChars("!#$&'*-.~^_`|,=%"));
+        assertTrue(isEncodedSafeValueChars("{"));
+        assertTrue(isEncodedSafeValueChars("}"));
+        assertTrue(isEncodedSafeValueChars("<"));
+        assertTrue(isEncodedSafeValueChars(">"));
+        assertTrue(isEncodedSafeValueChars("@"));
+        assertTrue(isEncodedSafeValueChars(";"));
+        assertTrue(isEncodedSafeValueChars(":"));
+        assertTrue(isEncodedSafeValueChars("\""));
+        assertTrue(isEncodedSafeValueChars("/"));
+        assertTrue(isEncodedSafeValueChars("?"));
+        assertTrue(isEncodedSafeValueChars("["));
+        assertTrue(isEncodedSafeValueChars("]"));
+        assertTrue(isEncodedSafeValueChars("\\"));
+
+        assertFalse(isEncodedSafeValueChars(" "));
+
+        assertFalse(isEncodedSafeValueChars("\u221E"));
+        assertFalse(isEncodedSafeValueChars("\u03C0"));
+        assertFalse(isEncodedSafeValueChars("\u03CA"));
+        assertFalse(isEncodedSafeValueChars("\u03D5"));
+        assertFalse(isEncodedSafeValueChars("\u03E0"));
     }
 
     @Test
     public void encodeDecode_NullString_Test() {
-        assertThat(toEncodedString(null), is(nullValue()));
+        assertThat(toEncodedKeyString(null), is(nullValue()));
+        assertThat(toEncodedValueString(null), is(nullValue()));
         assertThat(toDecodedString(null), is(nullValue()));
     }
 
     @Test
     public void encodeDecode_WithSpaces_Test() {
-        final String encoded = toEncodedString(STRING_WITH_SPACES);
-        assertTrue(isEncodedSafeChars(encoded));
-        assertThat(toDecodedString(encoded), is(STRING_WITH_SPACES));
+        final String encodedKey = toEncodedKeyString(STRING_WITH_SPACES);
+        assertTrue(isEncodedSafeKeyChars(encodedKey));
+        assertThat(toDecodedString(encodedKey), is(STRING_WITH_SPACES));
+
+        final String encodedValue = toEncodedValueString(STRING_WITH_SPACES);
+        assertTrue(isEncodedSafeValueChars(encodedValue));
+        assertThat(toDecodedString(encodedValue), is(STRING_WITH_SPACES));
     }
 
     @Test
     public void encodeDecode_WithSymbols_Test() {
-        final String encoded = toEncodedString(STRING_WITH_SYMBOLS);
-        assertTrue(isEncodedSafeChars(encoded));
-        assertThat(toDecodedString(encoded), is(STRING_WITH_SYMBOLS));
+        final String encodedKey = toEncodedKeyString(STRING_WITH_SYMBOLS);
+        assertTrue(isEncodedSafeKeyChars(encodedKey));
+        assertThat(toDecodedString(encodedKey), is(STRING_WITH_SYMBOLS));
+
+        final String encodedValue = toEncodedValueString(STRING_WITH_SYMBOLS);
+        assertTrue(isEncodedSafeValueChars(encodedValue));
+        assertThat(toDecodedString(encodedValue), is(STRING_WITH_SYMBOLS));
     }
 
     @Test
     public void encodeDecode_WithUnicodeSymbols_Test() {
-        final String encoded = toEncodedString(STRING_WITH_SYMBOLS_UNICODE);
-        assertTrue(isEncodedSafeChars(encoded));
-        assertThat(toDecodedString(encoded), is(STRING_WITH_SYMBOLS_UNICODE));
+        final String encodedKey = toEncodedKeyString(STRING_WITH_SYMBOLS_UNICODE);
+        assertTrue(isEncodedSafeKeyChars(encodedKey));
+        assertThat(toDecodedString(encodedKey), is(STRING_WITH_SYMBOLS_UNICODE));
+
+        final String encodedValue = toEncodedValueString(STRING_WITH_SYMBOLS_UNICODE);
+        assertTrue(isEncodedSafeValueChars(encodedValue));
+        assertThat(toDecodedString(encodedValue), is(STRING_WITH_SYMBOLS_UNICODE));
     }
 
     @Test
     public void encodeDecode_SpaceAndPlus_Test() {
         final String plusSpace = " +";
-        final String encoded = toEncodedString(plusSpace);
-        assertTrue(isEncodedSafeChars(encoded));
-        assertThat(toDecodedString(encoded), is(plusSpace));
+
+        final String encodedKey = toEncodedKeyString(plusSpace);
+        assertTrue(isEncodedSafeKeyChars(encodedKey));
+        assertThat(toDecodedString(encodedKey), is(plusSpace));
+
+        final String encodedValue = toEncodedValueString(plusSpace);
+        assertTrue(isEncodedSafeValueChars(encodedValue));
+        assertThat(toDecodedString(encodedValue), is(plusSpace));
     }
 
     @Test
     public void encodeDecode_Range_Test() {
         final String range = "Range=bytes=0-10,110-120";
-        final String encoded = toEncodedString(range);
-        assertTrue(isEncodedSafeChars(encoded));
 
-        assertThat(encoded, is(range));
+        final String encodedValue = toEncodedValueString(range);
+        assertTrue(isEncodedSafeValueChars(encodedValue));
+        assertThat(encodedValue, is(range));
         assertThat(toDecodedString(range), is(range));
     }
 
     @Test
     public void decodedDecode_LowerCase_Test() {
-        final String encodedToLower = toEncodedString(STRING_WITH_SYMBOLS).toLowerCase();
-        assertTrue(isEncodedSafeChars(encodedToLower));
-        assertThat(toDecodedString(encodedToLower), is(STRING_WITH_SYMBOLS));
+        final String encodedToLowerKey = toEncodedKeyString(STRING_WITH_SYMBOLS).toLowerCase();
+        assertTrue(isEncodedSafeKeyChars(encodedToLowerKey));
+        assertThat(toDecodedString(encodedToLowerKey), is(STRING_WITH_SYMBOLS));
+
+        final String encodedToLowerValue = toEncodedValueString(STRING_WITH_SYMBOLS).toLowerCase();
+        assertTrue(isEncodedSafeValueChars(encodedToLowerValue));
+        assertThat(toDecodedString(encodedToLowerValue), is(STRING_WITH_SYMBOLS));
     }
 
     @Test
     public void encodeDecode_DoubleEncoding_Test() {
-        final String encoded1 = toEncodedString(STRING_WITH_SYMBOLS);
-        final String encoded2 = toEncodedString(encoded1);
-        assertTrue(isEncodedSafeChars(encoded1));
-        assertTrue(isEncodedSafeChars(encoded2));
-
-        final String decoded1 = toDecodedString(encoded2);
-        final String decoded2 = toDecodedString(decoded1);
-        assertThat(decoded1, is(encoded1));
-        assertThat(decoded2, is(STRING_WITH_SYMBOLS));
+        final String encoded1Key = toEncodedKeyString(STRING_WITH_SYMBOLS);
+        final String encoded2Key = toEncodedKeyString(encoded1Key);
+        assertTrue(isEncodedSafeKeyChars(encoded1Key));
+        assertTrue(isEncodedSafeKeyChars(encoded2Key));
+
+        final String decoded1Key = toDecodedString(encoded2Key);
+        final String decoded2Key = toDecodedString(decoded1Key);
+        assertThat(decoded1Key, is(encoded1Key));
+        assertThat(decoded2Key, is(STRING_WITH_SYMBOLS));
+
+        final String encoded1Value = toEncodedValueString(STRING_WITH_SYMBOLS);
+        final String encoded2Value = toEncodedValueString(encoded1Value);
+        assertTrue(isEncodedSafeValueChars(encoded1Value));
+        assertTrue(isEncodedSafeValueChars(encoded2Value));
+
+        final String decoded1Value = toDecodedString(encoded2Value);
+        final String decoded2Value = toDecodedString(decoded1Value);
+        assertThat(decoded1Value, is(encoded1Value));
+        assertThat(decoded2Value, is(STRING_WITH_SYMBOLS));
     }
 }

From f925c6352414bc532ee3d737c841ca9f768541d6 Mon Sep 17 00:00:00 2001
From: Sharon Shabtai 
Date: Thu, 18 Apr 2019 14:02:32 -0600
Subject: [PATCH 172/229] cherry-pick

---
 .../metadata/MACMetadataRestore_Test.java     |  2 +-
 .../metadata/PosixMetadataRestore_Test.java   |  2 +-
 .../metadata/PosixMetadataStore_Test.java     |  8 +++---
 .../metadata/WindowsMetadataRestore_Test.java |  6 ++--
 .../metadata/WindowsMetadataStore_Test.java   | 14 +++++-----
 .../ds3client/integration/Util.java           |  2 +-
 .../ObjectStreamBuilder_Test.java             |  4 +--
 .../AdvancedBucketManagement_Test.java        | 16 +++++------
 .../integration/CustomParser_Test.java        |  4 +--
 .../integration/GroupManagement_Test.java     |  8 +++---
 .../integration/Regression_Test.java          |  2 +-
 .../ds3client/integration/Smoke_Test.java     |  4 +--
 .../integration/UsersAndGroups_Test.java      |  8 +++---
 .../ds3client/Ds3Client_Test.java             |  2 +-
 .../interfaces/RequestHeadersImpl_Test.java   | 28 +++++++++----------
 .../utils/ResponseParserUtils_Test.java       |  6 ++--
 .../spectrads3/GetPutJobToReplicate_Test.java |  2 +-
 .../helpers/JobPartTracker_Test.java          |  2 +-
 .../models/ModelFunctionality_Test.java       |  4 +--
 .../ds3client/models/ModelParsing_Test.java   |  8 +++---
 .../ds3client/models/ServiceList_Test.java    |  2 +-
 .../FailedRequestException_Test.java          | 14 +++++-----
 .../utils/SafeString_Manipulation_Test.java   | 20 ++++++-------
 .../MetadataStringManipulation_Test.java      | 20 ++++++-------
 24 files changed, 94 insertions(+), 94 deletions(-)

diff --git a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/MACMetadataRestore_Test.java b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/MACMetadataRestore_Test.java
index 9b0645bd6..29fb28c0f 100644
--- a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/MACMetadataRestore_Test.java
+++ b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/MACMetadataRestore_Test.java
@@ -41,7 +41,7 @@ public class MACMetadataRestore_Test {
     private final File file = new File(getClass().getClassLoader().getResource("LoremIpsum.txt").getFile());
 
     @Test
-    public void restoreFileTimes_Test() throws Exception {
+    public void restoreFileTimesTest() throws Exception {
 
         if (Platform.isMac()) {
             final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MM-yy:HH:mm");
diff --git a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/PosixMetadataRestore_Test.java b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/PosixMetadataRestore_Test.java
index 07ab02d07..611095302 100644
--- a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/PosixMetadataRestore_Test.java
+++ b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/PosixMetadataRestore_Test.java
@@ -53,7 +53,7 @@ public void checkPreconditions() {
     private final File file  = new File(getClass().getClassLoader().getResource("LoremIpsum.txt").getFile());
 
     @Test
-    public  void restoreFileTimes_Test() throws Exception{
+    public  void restoreFileTimesTest() throws Exception{
         final BasicHeader basicHeader[] = new BasicHeader[3];
         final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
         basicHeader[0] = new BasicHeader(MetadataKeyConstants.METADATA_PREFIX + MetadataKeyConstants.KEY_CREATION_TIME, String.valueOf(attr.creationTime().toMillis()));
diff --git a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/PosixMetadataStore_Test.java b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/PosixMetadataStore_Test.java
index dd2a25af9..edd05ff67 100644
--- a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/PosixMetadataStore_Test.java
+++ b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/PosixMetadataStore_Test.java
@@ -38,7 +38,7 @@ public class PosixMetadataStore_Test {
     private final PosixMetadataStore posixMetadataStore = new PosixMetadataStore(mMetadataMap);
 
     @Test
-    public void saveCreationTimeMetaData_Test() throws IOException {
+    public void saveCreationTimeMetaDataTest() throws IOException {
         final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
         final long creationTime = attr.creationTime().toMillis();
         posixMetadataStore.saveCreationTimeMetaData(attr);
@@ -46,7 +46,7 @@ public void saveCreationTimeMetaData_Test() throws IOException {
     }
 
     @Test
-    public void saveAccessTimeMetaData_Test() throws IOException {
+    public void saveAccessTimeMetaDataTest() throws IOException {
         final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
         final long lastAccessTime = attr.lastAccessTime().toMillis();
         posixMetadataStore.saveAccessTimeMetaData(attr);
@@ -55,7 +55,7 @@ public void saveAccessTimeMetaData_Test() throws IOException {
 
 
     @Test
-    public void saveLastModifiedTime_Test() throws IOException {
+    public void saveLastModifiedTimeTest() throws IOException {
         final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
         final long lastModifiedTime = attr.lastModifiedTime().toMillis();
         posixMetadataStore.saveLastModifiedTime(attr);
@@ -63,7 +63,7 @@ public void saveLastModifiedTime_Test() throws IOException {
     }
 
     @Test
-    public void saveOSMetaData_Test() throws IOException {
+    public void saveOSMetaDataTest() throws IOException {
         final String localOSName = MetaDataUtil.getOS();
         posixMetadataStore.saveOSMetaData(localOSName);
         Assert.assertEquals(mMetadataMap.build().get(MetadataKeyConstants.METADATA_PREFIX + MetadataKeyConstants.KEY_OS), String.valueOf(localOSName));
diff --git a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore_Test.java b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore_Test.java
index 3da51176b..5759b9612 100644
--- a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore_Test.java
+++ b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore_Test.java
@@ -56,7 +56,7 @@ public void checkPreconditions() {
     }
     
     @Test
-    public void restoreFileTimes_Test() throws Exception {
+    public void restoreFileTimesTest() throws Exception {
         final BasicHeader basicHeader[] = new BasicHeader[3];
 
         final Path filePath = ResourceUtils.loadFileResource(FILE_NAME);
@@ -75,7 +75,7 @@ public void restoreFileTimes_Test() throws Exception {
 
 
     @Test
-    public void restoreUserAndOwner_Test() throws Exception {
+    public void restoreUserAndOwnerTest() throws Exception {
         final ImmutableMap.Builder metadataMap = new ImmutableMap.Builder<>();
         final WindowsMetadataStore windowsMetadataStore = new WindowsMetadataStore(metadataMap);
         final Class aClass = WindowsMetadataStore.class;
@@ -117,7 +117,7 @@ public void restoreUserAndOwner_Test() throws Exception {
 
 
     @Test
-    public void restorePermissions_Test() throws NoSuchMethodException, IOException, URISyntaxException, InvocationTargetException, IllegalAccessException, InterruptedException {
+    public void restorePermissionsTest() throws NoSuchMethodException, IOException, URISyntaxException, InvocationTargetException, IllegalAccessException, InterruptedException {
         final ImmutableMap.Builder mMetadataMap = new ImmutableMap.Builder<>();
         final WindowsMetadataStore windowsMetadataStore = new WindowsMetadataStore(mMetadataMap);
         final Class aClass = WindowsMetadataStore.class;
diff --git a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/WindowsMetadataStore_Test.java b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/WindowsMetadataStore_Test.java
index e78f90df3..935ea155e 100644
--- a/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/WindowsMetadataStore_Test.java
+++ b/ds3-metadata/src/test/java/com/spectralogic/ds3client/metadata/WindowsMetadataStore_Test.java
@@ -38,7 +38,7 @@ public class WindowsMetadataStore_Test {
     private final WindowsMetadataStore windowsMetadataStore = new WindowsMetadataStore(mMetadataMap);
 
     @Test
-    public void saveCreationTimeMetaData_Test() throws IOException {
+    public void saveCreationTimeMetaDataTest() throws IOException {
         final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
         final long creationTime = attr.creationTime().toMillis();
         windowsMetadataStore.saveCreationTimeMetaData(attr);
@@ -46,7 +46,7 @@ public void saveCreationTimeMetaData_Test() throws IOException {
     }
 
     @Test
-    public void saveAccessTimeMetaData_Test() throws IOException {
+    public void saveAccessTimeMetaDataTest() throws IOException {
         final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
         final long lastAccessTime = attr.lastAccessTime().toMillis();
         windowsMetadataStore.saveAccessTimeMetaData(attr);
@@ -55,7 +55,7 @@ public void saveAccessTimeMetaData_Test() throws IOException {
 
 
     @Test
-    public void saveLastModifiedTime_Test() throws IOException {
+    public void saveLastModifiedTimeTest() throws IOException {
         final BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
         final long lastModifiedTime = attr.lastModifiedTime().toMillis();
         windowsMetadataStore.saveLastModifiedTime(attr);
@@ -63,14 +63,14 @@ public void saveLastModifiedTime_Test() throws IOException {
     }
 
     @Test
-    public void saveOSMetaData_Test() throws IOException {
+    public void saveOSMetaDataTest() throws IOException {
         final String localOSName = MetaDataUtil.getOS();
         windowsMetadataStore.saveOSMetaData(localOSName);
         Assert.assertEquals(mMetadataMap.build().get(METADATA_PREFIX + MetadataKeyConstants.KEY_OS), String.valueOf(localOSName));
     }
 
     @Test
-    public void saveWinDescriptor_Test() throws Exception {
+    public void saveWinDescriptorTest() throws Exception {
         if (MetaDataUtil.getOS().contains("Windows")) {
             final Class aClass = WindowsMetadataStore.class;
             final Method method = aClass.getDeclaredMethod("saveWindowsDescriptors", new Class[]{Path.class});
@@ -83,7 +83,7 @@ public void saveWinDescriptor_Test() throws Exception {
     }
 
     @Test
-    public void saveFlagMetadata_Test() throws Exception {
+    public void saveFlagMetadataTest() throws Exception {
         if (MetaDataUtil.getOS().contains("Windows")) {
             final Class aClass = WindowsMetadataStore.class;
             final Method method = aClass.getDeclaredMethod("saveFlagMetaData", new Class[]{Path.class});
@@ -94,7 +94,7 @@ public void saveFlagMetadata_Test() throws Exception {
     }
 
     @Test
-    public void saveWindowsfilePermissions_Test() throws Exception {
+    public void saveWindowsFilePermissionsTest() throws Exception {
         if (MetaDataUtil.getOS().contains("Windows")) {
             final Class aClass = WindowsMetadataStore.class;
             final Method method = aClass.getDeclaredMethod("saveWindowsfilePermissions", new Class[]{Path.class});
diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
index 540a06c86..90159f686 100644
--- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
+++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java
@@ -79,7 +79,7 @@ public static void forceCleanupBucketsOnBP(final Ds3Client client) throws IOExce
         }
     }
 
-    public static void assumeVersion1_2(final Ds3Client client) throws IOException {
+    public static void assumeVersionOneDotTwo(final Ds3Client client) throws IOException {
         final int majorVersion = Integer.parseInt(client.getSystemInformationSpectraS3(
                 new GetSystemInformationSpectraS3Request()).getSystemInformationResult().getBuildInformation().getVersion().split("\\.")[0]);
         assumeThat(majorVersion, is(1));
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/ObjectStreamBuilder_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/ObjectStreamBuilder_Test.java
index 90949bbac..ef04647e4 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/ObjectStreamBuilder_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/ObjectStreamBuilder_Test.java
@@ -82,7 +82,7 @@ public void afterRunningTestMethod() throws IOException {
     }
 
     @Test
-    public void objectInputStreamBuilder_Test() throws IOException, URISyntaxException {
+    public void objectInputStreamBuilderTest() throws IOException, URISyntaxException {
         //Verify that the object was transferred correctly to the BP in beforeRunningTestMethod
         final GetObjectDetailsSpectraS3Request objectDetailsRequest =
                 new GetObjectDetailsSpectraS3Request(OBJ_NAME, BUCKET_NAME);
@@ -93,7 +93,7 @@ public void objectInputStreamBuilder_Test() throws IOException, URISyntaxExcepti
     }
 
     @Test
-    public void objectOutputStreamBuilder_Test() throws IOException, URISyntaxException {
+    public void objectOutputStreamBuilderTest() throws IOException, URISyntaxException {
         //Retrieve the object on the BP using ObjectOutputStreamBuilder and verify contents
         final Ds3Object obj = new Ds3Object(OBJ_NAME, OBJ_BYTES.length);
         final Ds3ClientHelpers.Job job = HELPERS.startReadJob(BUCKET_NAME, Lists.newArrayList(obj));
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/AdvancedBucketManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/AdvancedBucketManagement_Test.java
index 13f4392f1..c1fc7a0a2 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/AdvancedBucketManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/AdvancedBucketManagement_Test.java
@@ -60,7 +60,7 @@ public static void teardown() throws IOException, SignatureException {
     }
 
     @Test
-    public void createDeleteDataPolicy_Test() throws IOException, SignatureException {
+    public void createDeleteDataPolicyTest() throws IOException, SignatureException {
         final String dataPolicyName = "ds3_java_sdk_create_delete_data_policy";
 
         try {
@@ -86,7 +86,7 @@ public void createDeleteDataPolicy_Test() throws IOException, SignatureException
     }
 
     @Test
-    public void createDeleteEmptyStorageDomain_Test() throws IOException, SignatureException {
+    public void createDeleteEmptyStorageDomainTest() throws IOException, SignatureException {
         final String storageDomainName = "create_delete_empty_storage_domain";
 
         try {
@@ -104,7 +104,7 @@ public void createDeleteEmptyStorageDomain_Test() throws IOException, SignatureE
     }
 
     @Test
-    public void createDeletePoolPartition_Test() throws IOException, SignatureException {
+    public void createDeletePoolPartitionTest() throws IOException, SignatureException {
         final String poolPartitionName = "create_delete_pool_partition";
 
         try {
@@ -130,7 +130,7 @@ public void createDeletePoolPartition_Test() throws IOException, SignatureExcept
     }
 
     @Test
-    public void createPoolStorageDomainMember_Test() throws IOException, SignatureException {
+    public void createPoolStorageDomainMemberTest() throws IOException, SignatureException {
         final String storageDomainName = "create_pool_storage_domain_member_sd";
         final String poolPartitionName = "create_pool_storage_domain_member_pp";
         UUID storageDomainMemberId = null;
@@ -172,7 +172,7 @@ public void createPoolStorageDomainMember_Test() throws IOException, SignatureEx
     }
 
     @Test
-    public void createBucketSpectraS3_Test() throws IOException, SignatureException {
+    public void createBucketSpectraS3Test() throws IOException, SignatureException {
         final String bucketName = "create_bucket_spectra_s3";
         try {
             final PutBucketSpectraS3Response response = client
@@ -186,7 +186,7 @@ public void createBucketSpectraS3_Test() throws IOException, SignatureException
     }
 
     @Test
-    public void createBucketSpectraS3_WithDataPolicy_Test() throws IOException, SignatureException {
+    public void createBucketSpectraS3WithDataPolicyTest() throws IOException, SignatureException {
         final String bucketName = "create_bucket_spectra_s3";
         final String dataPolicyName = "create_bucket_spectra_s3_data_policy";
         final String storageDomainName = "create_bucket_spectra_s3_storage_domain";
@@ -244,7 +244,7 @@ public void createBucketSpectraS3_WithDataPolicy_Test() throws IOException, Sign
     }
 
     @Test
-    public void putDuplicateObjects_VersioningKeepLatest_Test() throws IOException,
+    public void putDuplicateObjectsVersioningKeepLatestTest() throws IOException,
             SignatureException, URISyntaxException {
         final String bucketName = "duplicate_object_versioning_keep_latest";
         final String dataPolicyName = "duplicate_object_versioning_keep_latest_dp";
@@ -316,7 +316,7 @@ public void putDuplicateObjects_VersioningKeepLatest_Test() throws IOException,
     }
 
     @Test
-    public void putDuplicateObjects_VersioningNone_Test() throws IOException,
+    public void putDuplicateObjectsVersioningNoneTest() throws IOException,
             SignatureException, URISyntaxException {
         final String bucketName = "duplicate_object_versioning_none";
         final String dataPolicyName = "duplicate_object_versioning_none_dp";
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/CustomParser_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/CustomParser_Test.java
index 8323abcf5..0cacf8739 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/CustomParser_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/CustomParser_Test.java
@@ -95,7 +95,7 @@ public void afterRunningTestMethod() throws IOException {
     }
 
     @Test
-    public void getObjectCustomParser_Test() throws IOException, URISyntaxException {
+    public void getObjectCustomParserTest() throws IOException, URISyntaxException {
         final Ds3Object object = new Ds3Object(OBJ_NAME);
         final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client
                 .getBulkJobSpectraS3(new GetBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(object)));
@@ -137,7 +137,7 @@ public GetObjectResponse apply(final GetObjectCustomParserParameters getObjectPa
 
     @SuppressWarnings("deprecation")
     @Test (expected = FailedRequestException.class)
-    public void getObjectCustomParserNoObject_Test() throws IOException{
+    public void getObjectCustomParserNoObjectTest() throws IOException{
         final String objectName = "doesNotExistObject";
 
         final GetObjectRequest request = new GetObjectRequest(
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GroupManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GroupManagement_Test.java
index b955d9c4d..100928fe5 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GroupManagement_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GroupManagement_Test.java
@@ -58,7 +58,7 @@ public static void teardown() throws IOException, SignatureException {
     }
 
     @Test
-    public void createDeleteGroup_Test() throws IOException, SignatureException {
+    public void createDeleteGroupTest() throws IOException, SignatureException {
         final String groupName = "create_delete_group_sdk_test";
 
         try {
@@ -77,7 +77,7 @@ public void createDeleteGroup_Test() throws IOException, SignatureException {
     }
 
     @Test
-    public void createDataPolicyForGroup_Test() throws IOException, SignatureException {
+    public void createDataPolicyForGroupTest() throws IOException, SignatureException {
         final String groupName = "create_data_policy_for_group_g";
         final String dataPolicyName = "create_data_policy_for_group_dp";
         UUID aclId = null;
@@ -118,7 +118,7 @@ public void createDataPolicyForGroup_Test() throws IOException, SignatureExcepti
     }
 
     @Test
-    public void createGroupGroupMember_Test() throws IOException, SignatureException {
+    public void createGroupGroupMemberTest() throws IOException, SignatureException {
         final String parentGroupName = "create_group_group_member_parent";
         final String childGroupName = "create_group_group_member_child";
 
@@ -159,7 +159,7 @@ public void createGroupGroupMember_Test() throws IOException, SignatureException
     }
 
     @Test
-    public void createBucketAclForGroup_Test() throws IOException, SignatureException {
+    public void createBucketAclForGroupTest() throws IOException, SignatureException {
         final String bucketName = "create_bucket_acl_for_group_b";
         final String groupName = "create_bucket_acl_for_group_g";
 
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Regression_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Regression_Test.java
index d0823a147..a5fcb0045 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Regression_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Regression_Test.java
@@ -247,7 +247,7 @@ public void testPrefixForNestedDirectories() throws IOException {
 
     @Test
     public void emptyObjectTest() throws IOException {
-        Util.assumeVersion1_2(client);
+        Util.assumeVersionOneDotTwo(client);
         final String bucketName = "emptyObject";
         final List objects = Collections.singletonList(new Ds3Object("obj1.txt", 0));
 
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
index 8849ed328..858ce3642 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java
@@ -935,7 +935,7 @@ public void testRecoverReadJobDoesNotExist() throws IOException, JobRecoveryExce
 
     @Test
     public void putDirectory() throws IOException {
-        assumeVersion1_2(client);
+        assumeVersionOneDotTwo(client);
 
         final String bucketName = "putDir";
 
@@ -967,7 +967,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException {
 
     @Test
     public void putDirectoryWithOtherObjects() throws IOException {
-        assumeVersion1_2(client);
+        assumeVersionOneDotTwo(client);
 
         final String bucketName = "mixedPutDir";
 
diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
index 4ef77f7e7..4277dd584 100644
--- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
+++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/UsersAndGroups_Test.java
@@ -177,7 +177,7 @@ public void getUsersWithCombination() throws IOException, SignatureException {
     }
 
     @Test
-    public void ModifyUserDefaultDataPolicy() throws IOException, SignatureException {
+    public void modifyUserDefaultDataPolicy() throws IOException, SignatureException {
 
         final GetUserSpectraS3Response getSpectraResponse = client
                 .getUserSpectraS3(new GetUserSpectraS3Request(spectraUUID));
@@ -192,7 +192,7 @@ public void ModifyUserDefaultDataPolicy() throws IOException, SignatureException
     }
 
     @Test
-    public void ModifyUserName() throws IOException, SignatureException {
+    public void modifyUserName() throws IOException, SignatureException {
 
         final GetUserSpectraS3Response getSpectraResponse = client
                 .getUserSpectraS3(new GetUserSpectraS3Request(spectraUUID));
@@ -207,7 +207,7 @@ public void ModifyUserName() throws IOException, SignatureException {
 
 
     @Test
-    public void ModifyUserNameAndDataPolicy() throws IOException, SignatureException {
+    public void modifyUserNameAndDataPolicy() throws IOException, SignatureException {
 
         final GetUserSpectraS3Response getSpectraResponse = client
                 .getUserSpectraS3(new GetUserSpectraS3Request(spectraUUID));
@@ -221,7 +221,7 @@ public void ModifyUserNameAndDataPolicy() throws IOException, SignatureException
     }
 
     @Test
-    public void RegenerateUserSecretKey() throws IOException, SignatureException {
+    public void megenerateUserSecretKey() throws IOException, SignatureException {
         try {
             client.regenerateUserSecretKeySpectraS3(
                     new RegenerateUserSecretKeySpectraS3Request(UUID.randomUUID()));
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
index b075bc0da..4107e6073 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java
@@ -964,7 +964,7 @@ public void testGettingGitCommitHash() {
     }
 
     @Test
-    public void VerifySystemHealthSpectraS3() throws IOException {
+    public void verifySystemHealthSpectraS3() throws IOException {
         final String responsePayload = "0";
 
         final VerifySystemHealthSpectraS3Response response = MockNetwork
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/interfaces/RequestHeadersImpl_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/interfaces/RequestHeadersImpl_Test.java
index 729ff1129..ed60eeb2a 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/interfaces/RequestHeadersImpl_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/interfaces/RequestHeadersImpl_Test.java
@@ -36,25 +36,25 @@ private static RequestHeaders getTestRequestHeaders() {
     }
 
     @Test (expected = NullPointerException.class)
-    public void put_Null_Test() {
+    public void putNullTest() {
         final RequestHeaders rh = getTestRequestHeaders();
         rh.put(null, null);
     }
 
     @Test (expected = NullPointerException.class)
-    public void put_NullValue_Test() {
+    public void putNullValueTest() {
         final RequestHeaders rh = getTestRequestHeaders();
         rh.put("Key", null);
     }
 
     @Test (expected = NullPointerException.class)
-    public void put_NullKey_Test() {
+    public void putNullKeyTest() {
         final RequestHeaders rh = getTestRequestHeaders();
         rh.put(null, "Value");
     }
 
     @Test
-    public void put_PercentEncoding_Test() {
+    public void putPercentEncodingTest() {
         final RequestHeaders rh = getTestRequestHeaders();
         rh.put("Key With Space", "Val With Space");
         final Multimap result = rh.getMultimap();
@@ -62,13 +62,13 @@ public void put_PercentEncoding_Test() {
     }
 
     @Test (expected = NullPointerException.class)
-    public void get_Null_Test() {
+    public void getNullTest() {
         final RequestHeaders rh = getTestRequestHeaders();
         rh.get(null);
     }
 
     @Test
-    public void get_Test() {
+    public void getTest() {
         final RequestHeaders rh = getTestRequestHeaders();
         assertThat(rh.get("").size(), is(0));
         assertThat(rh.get("Key One").size(), is(2));
@@ -77,17 +77,17 @@ public void get_Test() {
     }
 
     @Test
-    public void size_Test() {
+    public void sizeTest() {
         assertThat(getTestRequestHeaders().size(), is(3));
     }
 
     @Test (expected = NullPointerException.class)
-    public void containsKey_NullString_Test() {
+    public void containsKeyNullStringTest() {
         assertFalse(getTestRequestHeaders().containsKey(null));
     }
 
     @Test
-    public void containsKey_Test() {
+    public void containsKeyTest() {
         final RequestHeaders rh = getTestRequestHeaders();
 
         System.out.println(rh.keySet());
@@ -100,13 +100,13 @@ public void containsKey_Test() {
     }
 
     @Test (expected = NullPointerException.class)
-    public void removeAll_Null_Test() {
+    public void removeAllNullTest() {
         final RequestHeaders rh = getTestRequestHeaders();
         assertThat(rh.removeAll(null).size(), is(0));
     }
 
     @Test
-    public void removeAll_Test() {
+    public void removeAllTest() {
         final RequestHeaders rh = getTestRequestHeaders();
 
         assertThat(rh.removeAll("").size(), is(0));
@@ -123,7 +123,7 @@ public void removeAll_Test() {
     }
 
     @Test
-    public void entries_Test() {
+    public void entriesTest() {
         final Collection> result = getTestRequestHeaders().entries();
         assertThat(result.size(), is(3));
 
@@ -142,7 +142,7 @@ public void entries_Test() {
     }
 
     @Test
-    public void getMultimap_Test() {
+    public void getMultimapTest() {
         final Multimap result = getTestRequestHeaders().getMultimap();
         assertThat(result.size(), is(3));
 
@@ -152,7 +152,7 @@ public void getMultimap_Test() {
     }
 
     @Test
-    public void keySet_Test() {
+    public void keySetTest() {
         final Set result = getTestRequestHeaders().keySet();
         assertThat(result.size(), is(2));
         assertThat(result, hasItem("Key One"));
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
index 7021d785a..1dc1c8dd9 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java
@@ -36,7 +36,7 @@ public class ResponseParserUtils_Test {
     private static final Logger LOG = LoggerFactory.getLogger(ResponseParserUtils_Test.class);
 
     @Test
-    public void getRequestId_Test() {
+    public void getRequestIdTest() {
         final String requestId = "123";
         final ImmutableMap headers = ImmutableMap.of(
                 "x-amz-request-id", requestId,
@@ -47,7 +47,7 @@ public void getRequestId_Test() {
     }
 
     @Test
-    public void getRequestId_WithoutRequestId_Test() {
+    public void getRequestIdWithoutRequestIdTest() {
         final ImmutableMap headers = ImmutableMap.of(
                 "Content-Type", "Text/xml");
 
@@ -56,7 +56,7 @@ public void getRequestId_WithoutRequestId_Test() {
     }
 
     @Test
-    public void getRequestId_EmptyHeaders_Test() {
+    public void getRequestIdEmptyHeadersTest() {
         final String result = getRequestId(new MockedHeaders(ImmutableMap.of()));
         assertThat(result, is(nullValue()));
     }
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/spectrads3/GetPutJobToReplicate_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/spectrads3/GetPutJobToReplicate_Test.java
index 8f44b380e..bf2801851 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/spectrads3/GetPutJobToReplicate_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/spectrads3/GetPutJobToReplicate_Test.java
@@ -28,7 +28,7 @@
 public class GetPutJobToReplicate_Test {
 
     @Test
-    public void getPutJobToReplicate_ProcessResponse_Test() throws IOException {
+    public void getPutJobToReplicateProcessResponseTest() throws IOException {
         final String responsePayload = "Some response payload";
         final ImmutableMap emptyMap = ImmutableMap.of();
         final WebResponse webResponse = new MockedWebResponse(responsePayload, 200, emptyMap);
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java
index 76a2b5cca..804aadffa 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java
@@ -54,7 +54,7 @@ public void trackerCallsTrackers() {
     }
 
     @Test
-    public void TrackerEventsForward()
+    public void trackerEventsForward()
     {
         final MockObjectPartTracker fooTracker = new MockObjectPartTracker("foo", Arrays.asList(false, true));
         final MockObjectPartTracker barTracker = new MockObjectPartTracker("bar", Arrays.asList(true, false));
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelFunctionality_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelFunctionality_Test.java
index 993c7021f..3b2d5d056 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelFunctionality_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelFunctionality_Test.java
@@ -23,7 +23,7 @@
 public class ModelFunctionality_Test {
 
     @Test
-    public void bulkObjectEqualsEmptyObjects_Test() {
+    public void bulkObjectEqualsEmptyObjectsTest() {
         final BulkObject obj1 = new BulkObject();
         final BulkObject obj2 = new BulkObject();
 
@@ -31,7 +31,7 @@ public void bulkObjectEqualsEmptyObjects_Test() {
     }
 
     @Test
-    public void bulkObjectNullableEquals_Test() {
+    public void bulkObjectNullableEqualsTest() {
         assertThat(BulkObject.nullableEquals(null, null), is(true));
         assertThat(BulkObject.nullableEquals("1", null), is(false));
         assertThat(BulkObject.nullableEquals("1", "1"), is(true));
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java
index 371395bba..2ae92ae96 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java
@@ -30,7 +30,7 @@
 public class ModelParsing_Test {
 
     @Test
-    public void listBucketResult_Parse_Test() throws IOException {
+    public void listBucketResultParseTest() throws IOException {
         final String input = "testmovies/movie1.mov" +
                 "jason04531ac9-6639-4bee-8c09-9c8f0fbdbdcb" +
                 "0movies/movie2.mov" +
@@ -49,7 +49,7 @@ public void listBucketResult_Parse_Test() throws IOException {
     }
 
     @Test
-    public void listBucketResult_CommonPrefixes_Test() throws IOException {
+    public void listBucketResultCommonPrefixesTest() throws IOException {
         final String input = "" +
                 "movies/" +
                 "scores/" +
@@ -68,7 +68,7 @@ public void listBucketResult_CommonPrefixes_Test() throws IOException {
     }
 
     @Test
-    public void completeMultipartUpload_Parse_Test() throws IOException {
+    public void completeMultipartUploadParseTest() throws IOException {
         final String input = "" +
                 "1a54357aff0632cce46d942af68356b38" +
                 "20c78aef83f66abc1fa1e8477f296d394" +
@@ -86,7 +86,7 @@ public void completeMultipartUpload_Parse_Test() throws IOException {
     }
 
     @Test
-    public void completeMultipartUpload_ToString_Test() {
+    public void completeMultipartUploadToStringTest() {
         final String expected = "" +
                 "1a54357aff0632cce46d942af68356b38" +
                 "20c78aef83f66abc1fa1e8477f296d394" +
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ServiceList_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ServiceList_Test.java
index 5a7f4e58b..a177235af 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ServiceList_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ServiceList_Test.java
@@ -28,7 +28,7 @@
 public class ServiceList_Test {
 
     @Test
-    public void service_list_serialization() throws IOException {
+    public void serviceListSerialization() throws IOException {
         final UUID id = UUID.randomUUID();
         final String stringResponse = "\n" +
                 "" + id.toString() + "ryantestBucket22013-12-11T23:20:09bulkTest2013-12-11T23:20:09bulkTest12013-12-11T23:20:09bulkTest22013-12-11T23:20:09bulkTest32013-12-11T23:20:09bulkTest42013-12-11T23:20:09bulkTest52013-12-11T23:20:09bulkTest62013-12-11T23:20:09testBucket32013-12-11T23:20:09testBucket12013-12-11T23:20:09testbucket2013-12-11T23:20:09";
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/networking/FailedRequestException_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/networking/FailedRequestException_Test.java
index fa25268c3..d06866810 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/networking/FailedRequestException_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/networking/FailedRequestException_Test.java
@@ -33,46 +33,46 @@ private static Error createTestError() {
     }
 
     @Test
-    public void buildRequestIdMessage_Test() {
+    public void buildRequestIdMessageTest() {
         final String result = buildRequestIdMessage("123");
         assertThat(result, is("for request #123"));
     }
 
     @Test
-    public void buildRequestIdMessage_EmptyId_Test() {
+    public void buildRequestIdMessageEmptyIdTest() {
         final String result = buildRequestIdMessage("");
         assertThat(result, is("for unknown request"));
     }
 
     @Test
-    public void buildRequestIdMessage_NullId_Test() {
+    public void buildRequestIdMessageNullIdTest() {
         final String result = buildRequestIdMessage(null);
         assertThat(result, is("for unknown request"));
     }
 
     @Test
-    public void buildExceptionMessage_NullErrorAndRequestId_Test() {
+    public void buildExceptionMessageNullErrorAndRequestIdTest() {
         final String expected = "Expected a status code of 200, 203 but got 400 for unknown request. Could not parse the response for additional information.";
         final String result = buildExceptionMessage(null, ImmutableList.of(200, 203), 400, null);
         assertThat(result, is(expected));
     }
 
     @Test
-    public void buildExceptionMessage_NullError_Test() {
+    public void buildExceptionMessageNullErrorTest() {
         final String expected = "Expected a status code of 200, 203 but got 400 for request #123. Could not parse the response for additional information.";
         final String result = buildExceptionMessage(null, ImmutableList.of(200, 203), 400, "123");
         assertThat(result, is(expected));
     }
 
     @Test
-    public void buildExceptionMessage_NullRequestId_Test() {
+    public void buildExceptionMessageNullRequestIdTest() {
         final String expected = "Expected a status code of 200, 203 but got 400 for unknown request. Error message: \"Error message\"";
         final String result = buildExceptionMessage(createTestError(), ImmutableList.of(200, 203), 400, null);
         assertThat(result, is(expected));
     }
 
     @Test
-    public void buildExceptionMessage_Test() {
+    public void buildExceptionMessageTest() {
         final String expected = "Expected a status code of 200, 203 but got 400 for request #123. Error message: \"Error message\"";
         final String result = buildExceptionMessage(createTestError(), ImmutableList.of(200, 203), 400, "123");
         assertThat(result, is(expected));
diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/SafeString_Manipulation_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/SafeString_Manipulation_Test.java
index 0fee2caa4..0441372c6 100644
--- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/SafeString_Manipulation_Test.java
+++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/SafeString_Manipulation_Test.java
@@ -26,13 +26,13 @@
 public class SafeString_Manipulation_Test {
 
     @Test
-    public void safeToString_Int_Test() {
+    public void safeToStringIntTest() {
         final int myInt = 5;
         assertThat(SafeStringManipulation.safeToString(myInt), is("5"));
     }
 
     @Test
-    public void safeToString_Integer_Test() {
+    public void safeToStringIntegerTest() {
         final Integer nullInt = null;
         assertThat(SafeStringManipulation.safeToString(nullInt), is(nullValue()));
 
@@ -41,13 +41,13 @@ public void safeToString_Integer_Test() {
     }
 
     @Test
-    public void safeToString_Boolean_Test() {
+    public void safeToStringBooleanTest() {
         final boolean myBoolean = true;
         assertThat(SafeStringManipulation.safeToString(myBoolean), is("true"));
     }
 
     @Test
-    public void safeToString_Double_Test() {
+    public void safeToStringDoubleTest() {
         final double myDouble = 2.1;
         assertThat(SafeStringManipulation.safeToString(myDouble), is("2.1"));
 
@@ -59,7 +59,7 @@ public void safeToString_Double_Test() {
     }
 
     @Test
-    public void safeToString_Long_Test() {
+    public void safeToStringLongTest() {
         final long myLong = 3;
         assertThat(SafeStringManipulation.safeToString(myLong), is("3"));
 
@@ -71,7 +71,7 @@ public void safeToString_Long_Test() {
     }
 
     @Test
-    public void safeToString_String_Test() {
+    public void safeToStringStringTest() {
         final String nullString = null;
         assertThat(SafeStringManipulation.safeToString(nullString), is(nullValue()));
 
@@ -80,7 +80,7 @@ public void safeToString_String_Test() {
     }
 
     @Test
-    public void safeToString_Date_Test() {
+    public void safeToStringDateTest() {
         final Date nullDate = null;
         assertThat(SafeStringManipulation.safeToString(nullDate), is(nullValue()));
 
@@ -93,7 +93,7 @@ private enum TestEnum {
     }
 
     @Test
-    public void safeToString_Enum_Test() {
+    public void safeToStringEnumTest() {
         final TestEnum nullEnum = null;
         assertThat(SafeStringManipulation.safeToString(nullEnum), is(nullValue()));
 
@@ -102,7 +102,7 @@ public void safeToString_Enum_Test() {
     }
 
     @Test
-    public void safeUrlEscape_Test() {
+    public void safeUrlEscapeTest() {
         assertThat(SafeStringManipulation.safeUrlEscape(null), is(nullValue()));
         assertThat(SafeStringManipulation.safeUrlEscape(""), is(""));
         assertThat(SafeStringManipulation.safeUrlEscape("one2three+four"), is("one2three%2Bfour"));
@@ -110,7 +110,7 @@ public void safeUrlEscape_Test() {
     }
 
     @Test
-    public void safeQueryParamEscape_Test() {
+    public void safeQueryParamEscapeTest() {
         assertThat(SafeStringManipulation.safeQueryParamEscape(null), is(nullValue()));
         assertThat(SafeStringManipulation.safeQueryParamEscape(""), is(""));
         assertThat(SafeStringManipulation.safeQueryParamEscape("one2three+four"), is("one2three%2Bfour"));
diff --git a/ds3-utils/src/test/java/com/spectralogic/ds3client/utils/MetadataStringManipulation_Test.java b/ds3-utils/src/test/java/com/spectralogic/ds3client/utils/MetadataStringManipulation_Test.java
index 5cace80ea..5c24443c3 100644
--- a/ds3-utils/src/test/java/com/spectralogic/ds3client/utils/MetadataStringManipulation_Test.java
+++ b/ds3-utils/src/test/java/com/spectralogic/ds3client/utils/MetadataStringManipulation_Test.java
@@ -53,7 +53,7 @@ private static boolean isEncodedSafeValueChars(final String str) {
     }
 
     @Test
-    public void encodedKeyPattern_Test() {
+    public void encodedKeyPatternTest() {
         assertTrue(isEncodedSafeKeyChars(""));
         assertTrue(isEncodedSafeKeyChars("abc123"));
         assertTrue(isEncodedSafeKeyChars("!#$&'*-.~^_`|%"));
@@ -81,7 +81,7 @@ public void encodedKeyPattern_Test() {
     }
 
     @Test
-    public void encodedValuePattern_Test() {
+    public void encodedValuePatternTest() {
         assertTrue(isEncodedSafeValueChars(""));
         assertTrue(isEncodedSafeValueChars("abc123"));
         assertTrue(isEncodedSafeValueChars("!#$&'*-.~^_`|,=%"));
@@ -109,14 +109,14 @@ public void encodedValuePattern_Test() {
     }
 
     @Test
-    public void encodeDecode_NullString_Test() {
+    public void encodeDecodeNullStringTest() {
         assertThat(toEncodedKeyString(null), is(nullValue()));
         assertThat(toEncodedValueString(null), is(nullValue()));
         assertThat(toDecodedString(null), is(nullValue()));
     }
 
     @Test
-    public void encodeDecode_WithSpaces_Test() {
+    public void encodeDecodeWithSpacesTest() {
         final String encodedKey = toEncodedKeyString(STRING_WITH_SPACES);
         assertTrue(isEncodedSafeKeyChars(encodedKey));
         assertThat(toDecodedString(encodedKey), is(STRING_WITH_SPACES));
@@ -127,7 +127,7 @@ public void encodeDecode_WithSpaces_Test() {
     }
 
     @Test
-    public void encodeDecode_WithSymbols_Test() {
+    public void encodeDecodeWithSymbolsTest() {
         final String encodedKey = toEncodedKeyString(STRING_WITH_SYMBOLS);
         assertTrue(isEncodedSafeKeyChars(encodedKey));
         assertThat(toDecodedString(encodedKey), is(STRING_WITH_SYMBOLS));
@@ -138,7 +138,7 @@ public void encodeDecode_WithSymbols_Test() {
     }
 
     @Test
-    public void encodeDecode_WithUnicodeSymbols_Test() {
+    public void encodeDecodeWithUnicodeSymbolsTest() {
         final String encodedKey = toEncodedKeyString(STRING_WITH_SYMBOLS_UNICODE);
         assertTrue(isEncodedSafeKeyChars(encodedKey));
         assertThat(toDecodedString(encodedKey), is(STRING_WITH_SYMBOLS_UNICODE));
@@ -149,7 +149,7 @@ public void encodeDecode_WithUnicodeSymbols_Test() {
     }
 
     @Test
-    public void encodeDecode_SpaceAndPlus_Test() {
+    public void encodeDecodeSpaceAndPlusTest() {
         final String plusSpace = " +";
 
         final String encodedKey = toEncodedKeyString(plusSpace);
@@ -162,7 +162,7 @@ public void encodeDecode_SpaceAndPlus_Test() {
     }
 
     @Test
-    public void encodeDecode_Range_Test() {
+    public void encodeDecodeRangeTest() {
         final String range = "Range=bytes=0-10,110-120";
 
         final String encodedValue = toEncodedValueString(range);
@@ -172,7 +172,7 @@ public void encodeDecode_Range_Test() {
     }
 
     @Test
-    public void decodedDecode_LowerCase_Test() {
+    public void decodedDecodeLowerCaseTest() {
         final String encodedToLowerKey = toEncodedKeyString(STRING_WITH_SYMBOLS).toLowerCase();
         assertTrue(isEncodedSafeKeyChars(encodedToLowerKey));
         assertThat(toDecodedString(encodedToLowerKey), is(STRING_WITH_SYMBOLS));
@@ -183,7 +183,7 @@ public void decodedDecode_LowerCase_Test() {
     }
 
     @Test
-    public void encodeDecode_DoubleEncoding_Test() {
+    public void encodeDecodeDoubleEncodingTest() {
         final String encoded1Key = toEncodedKeyString(STRING_WITH_SYMBOLS);
         final String encoded2Key = toEncodedKeyString(encoded1Key);
         assertTrue(isEncodedSafeKeyChars(encoded1Key));

From 9caac9eeb60e214cabd2ce73c947f2ec7ee19a7a Mon Sep 17 00:00:00 2001
From: Sharon Shabtai 
Date: Mon, 22 Apr 2019 11:12:02 -0600
Subject: [PATCH 173/229] cherry-pick copyright update

---
 build.gradle                                  | 20 +++++++++----------
 contract/5_0_x_contract.xml                   | 15 ++++++++++++++
 ds3-interfaces/build.gradle                   | 15 ++++++++++++++
 .../ds3client/helpers/MetadataAccess.java     |  2 +-
 .../helpers/MetadataReceivedListener.java     |  2 +-
 .../metadata/interfaces/MetadataRestore.java  |  2 +-
 .../metadata/interfaces/MetadataStore.java    |  2 +-
 .../ds3client/networking/Headers.java         |  2 +-
 .../ds3client/networking/Metadata.java        |  2 +-
 ds3-metadata/build.gradle                     |  2 +-
 .../metadata/AbstractMetadataRestore.java     |  2 +-
 .../metadata/AbstractMetadataStore.java       |  2 +-
 .../metadata/MACMetadataRestore.java          |  2 +-
 .../ds3client/metadata/MetaDataUtil.java      |  2 +-
 .../metadata/MetadataAccessImpl.java          |  2 +-
 .../metadata/MetadataKeyConstants.java        |  2 +-
 .../MetadataReceivedListenerImpl.java         |  2 +-
 .../metadata/MetadataRestoreFactory.java      |  2 +-
 .../metadata/MetadataStoreFactory.java        |  2 +-
 .../ds3client/metadata/PermissionsUtils.java  |  2 +-
 .../metadata/PosixMetadataRestore.java        |  2 +-
 .../metadata/PosixMetadataStore.java          |  2 +-
 .../metadata/WindowsMetadataException.java    |  2 +-
 .../metadata/WindowsMetadataRestore.java      |  2 +-
 .../metadata/WindowsMetadataStore.java        |  2 +-
 .../ds3client/metadata/jna/Advapi32.java      |  2 +-
 .../spectralogic/ds3client/MockedHeaders.java |  2 +-
 .../ds3client/MockedHeadersReturningKeys.java |  2 +-
 .../metadata/MACMetadataRestore_Test.java     |  2 +-
 .../metadata/MetadataAccessImpl_Test.java     |  2 +-
 .../MetadataReceivedListenerImpl_Test.java    |  2 +-
 .../metadata/MetadataRestoreFactory_Test.java |  2 +-
 .../metadata/PosixMetadataRestore_Test.java   |  2 +-
 .../metadata/PosixMetadataStore_Test.java     |  2 +-
 .../metadata/WindowsMetadataRestore_Test.java |  2 +-
 .../metadata/WindowsMetadataStore_Test.java   |  2 +-
 ds3-sdk-integration/build.gradle              |  2 +-
 .../ds3client/integration/NullChannel.java    |  2 +-
 .../integration/RandomDataChannelBuilder.java |  2 +-
 .../integration/RandomDataInputStream.java    |  2 +-
 .../integration/ResourceObjectPutter.java     |  2 +-
 .../integration/TransferredListener.java      |  2 +-
 .../ds3client/integration/Util.java           |  2 +-
 .../spectralogic/ds3client/MockedHeaders.java |  2 +-
 .../ds3client/MockedWebResponse.java          |  2 +-
 .../helpers/FileSystemHelper_Test.java        |  2 +-
 .../ObjectStreamBuilder_Test.java             |  2 +-
 .../RepeatStringObjectChannelBuilder.java     |  2 +-
 .../AdvancedBucketManagement_Test.java        |  2 +-
 .../integration/CustomParser_Test.java        |  2 +-
 .../integration/DataIntegrity_Test.java       |  2 +-
 .../integration/GetJobManagement_Test.java    |  2 +-
 .../integration/GroupManagement_Test.java     |  2 +-
 .../ds3client/integration/Insecure_Test.java  |  2 +-
 .../ds3client/integration/Iterators_Test.java |  2 +-
 .../ds3client/integration/Metadata_Test.java  |  2 +-
 .../NotificationsIntegration_test.java        |  2 +-
 .../integration/PutJobManagement_Test.java    |  2 +-
 .../integration/Regression_Test.java          |  2 +-
 .../ds3client/integration/ResourceTest.kt     |  2 +-
 .../ds3client/integration/Smoke_Test.java     |  2 +-
 .../SpectraS3PaginationLoader_Test.java       |  2 +-
 .../integration/UsersAndGroups_Test.java      |  2 +-
 .../integration/VersionedObject_Test.java     |  2 +-
 .../test/helpers/ABMTestHelper.java           |  2 +-
 .../test/helpers/Ds3ClientShim.java           |  2 +-
 .../test/helpers/Ds3ClientShimFactory.java    |  2 +-
 ...s3ClientShimWithFailedChunkAllocation.java |  2 +-
 .../Ds3ClientShimWithFailedGetObject.java     |  2 +-
 .../Ds3ClientShimWithFailedPutObject.java     |  2 +-
 .../test/helpers/JobStatusHelper.java         |  2 +-
 .../NotificationCleanupTestHelper.java        |  2 +-
 .../test/helpers/TempStorageIds.java          |  2 +-
 .../test/helpers/TempStorageUtil.java         |  2 +-
 ds3-sdk-samples/build.gradle                  |  2 +-
 .../samples/BulkGetWithMetadata.java          |  2 +-
 .../ds3client/samples/BulkPutExample.java     |  2 +-
 .../samples/BulkPutWithChecksums.java         |  2 +-
 .../samples/BulkPutWithMetadata.java          |  2 +-
 .../ds3client/samples/Ds3BulkGetExample.java  |  2 +-
 .../Ds3PutObjectRelativePathExample.java      |  2 +-
 .../samples/Ds3ServiceListExample.java        |  2 +-
 .../samples/PartialObjectGetExample.java      |  2 +-
 .../samples/PutEmptyFolderExample.java        |  2 +-
 .../ds3client/samples/RecoverJobExample.java  |  2 +-
 ds3-sdk/build.gradle                          |  2 +-
 .../spectralogic/ds3client/BulkCommand.java   |  2 +-
 .../ds3client/ConnectionDetailsImpl.java      |  2 +-
 .../com/spectralogic/ds3client/Ds3Client.java |  2 +-
 .../ds3client/Ds3ClientBuilder.java           |  2 +-
 .../spectralogic/ds3client/Ds3ClientImpl.java |  2 +-
 .../ds3client/Ds3InputStreamEntity.java       |  2 +-
 .../ds3client/GetObjectCustomParser.java      |  2 +-
 .../ds3client/annotations/Action.java         |  2 +-
 .../ds3client/annotations/Resource.java       |  2 +-
 .../annotations/ResponsePayloadModel.java     |  2 +-
 .../commands/AbortMultiPartUploadRequest.java |  2 +-
 .../AbortMultiPartUploadResponse.java         |  2 +-
 .../CompleteMultiPartUploadRequest.java       |  2 +-
 .../CompleteMultiPartUploadResponse.java      |  2 +-
 .../commands/DeleteBucketRequest.java         |  2 +-
 .../commands/DeleteBucketResponse.java        |  2 +-
 .../commands/DeleteObjectRequest.java         |  2 +-
 .../commands/DeleteObjectResponse.java        |  2 +-
 .../commands/DeleteObjectsRequest.java        |  2 +-
 .../commands/DeleteObjectsResponse.java       |  2 +-
 .../ds3client/commands/GetBucketRequest.java  |  2 +-
 .../ds3client/commands/GetBucketResponse.java |  2 +-
 .../ds3client/commands/GetObjectRequest.java  |  2 +-
 .../ds3client/commands/GetObjectResponse.java |  2 +-
 .../ds3client/commands/GetServiceRequest.java |  2 +-
 .../commands/GetServiceResponse.java          |  2 +-
 .../ds3client/commands/HeadBucketRequest.java |  2 +-
 .../commands/HeadBucketResponse.java          |  2 +-
 .../ds3client/commands/HeadObjectRequest.java |  2 +-
 .../commands/HeadObjectResponse.java          |  2 +-
 .../InitiateMultiPartUploadRequest.java       |  2 +-
 .../InitiateMultiPartUploadResponse.java      |  2 +-
 .../ListMultiPartUploadPartsRequest.java      |  2 +-
 .../ListMultiPartUploadPartsResponse.java     |  2 +-
 .../commands/ListMultiPartUploadsRequest.java |  2 +-
 .../ListMultiPartUploadsResponse.java         |  2 +-
 .../ds3client/commands/PutBucketRequest.java  |  2 +-
 .../ds3client/commands/PutBucketResponse.java |  2 +-
 .../PutMultiPartUploadPartRequest.java        |  2 +-
 .../PutMultiPartUploadPartResponse.java       |  2 +-
 .../ds3client/commands/PutObjectRequest.java  |  2 +-
 .../ds3client/commands/PutObjectResponse.java |  2 +-
 .../AbstractCreateNotificationRequest.java    |  2 +-
 .../AbstractDeleteNotificationRequest.java    |  2 +-
 .../AbstractGetNotificationRequest.java       |  2 +-
 .../interfaces/AbstractIdsPayloadRequest.java |  2 +-
 .../interfaces/AbstractPaginationRequest.java |  2 +-
 .../AbstractPaginationResponse.java           |  2 +-
 .../commands/interfaces/AbstractRequest.java  |  2 +-
 .../commands/interfaces/AbstractResponse.java |  2 +-
 .../commands/interfaces/BulkRequest.java      |  2 +-
 .../commands/interfaces/BulkResponse.java     |  2 +-
 .../commands/interfaces/Ds3Request.java       |  2 +-
 .../commands/interfaces/Ds3Response.java      |  2 +-
 .../interfaces/PaginationRequest.java         |  2 +-
 .../interfaces/PaginationResponse.java        |  2 +-
 .../commands/interfaces/RequestHeaders.java   |  2 +-
 .../interfaces/RequestHeadersImpl.java        |  2 +-
 .../ds3client/commands/package-info.java      |  2 +-
 .../AbortMultiPartUploadResponseParser.java   |  2 +-
 ...locateJobChunkSpectraS3ResponseParser.java |  2 +-
 ...ancelActiveJobSpectraS3ResponseParser.java |  2 +-
 ...lAllActiveJobsSpectraS3ResponseParser.java |  2 +-
 .../CancelAllJobsSpectraS3ResponseParser.java |  2 +-
 ...jectOnAllTapesSpectraS3ResponseParser.java |  2 +-
 ...ancelEjectTapeSpectraS3ResponseParser.java |  2 +-
 ...rmatOnAllTapesSpectraS3ResponseParser.java |  2 +-
 ...ncelFormatTapeSpectraS3ResponseParser.java |  2 +-
 ...portOnAllPoolsSpectraS3ResponseParser.java |  2 +-
 ...portOnAllTapesSpectraS3ResponseParser.java |  2 +-
 ...ncelImportPoolSpectraS3ResponseParser.java |  2 +-
 ...ncelImportTapeSpectraS3ResponseParser.java |  2 +-
 .../CancelJobSpectraS3ResponseParser.java     |  2 +-
 ...lineOnAllTapesSpectraS3ResponseParser.java |  2 +-
 ...ncelOnlineTapeSpectraS3ResponseParser.java |  2 +-
 ...rifyOnAllPoolsSpectraS3ResponseParser.java |  2 +-
 ...rifyOnAllTapesSpectraS3ResponseParser.java |  2 +-
 ...ncelVerifyPoolSpectraS3ResponseParser.java |  2 +-
 ...ncelVerifyTapeSpectraS3ResponseParser.java |  2 +-
 ...CleanTapeDriveSpectraS3ResponseParser.java |  2 +-
 ...llCanceledJobsSpectraS3ResponseParser.java |  2 +-
 ...lCompletedJobsSpectraS3ResponseParser.java |  2 +-
 ...obAzureTargetsSpectraS3ResponseParser.java |  2 +-
 ...BlobDs3TargetsSpectraS3ResponseParser.java |  2 +-
 ...spectBlobPoolsSpectraS3ResponseParser.java |  2 +-
 ...tBlobS3TargetsSpectraS3ResponseParser.java |  2 +-
 ...spectBlobTapesSpectraS3ResponseParser.java |  2 +-
 ...AggregatingJobSpectraS3ResponseParser.java |  2 +-
 ...ompactAllPoolsSpectraS3ResponseParser.java |  2 +-
 .../CompactPoolSpectraS3ResponseParser.java   |  2 +-
 ...CompleteMultiPartUploadResponseParser.java |  2 +-
 ...ainToDs3TargetSpectraS3ResponseParser.java |  2 +-
 ...DeallocatePoolSpectraS3ResponseParser.java |  2 +-
 ...gateCreateUserSpectraS3ResponseParser.java |  2 +-
 ...gateDeleteUserSpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...rgetBucketNameSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...eTargetFailureSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...eteAzureTargetSpectraS3ResponseParser.java |  2 +-
 ...eleteBucketAclSpectraS3ResponseParser.java |  2 +-
 .../parsers/DeleteBucketResponseParser.java   |  2 +-
 .../DeleteBucketSpectraS3ResponseParser.java  |  2 +-
 ...ersistenceRuleSpectraS3ResponseParser.java |  2 +-
 ...eDataPolicyAclSpectraS3ResponseParser.java |  2 +-
 ...leteDataPolicySpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...3TargetFailureSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...eleteDs3TargetSpectraS3ResponseParser.java |  2 +-
 ...derRecursivelySpectraS3ResponseParser.java |  2 +-
 ...eteGroupMemberSpectraS3ResponseParser.java |  2 +-
 .../DeleteGroupSpectraS3ResponseParser.java   |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 .../parsers/DeleteObjectResponseParser.java   |  2 +-
 .../parsers/DeleteObjectsResponseParser.java  |  2 +-
 ...nentlyLostPoolSpectraS3ResponseParser.java |  2 +-
 ...nentlyLostTapeSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...etePoolFailureSpectraS3ResponseParser.java |  2 +-
 ...ePoolPartitionSpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...rgetBucketNameSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...3TargetFailureSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...DeleteS3TargetSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...eDomainFailureSpectraS3ResponseParser.java |  2 +-
 ...geDomainMemberSpectraS3ResponseParser.java |  2 +-
 ...eStorageDomainSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nsityDirectiveSpectraS3ResponseParser.java |  2 +-
 ...eleteTapeDriveSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...eteTapeFailureSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...rtitionFailureSpectraS3ResponseParser.java |  2 +-
 ...eTapePartitionSpectraS3ResponseParser.java |  2 +-
 .../EjectAllTapesSpectraS3ResponseParser.java |  2 +-
 ...ageDomainBlobsSpectraS3ResponseParser.java |  2 +-
 ...tStorageDomainSpectraS3ResponseParser.java |  2 +-
 .../EjectTapeSpectraS3ResponseParser.java     |  2 +-
 ...eKeyValidationSpectraS3ResponseParser.java |  2 +-
 ...llCacheReclaimSpectraS3ResponseParser.java |  2 +-
 ...ronmentRefreshSpectraS3ResponseParser.java |  2 +-
 ...ronmentRefreshSpectraS3ResponseParser.java |  2 +-
 ...ronmentRefreshSpectraS3ResponseParser.java |  2 +-
 ...llForeignPoolsSpectraS3ResponseParser.java |  2 +-
 ...FormatAllTapesSpectraS3ResponseParser.java |  2 +-
 ...matForeignPoolSpectraS3ResponseParser.java |  2 +-
 .../FormatTapeSpectraS3ResponseParser.java    |  2 +-
 .../GetActiveJobSpectraS3ResponseParser.java  |  2 +-
 .../GetActiveJobsSpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...plicationRulesSpectraS3ResponseParser.java |  2 +-
 ...getBucketNamesSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...TargetFailuresSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...eadPreferencesSpectraS3ResponseParser.java |  2 +-
 ...GetAzureTargetSpectraS3ResponseParser.java |  2 +-
 ...etAzureTargetsSpectraS3ResponseParser.java |  2 +-
 ...lobPersistenceSpectraS3ResponseParser.java |  2 +-
 ...sOnAzureTargetSpectraS3ResponseParser.java |  2 +-
 ...obsOnDs3TargetSpectraS3ResponseParser.java |  2 +-
 ...GetBlobsOnPoolSpectraS3ResponseParser.java |  2 +-
 ...lobsOnS3TargetSpectraS3ResponseParser.java |  2 +-
 ...GetBlobsOnTapeSpectraS3ResponseParser.java |  2 +-
 .../GetBucketAclSpectraS3ResponseParser.java  |  2 +-
 .../GetBucketAclsSpectraS3ResponseParser.java |  2 +-
 ...apacitySummarySpectraS3ResponseParser.java |  2 +-
 .../parsers/GetBucketResponseParser.java      |  2 +-
 .../GetBucketSpectraS3ResponseParser.java     |  2 +-
 .../GetBucketsSpectraS3ResponseParser.java    |  2 +-
 .../GetBulkJobSpectraS3ResponseParser.java    |  2 +-
 ...acheFilesystemSpectraS3ResponseParser.java |  2 +-
 ...cheFilesystemsSpectraS3ResponseParser.java |  2 +-
 .../GetCacheStateSpectraS3ResponseParser.java |  2 +-
 ...GetCanceledJobSpectraS3ResponseParser.java |  2 +-
 ...etCanceledJobsSpectraS3ResponseParser.java |  2 +-
 ...etCompletedJobSpectraS3ResponseParser.java |  2 +-
 ...tCompletedJobsSpectraS3ResponseParser.java |  2 +-
 ...ataPathBackendSpectraS3ResponseParser.java |  2 +-
 ...ersistenceRuleSpectraS3ResponseParser.java |  2 +-
 ...rsistenceRulesSpectraS3ResponseParser.java |  2 +-
 ...BlobStoreTasksSpectraS3ResponseParser.java |  2 +-
 ...etDataPoliciesSpectraS3ResponseParser.java |  2 +-
 ...tDataPolicyAclSpectraS3ResponseParser.java |  2 +-
 ...DataPolicyAclsSpectraS3ResponseParser.java |  2 +-
 .../GetDataPolicySpectraS3ResponseParser.java |  2 +-
 ...plicationRulesSpectraS3ResponseParser.java |  2 +-
 ...tDegradedBlobsSpectraS3ResponseParser.java |  2 +-
 ...egradedBucketsSpectraS3ResponseParser.java |  2 +-
 ...rsistenceRulesSpectraS3ResponseParser.java |  2 +-
 ...plicationRulesSpectraS3ResponseParser.java |  2 +-
 ...plicationRulesSpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...plicationRulesSpectraS3ResponseParser.java |  2 +-
 ...etDataPoliciesSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...TargetFailuresSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...eadPreferencesSpectraS3ResponseParser.java |  2 +-
 .../GetDs3TargetSpectraS3ResponseParser.java  |  2 +-
 .../GetDs3TargetsSpectraS3ResponseParser.java |  2 +-
 ...GetFeatureKeysSpectraS3ResponseParser.java |  2 +-
 ...GetGroupMemberSpectraS3ResponseParser.java |  2 +-
 ...etGroupMembersSpectraS3ResponseParser.java |  2 +-
 .../GetGroupSpectraS3ResponseParser.java      |  2 +-
 .../GetGroupsSpectraS3ResponseParser.java     |  2 +-
 ...GetJobChunkDaoSpectraS3ResponseParser.java |  2 +-
 .../GetJobChunkSpectraS3ResponseParser.java   |  2 +-
 ...ientProcessingSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 .../GetJobSpectraS3ResponseParser.java        |  2 +-
 ...JobToReplicateSpectraS3ResponseParser.java |  2 +-
 .../GetJobsSpectraS3ResponseParser.java       |  2 +-
 .../GetNodeSpectraS3ResponseParser.java       |  2 +-
 .../GetNodesSpectraS3ResponseParser.java      |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...tObjectDetailsSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 .../parsers/GetObjectResponseParser.java      |  2 +-
 ...ObjectsDetailsSpectraS3ResponseParser.java |  2 +-
 ...ithFullDetailsSpectraS3ResponseParser.java |  2 +-
 ...mentForObjectsSpectraS3ResponseParser.java |  2 +-
 ...ithFullDetailsSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...etPoolFailuresSpectraS3ResponseParser.java |  2 +-
 ...tPoolPartitionSpectraS3ResponseParser.java |  2 +-
 ...PoolPartitionsSpectraS3ResponseParser.java |  2 +-
 .../GetPoolSpectraS3ResponseParser.java       |  2 +-
 .../GetPoolsSpectraS3ResponseParser.java      |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...plicationRulesSpectraS3ResponseParser.java |  2 +-
 ...getBucketNamesSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...TargetFailuresSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...eadPreferencesSpectraS3ResponseParser.java |  2 +-
 .../GetS3TargetSpectraS3ResponseParser.java   |  2 +-
 .../GetS3TargetsSpectraS3ResponseParser.java  |  2 +-
 .../parsers/GetServiceResponseParser.java     |  2 +-
 ...apacitySummarySpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...DomainFailuresSpectraS3ResponseParser.java |  2 +-
 ...geDomainMemberSpectraS3ResponseParser.java |  2 +-
 ...eDomainMembersSpectraS3ResponseParser.java |  2 +-
 ...tStorageDomainSpectraS3ResponseParser.java |  2 +-
 ...StorageDomainsSpectraS3ResponseParser.java |  2 +-
 ...obAzureTargetsSpectraS3ResponseParser.java |  2 +-
 ...BlobDs3TargetsSpectraS3ResponseParser.java |  2 +-
 ...spectBlobPoolsSpectraS3ResponseParser.java |  2 +-
 ...tBlobS3TargetsSpectraS3ResponseParser.java |  2 +-
 ...spectBlobTapesSpectraS3ResponseParser.java |  2 +-
 ...SuspectBucketsSpectraS3ResponseParser.java |  2 +-
 ...SuspectObjectsSpectraS3ResponseParser.java |  2 +-
 ...ithFullDetailsSpectraS3ResponseParser.java |  2 +-
 ...apacitySummarySpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...SystemFailuresSpectraS3ResponseParser.java |  2 +-
 ...temInformationSpectraS3ResponseParser.java |  2 +-
 ...nsityDirectiveSpectraS3ResponseParser.java |  2 +-
 ...sityDirectivesSpectraS3ResponseParser.java |  2 +-
 .../GetTapeDriveSpectraS3ResponseParser.java  |  2 +-
 .../GetTapeDrivesSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...etTapeFailuresSpectraS3ResponseParser.java |  2 +-
 ...tTapeLibrariesSpectraS3ResponseParser.java |  2 +-
 ...GetTapeLibrarySpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nRegistrationsSpectraS3ResponseParser.java |  2 +-
 ...titionFailuresSpectraS3ResponseParser.java |  2 +-
 ...tTapePartitionSpectraS3ResponseParser.java |  2 +-
 ...ithFullDetailsSpectraS3ResponseParser.java |  2 +-
 ...TapePartitionsSpectraS3ResponseParser.java |  2 +-
 ...ithFullDetailsSpectraS3ResponseParser.java |  2 +-
 .../GetTapeSpectraS3ResponseParser.java       |  2 +-
 .../GetTapesSpectraS3ResponseParser.java      |  2 +-
 .../GetUserSpectraS3ResponseParser.java       |  2 +-
 .../GetUsersSpectraS3ResponseParser.java      |  2 +-
 .../parsers/HeadBucketResponseParser.java     |  2 +-
 .../parsers/HeadObjectResponseParser.java     |  2 +-
 ...ImportAllPoolsSpectraS3ResponseParser.java |  2 +-
 ...ImportAllTapesSpectraS3ResponseParser.java |  2 +-
 ...ortAzureTargetSpectraS3ResponseParser.java |  2 +-
 .../ImportPoolSpectraS3ResponseParser.java    |  2 +-
 ...ImportS3TargetSpectraS3ResponseParser.java |  2 +-
 .../ImportTapeSpectraS3ResponseParser.java    |  2 +-
 ...InitiateMultiPartUploadResponseParser.java |  2 +-
 ...nspectAllTapesSpectraS3ResponseParser.java |  2 +-
 .../InspectTapeSpectraS3ResponseParser.java   |  2 +-
 ...istMultiPartUploadPartsResponseParser.java |  2 +-
 .../ListMultiPartUploadsResponseParser.java   |  2 +-
 ...getsAsDegradedSpectraS3ResponseParser.java |  2 +-
 ...getsAsDegradedSpectraS3ResponseParser.java |  2 +-
 ...oolsAsDegradedSpectraS3ResponseParser.java |  2 +-
 ...getsAsDegradedSpectraS3ResponseParser.java |  2 +-
 ...apesAsDegradedSpectraS3ResponseParser.java |  2 +-
 ...odifyActiveJobSpectraS3ResponseParser.java |  2 +-
 ...llAzureTargetsSpectraS3ResponseParser.java |  2 +-
 ...yAllDs3TargetsSpectraS3ResponseParser.java |  2 +-
 ...ModifyAllPoolsSpectraS3ResponseParser.java |  2 +-
 ...fyAllS3TargetsSpectraS3ResponseParser.java |  2 +-
 ...TapePartitionsSpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...ifyAzureTargetSpectraS3ResponseParser.java |  2 +-
 .../ModifyBucketSpectraS3ResponseParser.java  |  2 +-
 ...acheFilesystemSpectraS3ResponseParser.java |  2 +-
 ...ataPathBackendSpectraS3ResponseParser.java |  2 +-
 ...ersistenceRuleSpectraS3ResponseParser.java |  2 +-
 ...difyDataPolicySpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...odifyDs3TargetSpectraS3ResponseParser.java |  2 +-
 .../ModifyGroupSpectraS3ResponseParser.java   |  2 +-
 .../ModifyJobSpectraS3ResponseParser.java     |  2 +-
 .../ModifyNodeSpectraS3ResponseParser.java    |  2 +-
 ...yPoolPartitionSpectraS3ResponseParser.java |  2 +-
 .../ModifyPoolSpectraS3ResponseParser.java    |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...ModifyS3TargetSpectraS3ResponseParser.java |  2 +-
 ...geDomainMemberSpectraS3ResponseParser.java |  2 +-
 ...yStorageDomainSpectraS3ResponseParser.java |  2 +-
 ...odifyTapeDriveSpectraS3ResponseParser.java |  2 +-
 ...yTapePartitionSpectraS3ResponseParser.java |  2 +-
 .../ModifyTapeSpectraS3ResponseParser.java    |  2 +-
 .../ModifyUserSpectraS3ResponseParser.java    |  2 +-
 ...OnlineAllTapesSpectraS3ResponseParser.java |  2 +-
 .../OnlineTapeSpectraS3ResponseParser.java    |  2 +-
 ...teredDs3TargetSpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...rgetBucketNameSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...ketAclForGroupSpectraS3ResponseParser.java |  2 +-
 ...cketAclForUserSpectraS3ResponseParser.java |  2 +-
 .../parsers/PutBucketResponseParser.java      |  2 +-
 .../PutBucketSpectraS3ResponseParser.java     |  2 +-
 .../PutBulkJobSpectraS3ResponseParser.java    |  2 +-
 ...ersistenceRuleSpectraS3ResponseParser.java |  2 +-
 ...icyAclForGroupSpectraS3ResponseParser.java |  2 +-
 ...licyAclForUserSpectraS3ResponseParser.java |  2 +-
 .../PutDataPolicySpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...ketAclForGroupSpectraS3ResponseParser.java |  2 +-
 ...cketAclForUserSpectraS3ResponseParser.java |  2 +-
 ...icyAclForGroupSpectraS3ResponseParser.java |  2 +-
 ...licyAclForUserSpectraS3ResponseParser.java |  2 +-
 ...oupGroupMemberSpectraS3ResponseParser.java |  2 +-
 .../PutGroupSpectraS3ResponseParser.java      |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 .../PutMultiPartUploadPartResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 .../parsers/PutObjectResponseParser.java      |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...tPoolPartitionSpectraS3ResponseParser.java |  2 +-
 ...geDomainMemberSpectraS3ResponseParser.java |  2 +-
 ...eplicationRuleSpectraS3ResponseParser.java |  2 +-
 ...rgetBucketNameSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...ReadPreferenceSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...tStorageDomainSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...nsityDirectiveSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...onRegistrationSpectraS3ResponseParser.java |  2 +-
 ...geDomainMemberSpectraS3ResponseParser.java |  2 +-
 ...serGroupMemberSpectraS3ResponseParser.java |  2 +-
 ...ImportAllTapesSpectraS3ResponseParser.java |  2 +-
 .../RawImportTapeSpectraS3ResponseParser.java |  2 +-
 ...eUserSecretKeySpectraS3ResponseParser.java |  2 +-
 ...terAzureTargetSpectraS3ResponseParser.java |  2 +-
 ...isterDs3TargetSpectraS3ResponseParser.java |  2 +-
 ...gisterS3TargetSpectraS3ResponseParser.java |  2 +-
 ...eplicatePutJobSpectraS3ResponseParser.java |  2 +-
 ...anceIdentifierSpectraS3ResponseParser.java |  2 +-
 ...tageObjectsJobSpectraS3ResponseParser.java |  2 +-
 ...ncateActiveJobSpectraS3ResponseParser.java |  2 +-
 ...eAllActiveJobsSpectraS3ResponseParser.java |  2 +-
 ...runcateAllJobsSpectraS3ResponseParser.java |  2 +-
 .../TruncateJobSpectraS3ResponseParser.java   |  2 +-
 ...UndeleteObjectSpectraS3ResponseParser.java |  2 +-
 ...VerifyAllPoolsSpectraS3ResponseParser.java |  2 +-
 ...VerifyAllTapesSpectraS3ResponseParser.java |  2 +-
 ...ifyAzureTargetSpectraS3ResponseParser.java |  2 +-
 .../VerifyBulkJobSpectraS3ResponseParser.java |  2 +-
 ...erifyDs3TargetSpectraS3ResponseParser.java |  2 +-
 ...mentForObjectsSpectraS3ResponseParser.java |  2 +-
 ...ithFullDetailsSpectraS3ResponseParser.java |  2 +-
 .../VerifyPoolSpectraS3ResponseParser.java    |  2 +-
 ...VerifyS3TargetSpectraS3ResponseParser.java |  2 +-
 ...ToCreatePutJobSpectraS3ResponseParser.java |  2 +-
 ...fySystemHealthSpectraS3ResponseParser.java |  2 +-
 .../VerifyTapeSpectraS3ResponseParser.java    |  2 +-
 ...sMemberOfGroupSpectraS3ResponseParser.java |  2 +-
 .../interfaces/AbstractResponseParser.java    |  2 +-
 .../GetObjectCustomParserParameters.java      |  2 +-
 .../commands/parsers/utils/Function.java      |  2 +-
 .../parsers/utils/ResponseParserUtils.java    |  2 +-
 .../AllocateJobChunkSpectraS3Request.java     |  2 +-
 .../AllocateJobChunkSpectraS3Response.java    |  2 +-
 .../CancelActiveJobSpectraS3Request.java      |  2 +-
 .../CancelActiveJobSpectraS3Response.java     |  2 +-
 .../CancelAllActiveJobsSpectraS3Request.java  |  2 +-
 .../CancelAllActiveJobsSpectraS3Response.java |  2 +-
 .../CancelAllJobsSpectraS3Request.java        |  2 +-
 .../CancelAllJobsSpectraS3Response.java       |  2 +-
 ...CancelEjectOnAllTapesSpectraS3Request.java |  2 +-
 ...ancelEjectOnAllTapesSpectraS3Response.java |  2 +-
 .../CancelEjectTapeSpectraS3Request.java      |  2 +-
 .../CancelEjectTapeSpectraS3Response.java     |  2 +-
 ...ancelFormatOnAllTapesSpectraS3Request.java |  2 +-
 ...ncelFormatOnAllTapesSpectraS3Response.java |  2 +-
 .../CancelFormatTapeSpectraS3Request.java     |  2 +-
 .../CancelFormatTapeSpectraS3Response.java    |  2 +-
 ...ancelImportOnAllPoolsSpectraS3Request.java |  2 +-
 ...ncelImportOnAllPoolsSpectraS3Response.java |  2 +-
 ...ancelImportOnAllTapesSpectraS3Request.java |  2 +-
 ...ncelImportOnAllTapesSpectraS3Response.java |  2 +-
 .../CancelImportPoolSpectraS3Request.java     |  2 +-
 .../CancelImportPoolSpectraS3Response.java    |  2 +-
 .../CancelImportTapeSpectraS3Request.java     |  2 +-
 .../CancelImportTapeSpectraS3Response.java    |  2 +-
 .../spectrads3/CancelJobSpectraS3Request.java |  2 +-
 .../CancelJobSpectraS3Response.java           |  2 +-
 ...ancelOnlineOnAllTapesSpectraS3Request.java |  2 +-
 ...ncelOnlineOnAllTapesSpectraS3Response.java |  2 +-
 .../CancelOnlineTapeSpectraS3Request.java     |  2 +-
 .../CancelOnlineTapeSpectraS3Response.java    |  2 +-
 ...ancelVerifyOnAllPoolsSpectraS3Request.java |  2 +-
 ...ncelVerifyOnAllPoolsSpectraS3Response.java |  2 +-
 ...ancelVerifyOnAllTapesSpectraS3Request.java |  2 +-
 ...ncelVerifyOnAllTapesSpectraS3Response.java |  2 +-
 .../CancelVerifyPoolSpectraS3Request.java     |  2 +-
 .../CancelVerifyPoolSpectraS3Response.java    |  2 +-
 .../CancelVerifyTapeSpectraS3Request.java     |  2 +-
 .../CancelVerifyTapeSpectraS3Response.java    |  2 +-
 .../CleanTapeDriveSpectraS3Request.java       |  2 +-
 .../CleanTapeDriveSpectraS3Response.java      |  2 +-
 .../ClearAllCanceledJobsSpectraS3Request.java |  2 +-
 ...ClearAllCanceledJobsSpectraS3Response.java |  2 +-
 ...ClearAllCompletedJobsSpectraS3Request.java |  2 +-
 ...learAllCompletedJobsSpectraS3Response.java |  2 +-
 ...spectBlobAzureTargetsSpectraS3Request.java |  2 +-
 ...pectBlobAzureTargetsSpectraS3Response.java |  2 +-
 ...SuspectBlobDs3TargetsSpectraS3Request.java |  2 +-
 ...uspectBlobDs3TargetsSpectraS3Response.java |  2 +-
 ...ClearSuspectBlobPoolsSpectraS3Request.java |  2 +-
 ...learSuspectBlobPoolsSpectraS3Response.java |  2 +-
 ...rSuspectBlobS3TargetsSpectraS3Request.java |  2 +-
 ...SuspectBlobS3TargetsSpectraS3Response.java |  2 +-
 ...ClearSuspectBlobTapesSpectraS3Request.java |  2 +-
 ...learSuspectBlobTapesSpectraS3Response.java |  2 +-
 .../CloseAggregatingJobSpectraS3Request.java  |  2 +-
 .../CloseAggregatingJobSpectraS3Response.java |  2 +-
 .../CompactAllPoolsSpectraS3Request.java      |  2 +-
 .../CompactAllPoolsSpectraS3Response.java     |  2 +-
 .../CompactPoolSpectraS3Request.java          |  2 +-
 .../CompactPoolSpectraS3Response.java         |  2 +-
 ...rageDomainToDs3TargetSpectraS3Request.java |  2 +-
 ...ageDomainToDs3TargetSpectraS3Response.java |  2 +-
 .../DeallocatePoolSpectraS3Request.java       |  2 +-
 .../DeallocatePoolSpectraS3Response.java      |  2 +-
 .../DelegateCreateUserSpectraS3Request.java   |  2 +-
 .../DelegateCreateUserSpectraS3Response.java  |  2 +-
 .../DelegateDeleteUserSpectraS3Request.java   |  2 +-
 .../DelegateDeleteUserSpectraS3Response.java  |  2 +-
 ...reDataReplicationRuleSpectraS3Request.java |  2 +-
 ...eDataReplicationRuleSpectraS3Response.java |  2 +-
 ...AzureTargetBucketNameSpectraS3Request.java |  2 +-
 ...zureTargetBucketNameSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...eteAzureTargetFailureSpectraS3Request.java |  2 +-
 ...teAzureTargetFailureSpectraS3Response.java |  2 +-
 ...eTargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 .../DeleteAzureTargetSpectraS3Request.java    |  2 +-
 .../DeleteAzureTargetSpectraS3Response.java   |  2 +-
 .../DeleteBucketAclSpectraS3Request.java      |  2 +-
 .../DeleteBucketAclSpectraS3Response.java     |  2 +-
 .../DeleteBucketSpectraS3Request.java         |  2 +-
 .../DeleteBucketSpectraS3Response.java        |  2 +-
 ...teDataPersistenceRuleSpectraS3Request.java |  2 +-
 ...eDataPersistenceRuleSpectraS3Response.java |  2 +-
 .../DeleteDataPolicyAclSpectraS3Request.java  |  2 +-
 .../DeleteDataPolicyAclSpectraS3Response.java |  2 +-
 .../DeleteDataPolicySpectraS3Request.java     |  2 +-
 .../DeleteDataPolicySpectraS3Response.java    |  2 +-
 ...s3DataReplicationRuleSpectraS3Request.java |  2 +-
 ...3DataReplicationRuleSpectraS3Response.java |  2 +-
 ...eleteDs3TargetFailureSpectraS3Request.java |  2 +-
 ...leteDs3TargetFailureSpectraS3Response.java |  2 +-
 ...3TargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 .../DeleteDs3TargetSpectraS3Request.java      |  2 +-
 .../DeleteDs3TargetSpectraS3Response.java     |  2 +-
 ...leteFolderRecursivelySpectraS3Request.java |  2 +-
 ...eteFolderRecursivelySpectraS3Response.java |  2 +-
 .../DeleteGroupMemberSpectraS3Request.java    |  2 +-
 .../DeleteGroupMemberSpectraS3Response.java   |  2 +-
 .../DeleteGroupSpectraS3Request.java          |  2 +-
 .../DeleteGroupSpectraS3Response.java         |  2 +-
 ...tePermanentlyLostPoolSpectraS3Request.java |  2 +-
 ...ePermanentlyLostPoolSpectraS3Response.java |  2 +-
 ...tePermanentlyLostTapeSpectraS3Request.java |  2 +-
 ...ePermanentlyLostTapeSpectraS3Response.java |  2 +-
 .../DeletePoolFailureSpectraS3Request.java    |  2 +-
 .../DeletePoolFailureSpectraS3Response.java   |  2 +-
 .../DeletePoolPartitionSpectraS3Request.java  |  2 +-
 .../DeletePoolPartitionSpectraS3Response.java |  2 +-
 ...S3DataReplicationRuleSpectraS3Request.java |  2 +-
 ...3DataReplicationRuleSpectraS3Response.java |  2 +-
 ...eteS3TargetBucketNameSpectraS3Request.java |  2 +-
 ...teS3TargetBucketNameSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...DeleteS3TargetFailureSpectraS3Request.java |  2 +-
 ...eleteS3TargetFailureSpectraS3Response.java |  2 +-
 ...3TargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 .../DeleteS3TargetSpectraS3Request.java       |  2 +-
 .../DeleteS3TargetSpectraS3Response.java      |  2 +-
 ...eStorageDomainFailureSpectraS3Request.java |  2 +-
 ...StorageDomainFailureSpectraS3Response.java |  2 +-
 ...teStorageDomainMemberSpectraS3Request.java |  2 +-
 ...eStorageDomainMemberSpectraS3Response.java |  2 +-
 .../DeleteStorageDomainSpectraS3Request.java  |  2 +-
 .../DeleteStorageDomainSpectraS3Response.java |  2 +-
 ...eTapeDensityDirectiveSpectraS3Request.java |  2 +-
 ...TapeDensityDirectiveSpectraS3Response.java |  2 +-
 .../DeleteTapeDriveSpectraS3Request.java      |  2 +-
 .../DeleteTapeDriveSpectraS3Response.java     |  2 +-
 .../DeleteTapeFailureSpectraS3Request.java    |  2 +-
 .../DeleteTapeFailureSpectraS3Response.java   |  2 +-
 ...eTapePartitionFailureSpectraS3Request.java |  2 +-
 ...TapePartitionFailureSpectraS3Response.java |  2 +-
 .../DeleteTapePartitionSpectraS3Request.java  |  2 +-
 .../DeleteTapePartitionSpectraS3Response.java |  2 +-
 .../EjectAllTapesSpectraS3Request.java        |  2 +-
 .../EjectAllTapesSpectraS3Response.java       |  2 +-
 ...ectStorageDomainBlobsSpectraS3Request.java |  2 +-
 ...ctStorageDomainBlobsSpectraS3Response.java |  2 +-
 .../EjectStorageDomainSpectraS3Request.java   |  2 +-
 .../EjectStorageDomainSpectraS3Response.java  |  2 +-
 .../spectrads3/EjectTapeSpectraS3Request.java |  2 +-
 .../EjectTapeSpectraS3Response.java           |  2 +-
 ...eFeatureKeyValidationSpectraS3Request.java |  2 +-
 ...FeatureKeyValidationSpectraS3Response.java |  2 +-
 ...ForceFullCacheReclaimSpectraS3Request.java |  2 +-
 ...orceFullCacheReclaimSpectraS3Response.java |  2 +-
 ...oolEnvironmentRefreshSpectraS3Request.java |  2 +-
 ...olEnvironmentRefreshSpectraS3Response.java |  2 +-
 ...apeEnvironmentRefreshSpectraS3Request.java |  2 +-
 ...peEnvironmentRefreshSpectraS3Response.java |  2 +-
 ...getEnvironmentRefreshSpectraS3Request.java |  2 +-
 ...etEnvironmentRefreshSpectraS3Response.java |  2 +-
 ...FormatAllForeignPoolsSpectraS3Request.java |  2 +-
 ...ormatAllForeignPoolsSpectraS3Response.java |  2 +-
 .../FormatAllTapesSpectraS3Request.java       |  2 +-
 .../FormatAllTapesSpectraS3Response.java      |  2 +-
 .../FormatForeignPoolSpectraS3Request.java    |  2 +-
 .../FormatForeignPoolSpectraS3Response.java   |  2 +-
 .../FormatTapeSpectraS3Request.java           |  2 +-
 .../FormatTapeSpectraS3Response.java          |  2 +-
 .../GetActiveJobSpectraS3Request.java         |  2 +-
 .../GetActiveJobSpectraS3Response.java        |  2 +-
 .../GetActiveJobsSpectraS3Request.java        |  2 +-
 .../GetActiveJobsSpectraS3Response.java       |  2 +-
 ...reDataReplicationRuleSpectraS3Request.java |  2 +-
 ...eDataReplicationRuleSpectraS3Response.java |  2 +-
 ...eDataReplicationRulesSpectraS3Request.java |  2 +-
 ...DataReplicationRulesSpectraS3Response.java |  2 +-
 ...zureTargetBucketNamesSpectraS3Request.java |  2 +-
 ...ureTargetBucketNamesSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...etAzureTargetFailuresSpectraS3Request.java |  2 +-
 ...tAzureTargetFailuresSpectraS3Response.java |  2 +-
 ...eTargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 ...TargetReadPreferencesSpectraS3Request.java |  2 +-
 ...argetReadPreferencesSpectraS3Response.java |  2 +-
 .../GetAzureTargetSpectraS3Request.java       |  2 +-
 .../GetAzureTargetSpectraS3Response.java      |  2 +-
 .../GetAzureTargetsSpectraS3Request.java      |  2 +-
 .../GetAzureTargetsSpectraS3Response.java     |  2 +-
 .../GetBlobPersistenceSpectraS3Request.java   |  2 +-
 .../GetBlobPersistenceSpectraS3Response.java  |  2 +-
 ...GetBlobsOnAzureTargetSpectraS3Request.java |  2 +-
 ...etBlobsOnAzureTargetSpectraS3Response.java |  2 +-
 .../GetBlobsOnDs3TargetSpectraS3Request.java  |  2 +-
 .../GetBlobsOnDs3TargetSpectraS3Response.java |  2 +-
 .../GetBlobsOnPoolSpectraS3Request.java       |  2 +-
 .../GetBlobsOnPoolSpectraS3Response.java      |  2 +-
 .../GetBlobsOnS3TargetSpectraS3Request.java   |  2 +-
 .../GetBlobsOnS3TargetSpectraS3Response.java  |  2 +-
 .../GetBlobsOnTapeSpectraS3Request.java       |  2 +-
 .../GetBlobsOnTapeSpectraS3Response.java      |  2 +-
 .../GetBucketAclSpectraS3Request.java         |  2 +-
 .../GetBucketAclSpectraS3Response.java        |  2 +-
 .../GetBucketAclsSpectraS3Request.java        |  2 +-
 .../GetBucketAclsSpectraS3Response.java       |  2 +-
 ...BucketCapacitySummarySpectraS3Request.java |  2 +-
 ...ucketCapacitySummarySpectraS3Response.java |  2 +-
 .../spectrads3/GetBucketSpectraS3Request.java |  2 +-
 .../GetBucketSpectraS3Response.java           |  2 +-
 .../GetBucketsSpectraS3Request.java           |  2 +-
 .../GetBucketsSpectraS3Response.java          |  2 +-
 .../GetBulkJobSpectraS3Request.java           |  2 +-
 .../GetBulkJobSpectraS3Response.java          |  2 +-
 .../GetCacheFilesystemSpectraS3Request.java   |  2 +-
 .../GetCacheFilesystemSpectraS3Response.java  |  2 +-
 .../GetCacheFilesystemsSpectraS3Request.java  |  2 +-
 .../GetCacheFilesystemsSpectraS3Response.java |  2 +-
 .../GetCacheStateSpectraS3Request.java        |  2 +-
 .../GetCacheStateSpectraS3Response.java       |  2 +-
 .../GetCanceledJobSpectraS3Request.java       |  2 +-
 .../GetCanceledJobSpectraS3Response.java      |  2 +-
 .../GetCanceledJobsSpectraS3Request.java      |  2 +-
 .../GetCanceledJobsSpectraS3Response.java     |  2 +-
 .../GetCompletedJobSpectraS3Request.java      |  2 +-
 .../GetCompletedJobSpectraS3Response.java     |  2 +-
 .../GetCompletedJobsSpectraS3Request.java     |  2 +-
 .../GetCompletedJobsSpectraS3Response.java    |  2 +-
 .../GetDataPathBackendSpectraS3Request.java   |  2 +-
 .../GetDataPathBackendSpectraS3Response.java  |  2 +-
 ...etDataPersistenceRuleSpectraS3Request.java |  2 +-
 ...tDataPersistenceRuleSpectraS3Response.java |  2 +-
 ...tDataPersistenceRulesSpectraS3Request.java |  2 +-
 ...DataPersistenceRulesSpectraS3Response.java |  2 +-
 ...PlannerBlobStoreTasksSpectraS3Request.java |  2 +-
 ...lannerBlobStoreTasksSpectraS3Response.java |  2 +-
 .../GetDataPoliciesSpectraS3Request.java      |  2 +-
 .../GetDataPoliciesSpectraS3Response.java     |  2 +-
 .../GetDataPolicyAclSpectraS3Request.java     |  2 +-
 .../GetDataPolicyAclSpectraS3Response.java    |  2 +-
 .../GetDataPolicyAclsSpectraS3Request.java    |  2 +-
 .../GetDataPolicyAclsSpectraS3Response.java   |  2 +-
 .../GetDataPolicySpectraS3Request.java        |  2 +-
 .../GetDataPolicySpectraS3Response.java       |  2 +-
 ...eDataReplicationRulesSpectraS3Request.java |  2 +-
 ...DataReplicationRulesSpectraS3Response.java |  2 +-
 .../GetDegradedBlobsSpectraS3Request.java     |  2 +-
 .../GetDegradedBlobsSpectraS3Response.java    |  2 +-
 .../GetDegradedBucketsSpectraS3Request.java   |  2 +-
 .../GetDegradedBucketsSpectraS3Response.java  |  2 +-
 ...dDataPersistenceRulesSpectraS3Request.java |  2 +-
 ...DataPersistenceRulesSpectraS3Response.java |  2 +-
 ...3DataReplicationRulesSpectraS3Request.java |  2 +-
 ...DataReplicationRulesSpectraS3Response.java |  2 +-
 ...3DataReplicationRulesSpectraS3Request.java |  2 +-
 ...DataReplicationRulesSpectraS3Response.java |  2 +-
 ...s3DataReplicationRuleSpectraS3Request.java |  2 +-
 ...3DataReplicationRuleSpectraS3Response.java |  2 +-
 ...3DataReplicationRulesSpectraS3Request.java |  2 +-
 ...DataReplicationRulesSpectraS3Response.java |  2 +-
 ...Ds3TargetDataPoliciesSpectraS3Request.java |  2 +-
 ...s3TargetDataPoliciesSpectraS3Response.java |  2 +-
 .../GetDs3TargetFailuresSpectraS3Request.java |  2 +-
 ...GetDs3TargetFailuresSpectraS3Response.java |  2 +-
 ...3TargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 ...TargetReadPreferencesSpectraS3Request.java |  2 +-
 ...argetReadPreferencesSpectraS3Response.java |  2 +-
 .../GetDs3TargetSpectraS3Request.java         |  2 +-
 .../GetDs3TargetSpectraS3Response.java        |  2 +-
 .../GetDs3TargetsSpectraS3Request.java        |  2 +-
 .../GetDs3TargetsSpectraS3Response.java       |  2 +-
 .../GetFeatureKeysSpectraS3Request.java       |  2 +-
 .../GetFeatureKeysSpectraS3Response.java      |  2 +-
 .../GetGroupMemberSpectraS3Request.java       |  2 +-
 .../GetGroupMemberSpectraS3Response.java      |  2 +-
 .../GetGroupMembersSpectraS3Request.java      |  2 +-
 .../GetGroupMembersSpectraS3Response.java     |  2 +-
 .../spectrads3/GetGroupSpectraS3Request.java  |  2 +-
 .../spectrads3/GetGroupSpectraS3Response.java |  2 +-
 .../spectrads3/GetGroupsSpectraS3Request.java |  2 +-
 .../GetGroupsSpectraS3Response.java           |  2 +-
 .../GetJobChunkDaoSpectraS3Request.java       |  2 +-
 .../GetJobChunkDaoSpectraS3Response.java      |  2 +-
 .../GetJobChunkSpectraS3Request.java          |  2 +-
 .../GetJobChunkSpectraS3Response.java         |  2 +-
 ...dyForClientProcessingSpectraS3Request.java |  2 +-
 ...yForClientProcessingSpectraS3Response.java |  2 +-
 .../spectrads3/GetJobSpectraS3Request.java    |  2 +-
 .../spectrads3/GetJobSpectraS3Response.java   |  2 +-
 .../GetJobToReplicateSpectraS3Request.java    |  2 +-
 .../GetJobToReplicateSpectraS3Response.java   |  2 +-
 .../spectrads3/GetJobsSpectraS3Request.java   |  2 +-
 .../spectrads3/GetJobsSpectraS3Response.java  |  2 +-
 .../spectrads3/GetNodeSpectraS3Request.java   |  2 +-
 .../spectrads3/GetNodeSpectraS3Response.java  |  2 +-
 .../spectrads3/GetNodesSpectraS3Request.java  |  2 +-
 .../spectrads3/GetNodesSpectraS3Response.java |  2 +-
 .../GetObjectDetailsSpectraS3Request.java     |  2 +-
 .../GetObjectDetailsSpectraS3Response.java    |  2 +-
 .../GetObjectsDetailsSpectraS3Request.java    |  2 +-
 .../GetObjectsDetailsSpectraS3Response.java   |  2 +-
 ...bjectsWithFullDetailsSpectraS3Request.java |  2 +-
 ...jectsWithFullDetailsSpectraS3Response.java |  2 +-
 ...alPlacementForObjectsSpectraS3Request.java |  2 +-
 ...lPlacementForObjectsSpectraS3Response.java |  2 +-
 ...bjectsWithFullDetailsSpectraS3Request.java |  2 +-
 ...jectsWithFullDetailsSpectraS3Response.java |  2 +-
 .../GetPoolFailuresSpectraS3Request.java      |  2 +-
 .../GetPoolFailuresSpectraS3Response.java     |  2 +-
 .../GetPoolPartitionSpectraS3Request.java     |  2 +-
 .../GetPoolPartitionSpectraS3Response.java    |  2 +-
 .../GetPoolPartitionsSpectraS3Request.java    |  2 +-
 .../GetPoolPartitionsSpectraS3Response.java   |  2 +-
 .../spectrads3/GetPoolSpectraS3Request.java   |  2 +-
 .../spectrads3/GetPoolSpectraS3Response.java  |  2 +-
 .../spectrads3/GetPoolsSpectraS3Request.java  |  2 +-
 .../spectrads3/GetPoolsSpectraS3Response.java |  2 +-
 ...S3DataReplicationRuleSpectraS3Request.java |  2 +-
 ...3DataReplicationRuleSpectraS3Response.java |  2 +-
 ...3DataReplicationRulesSpectraS3Request.java |  2 +-
 ...DataReplicationRulesSpectraS3Response.java |  2 +-
 ...etS3TargetBucketNamesSpectraS3Request.java |  2 +-
 ...tS3TargetBucketNamesSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 .../GetS3TargetFailuresSpectraS3Request.java  |  2 +-
 .../GetS3TargetFailuresSpectraS3Response.java |  2 +-
 ...3TargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 ...TargetReadPreferencesSpectraS3Request.java |  2 +-
 ...argetReadPreferencesSpectraS3Response.java |  2 +-
 .../GetS3TargetSpectraS3Request.java          |  2 +-
 .../GetS3TargetSpectraS3Response.java         |  2 +-
 .../GetS3TargetsSpectraS3Request.java         |  2 +-
 .../GetS3TargetsSpectraS3Response.java        |  2 +-
 ...DomainCapacitySummarySpectraS3Request.java |  2 +-
 ...omainCapacitySummarySpectraS3Response.java |  2 +-
 ...StorageDomainFailuresSpectraS3Request.java |  2 +-
 ...torageDomainFailuresSpectraS3Response.java |  2 +-
 ...etStorageDomainMemberSpectraS3Request.java |  2 +-
 ...tStorageDomainMemberSpectraS3Response.java |  2 +-
 ...tStorageDomainMembersSpectraS3Request.java |  2 +-
 ...StorageDomainMembersSpectraS3Response.java |  2 +-
 .../GetStorageDomainSpectraS3Request.java     |  2 +-
 .../GetStorageDomainSpectraS3Response.java    |  2 +-
 .../GetStorageDomainsSpectraS3Request.java    |  2 +-
 .../GetStorageDomainsSpectraS3Response.java   |  2 +-
 ...spectBlobAzureTargetsSpectraS3Request.java |  2 +-
 ...pectBlobAzureTargetsSpectraS3Response.java |  2 +-
 ...SuspectBlobDs3TargetsSpectraS3Request.java |  2 +-
 ...uspectBlobDs3TargetsSpectraS3Response.java |  2 +-
 .../GetSuspectBlobPoolsSpectraS3Request.java  |  2 +-
 .../GetSuspectBlobPoolsSpectraS3Response.java |  2 +-
 ...tSuspectBlobS3TargetsSpectraS3Request.java |  2 +-
 ...SuspectBlobS3TargetsSpectraS3Response.java |  2 +-
 .../GetSuspectBlobTapesSpectraS3Request.java  |  2 +-
 .../GetSuspectBlobTapesSpectraS3Response.java |  2 +-
 .../GetSuspectBucketsSpectraS3Request.java    |  2 +-
 .../GetSuspectBucketsSpectraS3Response.java   |  2 +-
 .../GetSuspectObjectsSpectraS3Request.java    |  2 +-
 .../GetSuspectObjectsSpectraS3Response.java   |  2 +-
 ...bjectsWithFullDetailsSpectraS3Request.java |  2 +-
 ...jectsWithFullDetailsSpectraS3Response.java |  2 +-
 ...SystemCapacitySummarySpectraS3Request.java |  2 +-
 ...ystemCapacitySummarySpectraS3Response.java |  2 +-
 .../GetSystemFailuresSpectraS3Request.java    |  2 +-
 .../GetSystemFailuresSpectraS3Response.java   |  2 +-
 .../GetSystemInformationSpectraS3Request.java |  2 +-
 ...GetSystemInformationSpectraS3Response.java |  2 +-
 ...tTapeDensityDirectiveSpectraS3Request.java |  2 +-
 ...TapeDensityDirectiveSpectraS3Response.java |  2 +-
 ...TapeDensityDirectivesSpectraS3Request.java |  2 +-
 ...apeDensityDirectivesSpectraS3Response.java |  2 +-
 .../GetTapeDriveSpectraS3Request.java         |  2 +-
 .../GetTapeDriveSpectraS3Response.java        |  2 +-
 .../GetTapeDrivesSpectraS3Request.java        |  2 +-
 .../GetTapeDrivesSpectraS3Response.java       |  2 +-
 .../GetTapeFailuresSpectraS3Request.java      |  2 +-
 .../GetTapeFailuresSpectraS3Response.java     |  2 +-
 .../GetTapeLibrariesSpectraS3Request.java     |  2 +-
 .../GetTapeLibrariesSpectraS3Response.java    |  2 +-
 .../GetTapeLibrarySpectraS3Request.java       |  2 +-
 .../GetTapeLibrarySpectraS3Response.java      |  2 +-
 ...TapePartitionFailuresSpectraS3Request.java |  2 +-
 ...apePartitionFailuresSpectraS3Response.java |  2 +-
 .../GetTapePartitionSpectraS3Request.java     |  2 +-
 .../GetTapePartitionSpectraS3Response.java    |  2 +-
 ...titionWithFullDetailsSpectraS3Request.java |  2 +-
 ...itionWithFullDetailsSpectraS3Response.java |  2 +-
 .../GetTapePartitionsSpectraS3Request.java    |  2 +-
 .../GetTapePartitionsSpectraS3Response.java   |  2 +-
 ...itionsWithFullDetailsSpectraS3Request.java |  2 +-
 ...tionsWithFullDetailsSpectraS3Response.java |  2 +-
 .../spectrads3/GetTapeSpectraS3Request.java   |  2 +-
 .../spectrads3/GetTapeSpectraS3Response.java  |  2 +-
 .../spectrads3/GetTapesSpectraS3Request.java  |  2 +-
 .../spectrads3/GetTapesSpectraS3Response.java |  2 +-
 .../spectrads3/GetUserSpectraS3Request.java   |  2 +-
 .../spectrads3/GetUserSpectraS3Response.java  |  2 +-
 .../spectrads3/GetUsersSpectraS3Request.java  |  2 +-
 .../spectrads3/GetUsersSpectraS3Response.java |  2 +-
 .../ImportAllPoolsSpectraS3Request.java       |  2 +-
 .../ImportAllPoolsSpectraS3Response.java      |  2 +-
 .../ImportAllTapesSpectraS3Request.java       |  2 +-
 .../ImportAllTapesSpectraS3Response.java      |  2 +-
 .../ImportAzureTargetSpectraS3Request.java    |  2 +-
 .../ImportAzureTargetSpectraS3Response.java   |  2 +-
 .../ImportPoolSpectraS3Request.java           |  2 +-
 .../ImportPoolSpectraS3Response.java          |  2 +-
 .../ImportS3TargetSpectraS3Request.java       |  2 +-
 .../ImportS3TargetSpectraS3Response.java      |  2 +-
 .../ImportTapeSpectraS3Request.java           |  2 +-
 .../ImportTapeSpectraS3Response.java          |  2 +-
 .../InspectAllTapesSpectraS3Request.java      |  2 +-
 .../InspectAllTapesSpectraS3Response.java     |  2 +-
 .../InspectTapeSpectraS3Request.java          |  2 +-
 .../InspectTapeSpectraS3Response.java         |  2 +-
 ...zureTargetsAsDegradedSpectraS3Request.java |  2 +-
 ...ureTargetsAsDegradedSpectraS3Response.java |  2 +-
 ...bDs3TargetsAsDegradedSpectraS3Request.java |  2 +-
 ...Ds3TargetsAsDegradedSpectraS3Response.java |  2 +-
 ...ctBlobPoolsAsDegradedSpectraS3Request.java |  2 +-
 ...tBlobPoolsAsDegradedSpectraS3Response.java |  2 +-
 ...obS3TargetsAsDegradedSpectraS3Request.java |  2 +-
 ...bS3TargetsAsDegradedSpectraS3Response.java |  2 +-
 ...ctBlobTapesAsDegradedSpectraS3Request.java |  2 +-
 ...tBlobTapesAsDegradedSpectraS3Response.java |  2 +-
 .../ModifyActiveJobSpectraS3Request.java      |  2 +-
 .../ModifyActiveJobSpectraS3Response.java     |  2 +-
 ...ModifyAllAzureTargetsSpectraS3Request.java |  2 +-
 ...odifyAllAzureTargetsSpectraS3Response.java |  2 +-
 .../ModifyAllDs3TargetsSpectraS3Request.java  |  2 +-
 .../ModifyAllDs3TargetsSpectraS3Response.java |  2 +-
 .../ModifyAllPoolsSpectraS3Request.java       |  2 +-
 .../ModifyAllPoolsSpectraS3Response.java      |  2 +-
 .../ModifyAllS3TargetsSpectraS3Request.java   |  2 +-
 .../ModifyAllS3TargetsSpectraS3Response.java  |  2 +-
 ...difyAllTapePartitionsSpectraS3Request.java |  2 +-
 ...ifyAllTapePartitionsSpectraS3Response.java |  2 +-
 ...reDataReplicationRuleSpectraS3Request.java |  2 +-
 ...eDataReplicationRuleSpectraS3Response.java |  2 +-
 .../ModifyAzureTargetSpectraS3Request.java    |  2 +-
 .../ModifyAzureTargetSpectraS3Response.java   |  2 +-
 .../ModifyBucketSpectraS3Request.java         |  2 +-
 .../ModifyBucketSpectraS3Response.java        |  2 +-
 ...ModifyCacheFilesystemSpectraS3Request.java |  2 +-
 ...odifyCacheFilesystemSpectraS3Response.java |  2 +-
 ...ModifyDataPathBackendSpectraS3Request.java |  2 +-
 ...odifyDataPathBackendSpectraS3Response.java |  2 +-
 ...fyDataPersistenceRuleSpectraS3Request.java |  2 +-
 ...yDataPersistenceRuleSpectraS3Response.java |  2 +-
 .../ModifyDataPolicySpectraS3Request.java     |  2 +-
 .../ModifyDataPolicySpectraS3Response.java    |  2 +-
 ...s3DataReplicationRuleSpectraS3Request.java |  2 +-
 ...3DataReplicationRuleSpectraS3Response.java |  2 +-
 .../ModifyDs3TargetSpectraS3Request.java      |  2 +-
 .../ModifyDs3TargetSpectraS3Response.java     |  2 +-
 .../ModifyGroupSpectraS3Request.java          |  2 +-
 .../ModifyGroupSpectraS3Response.java         |  2 +-
 .../spectrads3/ModifyJobSpectraS3Request.java |  2 +-
 .../ModifyJobSpectraS3Response.java           |  2 +-
 .../ModifyNodeSpectraS3Request.java           |  2 +-
 .../ModifyNodeSpectraS3Response.java          |  2 +-
 .../ModifyPoolPartitionSpectraS3Request.java  |  2 +-
 .../ModifyPoolPartitionSpectraS3Response.java |  2 +-
 .../ModifyPoolSpectraS3Request.java           |  2 +-
 .../ModifyPoolSpectraS3Response.java          |  2 +-
 ...S3DataReplicationRuleSpectraS3Request.java |  2 +-
 ...3DataReplicationRuleSpectraS3Response.java |  2 +-
 .../ModifyS3TargetSpectraS3Request.java       |  2 +-
 .../ModifyS3TargetSpectraS3Response.java      |  2 +-
 ...fyStorageDomainMemberSpectraS3Request.java |  2 +-
 ...yStorageDomainMemberSpectraS3Response.java |  2 +-
 .../ModifyStorageDomainSpectraS3Request.java  |  2 +-
 .../ModifyStorageDomainSpectraS3Response.java |  2 +-
 .../ModifyTapeDriveSpectraS3Request.java      |  2 +-
 .../ModifyTapeDriveSpectraS3Response.java     |  2 +-
 .../ModifyTapePartitionSpectraS3Request.java  |  2 +-
 .../ModifyTapePartitionSpectraS3Response.java |  2 +-
 .../ModifyTapeSpectraS3Request.java           |  2 +-
 .../ModifyTapeSpectraS3Response.java          |  2 +-
 .../ModifyUserSpectraS3Request.java           |  2 +-
 .../ModifyUserSpectraS3Response.java          |  2 +-
 .../OnlineAllTapesSpectraS3Request.java       |  2 +-
 .../OnlineAllTapesSpectraS3Response.java      |  2 +-
 .../OnlineTapeSpectraS3Request.java           |  2 +-
 .../OnlineTapeSpectraS3Response.java          |  2 +-
 ...ckRegisteredDs3TargetSpectraS3Request.java |  2 +-
 ...kRegisteredDs3TargetSpectraS3Response.java |  2 +-
 ...reDataReplicationRuleSpectraS3Request.java |  2 +-
 ...eDataReplicationRuleSpectraS3Response.java |  2 +-
 ...AzureTargetBucketNameSpectraS3Request.java |  2 +-
 ...zureTargetBucketNameSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...eTargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 .../PutBucketAclForGroupSpectraS3Request.java |  2 +-
 ...PutBucketAclForGroupSpectraS3Response.java |  2 +-
 .../PutBucketAclForUserSpectraS3Request.java  |  2 +-
 .../PutBucketAclForUserSpectraS3Response.java |  2 +-
 .../spectrads3/PutBucketSpectraS3Request.java |  2 +-
 .../PutBucketSpectraS3Response.java           |  2 +-
 .../PutBulkJobSpectraS3Request.java           |  2 +-
 .../PutBulkJobSpectraS3Response.java          |  2 +-
 ...utDataPersistenceRuleSpectraS3Request.java |  2 +-
 ...tDataPersistenceRuleSpectraS3Response.java |  2 +-
 ...DataPolicyAclForGroupSpectraS3Request.java |  2 +-
 ...ataPolicyAclForGroupSpectraS3Response.java |  2 +-
 ...tDataPolicyAclForUserSpectraS3Request.java |  2 +-
 ...DataPolicyAclForUserSpectraS3Response.java |  2 +-
 .../PutDataPolicySpectraS3Request.java        |  2 +-
 .../PutDataPolicySpectraS3Response.java       |  2 +-
 ...s3DataReplicationRuleSpectraS3Request.java |  2 +-
 ...3DataReplicationRuleSpectraS3Response.java |  2 +-
 ...3TargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 ...obalBucketAclForGroupSpectraS3Request.java |  2 +-
 ...balBucketAclForGroupSpectraS3Response.java |  2 +-
 ...lobalBucketAclForUserSpectraS3Request.java |  2 +-
 ...obalBucketAclForUserSpectraS3Response.java |  2 +-
 ...DataPolicyAclForGroupSpectraS3Request.java |  2 +-
 ...ataPolicyAclForGroupSpectraS3Response.java |  2 +-
 ...lDataPolicyAclForUserSpectraS3Request.java |  2 +-
 ...DataPolicyAclForUserSpectraS3Response.java |  2 +-
 .../PutGroupGroupMemberSpectraS3Request.java  |  2 +-
 .../PutGroupGroupMemberSpectraS3Response.java |  2 +-
 .../spectrads3/PutGroupSpectraS3Request.java  |  2 +-
 .../spectrads3/PutGroupSpectraS3Response.java |  2 +-
 .../PutPoolPartitionSpectraS3Request.java     |  2 +-
 .../PutPoolPartitionSpectraS3Response.java    |  2 +-
 ...olStorageDomainMemberSpectraS3Request.java |  2 +-
 ...lStorageDomainMemberSpectraS3Response.java |  2 +-
 ...S3DataReplicationRuleSpectraS3Request.java |  2 +-
 ...3DataReplicationRuleSpectraS3Response.java |  2 +-
 ...PutS3TargetBucketNameSpectraS3Request.java |  2 +-
 ...utS3TargetBucketNameSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...3TargetReadPreferenceSpectraS3Request.java |  2 +-
 ...TargetReadPreferenceSpectraS3Response.java |  2 +-
 .../PutStorageDomainSpectraS3Request.java     |  2 +-
 .../PutStorageDomainSpectraS3Response.java    |  2 +-
 ...tTapeDensityDirectiveSpectraS3Request.java |  2 +-
 ...TapeDensityDirectiveSpectraS3Response.java |  2 +-
 ...peStorageDomainMemberSpectraS3Request.java |  2 +-
 ...eStorageDomainMemberSpectraS3Response.java |  2 +-
 .../PutUserGroupMemberSpectraS3Request.java   |  2 +-
 .../PutUserGroupMemberSpectraS3Response.java  |  2 +-
 .../RawImportAllTapesSpectraS3Request.java    |  2 +-
 .../RawImportAllTapesSpectraS3Response.java   |  2 +-
 .../RawImportTapeSpectraS3Request.java        |  2 +-
 .../RawImportTapeSpectraS3Response.java       |  2 +-
 ...generateUserSecretKeySpectraS3Request.java |  2 +-
 ...enerateUserSecretKeySpectraS3Response.java |  2 +-
 .../RegisterAzureTargetSpectraS3Request.java  |  2 +-
 .../RegisterAzureTargetSpectraS3Response.java |  2 +-
 .../RegisterDs3TargetSpectraS3Request.java    |  2 +-
 .../RegisterDs3TargetSpectraS3Response.java   |  2 +-
 .../RegisterS3TargetSpectraS3Request.java     |  2 +-
 .../RegisterS3TargetSpectraS3Response.java    |  2 +-
 .../ReplicatePutJobSpectraS3Request.java      |  2 +-
 .../ReplicatePutJobSpectraS3Response.java     |  2 +-
 ...setInstanceIdentifierSpectraS3Request.java |  2 +-
 ...etInstanceIdentifierSpectraS3Response.java |  2 +-
 .../StageObjectsJobSpectraS3Request.java      |  2 +-
 .../StageObjectsJobSpectraS3Response.java     |  2 +-
 .../TruncateActiveJobSpectraS3Request.java    |  2 +-
 .../TruncateActiveJobSpectraS3Response.java   |  2 +-
 ...TruncateAllActiveJobsSpectraS3Request.java |  2 +-
 ...runcateAllActiveJobsSpectraS3Response.java |  2 +-
 .../TruncateAllJobsSpectraS3Request.java      |  2 +-
 .../TruncateAllJobsSpectraS3Response.java     |  2 +-
 .../TruncateJobSpectraS3Request.java          |  2 +-
 .../TruncateJobSpectraS3Response.java         |  2 +-
 .../UndeleteObjectSpectraS3Request.java       |  2 +-
 .../UndeleteObjectSpectraS3Response.java      |  2 +-
 .../VerifyAllPoolsSpectraS3Request.java       |  2 +-
 .../VerifyAllPoolsSpectraS3Response.java      |  2 +-
 .../VerifyAllTapesSpectraS3Request.java       |  2 +-
 .../VerifyAllTapesSpectraS3Response.java      |  2 +-
 .../VerifyAzureTargetSpectraS3Request.java    |  2 +-
 .../VerifyAzureTargetSpectraS3Response.java   |  2 +-
 .../VerifyBulkJobSpectraS3Request.java        |  2 +-
 .../VerifyBulkJobSpectraS3Response.java       |  2 +-
 .../VerifyDs3TargetSpectraS3Request.java      |  2 +-
 .../VerifyDs3TargetSpectraS3Response.java     |  2 +-
 ...alPlacementForObjectsSpectraS3Request.java |  2 +-
 ...lPlacementForObjectsSpectraS3Response.java |  2 +-
 ...bjectsWithFullDetailsSpectraS3Request.java |  2 +-
 ...jectsWithFullDetailsSpectraS3Response.java |  2 +-
 .../VerifyPoolSpectraS3Request.java           |  2 +-
 .../VerifyPoolSpectraS3Response.java          |  2 +-
 .../VerifyS3TargetSpectraS3Request.java       |  2 +-
 .../VerifyS3TargetSpectraS3Response.java      |  2 +-
 ...ifySafeToCreatePutJobSpectraS3Request.java |  2 +-
 ...fySafeToCreatePutJobSpectraS3Response.java |  2 +-
 .../VerifySystemHealthSpectraS3Request.java   |  2 +-
 .../VerifySystemHealthSpectraS3Response.java  |  2 +-
 .../VerifyTapeSpectraS3Request.java           |  2 +-
 .../VerifyTapeSpectraS3Response.java          |  2 +-
 ...fyUserIsMemberOfGroupSpectraS3Request.java |  2 +-
 ...yUserIsMemberOfGroupSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ficationRegistrationsSpectraS3Request.java |  2 +-
 ...icationRegistrationsSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 ...ificationRegistrationSpectraS3Request.java |  2 +-
 ...ficationRegistrationSpectraS3Response.java |  2 +-
 .../ContentLengthNotMatchException.java       |  2 +-
 .../exceptions/Ds3NoMoreRetriesException.java |  2 +-
 ...FolderNameMissingTrailingForwardSlash.java |  2 +-
 .../exceptions/InvalidCertificate.java        |  2 +-
 .../RetryAfterExpectedException.java          |  2 +-
 ...bleToConvertToCommonPrefixesException.java |  2 +-
 .../UnableToConvertToContentsException.java   |  2 +-
 .../ds3client/helpers/ChecksumFunction.java   |  2 +-
 .../ds3client/helpers/ChecksumListener.java   |  2 +-
 .../helpers/DataTransferredListener.java      |  2 +-
 .../ds3client/helpers/Ds3ClientHelpers.java   |  2 +-
 .../helpers/Ds3ClientHelpersImpl.java         |  2 +-
 .../helpers/ExceptionClassifier.java          |  2 +-
 .../ds3client/helpers/FileObjectGetter.java   |  2 +-
 .../ds3client/helpers/FileObjectPutter.java   |  2 +-
 .../ds3client/helpers/FileSystemHelper.java   |  2 +-
 .../helpers/FileSystemHelperImpl.java         |  2 +-
 .../ds3client/helpers/FolderNameFilter.java   |  2 +-
 .../ds3client/helpers/JobImpl.java            |  2 +-
 .../ds3client/helpers/JobPart.java            |  2 +-
 .../ds3client/helpers/JobPartTracker.java     |  2 +-
 .../helpers/JobPartTrackerFactory.java        |  2 +-
 .../ds3client/helpers/JobPartTrackerImpl.java |  2 +-
 .../helpers/JobRecoveryException.java         |  2 +-
 .../JobRecoveryNotActiveException.java        |  2 +-
 .../helpers/JobRecoveryTypeException.java     |  2 +-
 .../ds3client/helpers/JobState.java           |  2 +-
 .../helpers/ObjectChannelBuilderLogger.java   |  2 +-
 .../helpers/ObjectCompletedListener.java      |  2 +-
 .../ds3client/helpers/ObjectPart.java         |  2 +-
 .../helpers/ObjectPartComparator.java         |  2 +-
 .../ds3client/helpers/ObjectPartTracker.java  |  2 +-
 .../helpers/ObjectPartTrackerImpl.java        |  2 +-
 .../ObjectStorageSpaceVerificationResult.java |  2 +-
 .../ds3client/helpers/ReadJobImpl.java        |  2 +-
 .../helpers/RecoverableIOException.java       |  2 +-
 .../helpers/TruncateNotAllowedException.java  |  2 +-
 .../helpers/UnrecoverableIOException.java     |  2 +-
 .../helpers/WaitingForChunksListener.java     |  2 +-
 .../ds3client/helpers/WriteJobImpl.java       |  2 +-
 .../IoLoggingChannelBuilder.java              |  2 +-
 .../ObjectInputStreamBuilder.java             |  2 +-
 .../ObjectOutputStreamBuilder.java            |  2 +-
 .../PrefixAdderObjectChannelBuilder.java      |  2 +-
 .../PrefixRemoverObjectChannelBuilder.java    |  2 +-
 .../ReadOnlySeekableByteChannel.java          |  2 +-
 .../WriteOnlySeekableByteChannel.java         |  2 +-
 .../helpers/channels/BlobComparator.java      |  2 +-
 .../channels/RangedSeekableByteChannel.java   |  2 +-
 .../channels/WindowedChannelFactory.java      |  2 +-
 .../channels/WindowedSeekableByteChannel.java |  2 +-
 .../helpers/events/ConcurrentEventRunner.java |  2 +-
 .../ds3client/helpers/events/EventRunner.java |  2 +-
 .../helpers/events/MetadataEvent.java         | 20 +++++++++----------
 .../helpers/events/SameThreadEventRunner.java |  2 +-
 .../helpers/options/ReadJobOptions.java       |  2 +-
 .../helpers/options/WriteJobOptions.java      |  2 +-
 .../ds3client/helpers/package-info.java       |  2 +-
 .../helpers/pagination/FileSystemKey.java     |  2 +-
 .../pagination/GetBucketKeyLoader.java        |  2 +-
 .../pagination/GetBucketKeyLoaderFactory.java |  2 +-
 .../pagination/GetBucketLoaderFactory.java    |  2 +-
 .../GetObjectsFullDetailsLoader.java          |  2 +-
 .../GetObjectsFullDetailsLoaderFactory.java   |  2 +-
 .../helpers/pagination/PaginatingCommand.java |  2 +-
 .../pagination/SpectraS3PaginationLoader.java |  2 +-
 ...etObjectsFullDetailsPaginatingCommand.java |  2 +-
 .../helpers/pagination/package-info.java      |  2 +-
 .../helpers/strategy/StrategyUtils.java       |  2 +-
 .../blobstrategy/AbstractBlobStrategy.java    | 20 +++++++++----------
 ...ckPearlChunkAttemptRetryDelayBehavior.java | 20 +++++++++----------
 .../strategy/blobstrategy/BlobStrategy.java   | 20 +++++++++----------
 .../blobstrategy/BlobStrategyMaker.java       | 20 +++++++++----------
 .../ChunkAttemptRetryBehavior.java            | 20 +++++++++----------
 .../ChunkAttemptRetryDelayBehavior.java       | 20 +++++++++----------
 ...DefinedChunkAttemptRetryDelayBehavior.java | 20 +++++++++----------
 ...inueForeverChunkAttemptsRetryBehavior.java | 20 +++++++++----------
 .../GetSequentialBlobStrategy.java            |  2 +-
 .../MaxChunkAttemptsRetryBehavior.java        | 20 +++++++++----------
 .../PutSequentialBlobStrategy.java            |  2 +-
 .../channelstrategy/ChannelPreparable.java    | 20 +++++++++----------
 .../channelstrategy/ChannelStrategy.java      | 20 +++++++++----------
 .../NullChannelPreparable.java                | 20 +++++++++----------
 .../RandomAccessChannelStrategy.java          | 20 +++++++++----------
 .../SeekableByteChannelDecorator.java         | 20 +++++++++----------
 .../SequentialChannelStrategy.java            | 20 +++++++++----------
 .../SequentialFileReaderChannelStrategy.java  | 20 +++++++++----------
 .../SequentialFileWriterChannelStrategy.java  | 20 +++++++++----------
 .../TruncatingChannelPreparable.java          | 20 +++++++++----------
 .../transferstrategy/AbstractObserver.java    | 20 +++++++++----------
 .../AbstractTransferStrategy.java             | 20 +++++++++----------
 .../BlobTransferredEventObserver.java         | 20 +++++++++----------
 .../CanceledEventObserver.java                | 20 +++++++++----------
 .../transferstrategy/ChecksumEvent.java       | 20 +++++++++----------
 .../transferstrategy/ChecksumObserver.java    | 20 +++++++++----------
 ...ContinueForeverTransferRetryDecorator.java | 20 +++++++++----------
 .../DataTransferredObserver.java              | 20 +++++++++----------
 .../transferstrategy/EventDispatcher.java     | 20 +++++++++----------
 .../transferstrategy/EventDispatcherImpl.java | 20 +++++++++----------
 .../FailureEventObserver.java                 | 20 +++++++++----------
 .../GetJobNetworkFailureRetryDecorator.java   | 20 +++++++++----------
 .../GetJobPartialBlobTransferMethod.java      | 20 +++++++++----------
 .../GetJobTransferMethod.java                 | 20 +++++++++----------
 ...MaxNumObjectTransferAttemptsDecorator.java | 20 +++++++++----------
 .../MetaDataReceivedObserver.java             | 20 +++++++++----------
 .../MultiThreadedTransferStrategy.java        | 20 +++++++++----------
 .../ObjectCompletedObserver.java              | 20 +++++++++----------
 .../strategy/transferstrategy/Observer.java   | 20 +++++++++----------
 .../PutJobTransferMethod.java                 | 20 +++++++++----------
 .../transferstrategy/RangeHelper.java         | 20 +++++++++----------
 .../SingleThreadedTransferStrategy.java       | 20 +++++++++----------
 .../transferstrategy/TransferMethod.java      | 20 +++++++++----------
 .../transferstrategy/TransferMethodMaker.java | 20 +++++++++----------
 .../TransferRetryDecorator.java               | 20 +++++++++----------
 .../transferstrategy/TransferStrategy.java    | 20 +++++++++----------
 .../TransferStrategyBuilder.java              | 20 +++++++++----------
 .../transferstrategy/UpdateStrategy.java      | 20 +++++++++----------
 .../WaitingForChunksObserver.java             | 20 +++++++++----------
 .../helpers/util/PartialObjectHelpers.java    |  2 +-
 .../ds3client/models/ActiveJob.java           |  2 +-
 .../ds3client/models/ActiveJobList.java       |  2 +-
 .../ds3client/models/AutoInspectMode.java     |  2 +-
 .../models/AzureDataReplicationRule.java      |  2 +-
 .../models/AzureDataReplicationRuleList.java  |  2 +-
 .../ds3client/models/AzureTarget.java         |  2 +-
 .../models/AzureTargetBucketName.java         |  2 +-
 .../models/AzureTargetBucketNameList.java     |  2 +-
 .../ds3client/models/AzureTargetFailure.java  |  2 +-
 .../models/AzureTargetFailureList.java        |  2 +-
 ...TargetFailureNotificationRegistration.java |  2 +-
 ...etFailureNotificationRegistrationList.java |  2 +-
 .../ds3client/models/AzureTargetList.java     |  2 +-
 .../models/AzureTargetReadPreference.java     |  2 +-
 .../models/AzureTargetReadPreferenceList.java |  2 +-
 .../spectralogic/ds3client/models/Blob.java   |  2 +-
 .../models/BlobStoreTaskInformation.java      |  2 +-
 .../ds3client/models/BlobStoreTaskState.java  |  2 +-
 .../models/BlobStoreTasksInformation.java     |  2 +-
 .../spectralogic/ds3client/models/Bucket.java |  2 +-
 .../ds3client/models/BucketAcl.java           |  2 +-
 .../ds3client/models/BucketAclList.java       |  2 +-
 .../ds3client/models/BucketAclPermission.java |  2 +-
 .../ds3client/models/BucketDetails.java       |  2 +-
 .../ds3client/models/BucketList.java          |  2 +-
 .../ds3client/models/BuildInformation.java    |  2 +-
 .../ds3client/models/BulkObject.java          |  2 +-
 .../ds3client/models/BulkObjectList.java      |  2 +-
 .../models/CacheEntryInformation.java         |  2 +-
 .../ds3client/models/CacheEntryState.java     |  2 +-
 .../ds3client/models/CacheFilesystem.java     |  2 +-
 .../models/CacheFilesystemInformation.java    |  2 +-
 .../ds3client/models/CacheFilesystemList.java |  2 +-
 .../ds3client/models/CacheInformation.java    |  2 +-
 .../ds3client/models/CanceledJob.java         |  2 +-
 .../ds3client/models/CanceledJobList.java     |  2 +-
 .../models/CapacitySummaryContainer.java      |  2 +-
 .../ds3client/models/ChecksumType.java        |  2 +-
 .../models/CompleteMultipartUploadResult.java |  2 +-
 .../ds3client/models/CompletedJob.java        |  2 +-
 .../ds3client/models/CompletedJobList.java    |  2 +-
 .../ds3client/models/Contents.java            |  2 +-
 .../ds3client/models/DataIsolationLevel.java  |  2 +-
 .../ds3client/models/DataPathBackend.java     |  2 +-
 .../ds3client/models/DataPersistenceRule.java |  2 +-
 .../models/DataPersistenceRuleList.java       |  2 +-
 .../models/DataPersistenceRuleType.java       |  2 +-
 .../models/DataPlacementRuleState.java        |  2 +-
 .../ds3client/models/DataPolicy.java          |  2 +-
 .../ds3client/models/DataPolicyAcl.java       |  2 +-
 .../ds3client/models/DataPolicyAclList.java   |  2 +-
 .../ds3client/models/DataPolicyList.java      |  2 +-
 .../models/DataReplicationRuleType.java       |  2 +-
 .../models/DatabasePhysicalSpaceState.java    |  2 +-
 .../ds3client/models/DegradedBlob.java        |  2 +-
 .../ds3client/models/DegradedBlobList.java    |  2 +-
 .../ds3client/models/DeleteObjectError.java   |  2 +-
 .../ds3client/models/DeleteResult.java        |  2 +-
 .../ds3client/models/DetailedS3Object.java    |  2 +-
 .../models/DetailedS3ObjectList.java          |  2 +-
 .../ds3client/models/DetailedTapeFailure.java |  2 +-
 .../models/DetailedTapeFailureList.java       |  2 +-
 .../models/DetailedTapePartition.java         |  2 +-
 .../models/Ds3DataReplicationRule.java        |  2 +-
 .../models/Ds3DataReplicationRuleList.java    |  2 +-
 .../ds3client/models/Ds3Target.java           |  2 +-
 .../Ds3TargetAccessControlReplication.java    |  2 +-
 .../ds3client/models/Ds3TargetFailure.java    |  2 +-
 .../models/Ds3TargetFailureList.java          |  2 +-
 ...TargetFailureNotificationRegistration.java |  2 +-
 ...etFailureNotificationRegistrationList.java |  2 +-
 .../ds3client/models/Ds3TargetList.java       |  2 +-
 .../models/Ds3TargetReadPreference.java       |  2 +-
 .../models/Ds3TargetReadPreferenceList.java   |  2 +-
 .../spectralogic/ds3client/models/Error.java  |  2 +-
 .../ds3client/models/FeatureKey.java          |  2 +-
 .../ds3client/models/FeatureKeyList.java      |  2 +-
 .../ds3client/models/FeatureKeyType.java      |  2 +-
 .../spectralogic/ds3client/models/Group.java  |  2 +-
 .../ds3client/models/GroupList.java           |  2 +-
 .../ds3client/models/GroupMember.java         |  2 +-
 .../ds3client/models/GroupMemberList.java     |  2 +-
 .../models/HealthVerificationResult.java      |  2 +-
 .../models/HttpResponseFormatType.java        |  2 +-
 .../models/ImportExportConfiguration.java     |  2 +-
 .../models/InitiateMultipartUploadResult.java |  2 +-
 .../spectralogic/ds3client/models/Job.java    |  2 +-
 .../ds3client/models/JobChunk.java            |  2 +-
 .../models/JobChunkBlobStoreState.java        |  2 +-
 ...obChunkClientProcessingOrderGuarantee.java |  2 +-
 .../JobCompletedNotificationRegistration.java |  2 +-
 ...CompletedNotificationRegistrationList.java |  2 +-
 .../JobCreatedNotificationRegistration.java   |  2 +-
 ...obCreatedNotificationRegistrationList.java |  2 +-
 ...reationFailedNotificationRegistration.java |  2 +-
 ...ionFailedNotificationRegistrationList.java |  2 +-
 .../ds3client/models/JobList.java             |  2 +-
 .../ds3client/models/JobNode.java             |  2 +-
 .../ds3client/models/JobRequestType.java      |  2 +-
 .../ds3client/models/JobRestore.java          |  2 +-
 .../ds3client/models/JobStatus.java           |  2 +-
 .../models/ListAllMyBucketsResult.java        |  2 +-
 .../ds3client/models/ListBucketResult.java    |  2 +-
 .../models/ListMultiPartUploadsResult.java    |  2 +-
 .../ds3client/models/ListPartsResult.java     |  2 +-
 .../ds3client/models/LtfsFileNamingMode.java  |  2 +-
 .../ds3client/models/MasterObjectList.java    |  2 +-
 .../ds3client/models/MultiPartUpload.java     |  2 +-
 .../ds3client/models/MultiPartUploadPart.java |  2 +-
 .../models/NamedDetailedTapePartition.java    |  2 +-
 .../NamedDetailedTapePartitionList.java       |  2 +-
 .../models/NamingConventionType.java          |  2 +-
 .../spectralogic/ds3client/models/Node.java   |  2 +-
 .../ds3client/models/NodeList.java            |  2 +-
 .../ds3client/models/Objects.java             |  2 +-
 .../ds3client/models/PhysicalPlacement.java   |  2 +-
 .../spectralogic/ds3client/models/Pool.java   |  2 +-
 .../ds3client/models/PoolFailure.java         |  2 +-
 .../ds3client/models/PoolFailureList.java     |  2 +-
 .../PoolFailureNotificationRegistration.java  |  2 +-
 ...olFailureNotificationRegistrationList.java |  2 +-
 .../ds3client/models/PoolFailureType.java     |  2 +-
 .../ds3client/models/PoolHealth.java          |  2 +-
 .../ds3client/models/PoolList.java            |  2 +-
 .../ds3client/models/PoolPartition.java       |  2 +-
 .../ds3client/models/PoolPartitionList.java   |  2 +-
 .../ds3client/models/PoolState.java           |  2 +-
 .../ds3client/models/PoolType.java            |  2 +-
 .../ds3client/models/Priority.java            |  2 +-
 .../ds3client/models/Quiesced.java            |  2 +-
 .../ds3client/models/RequestType.java         |  2 +-
 .../ds3client/models/ReservedTaskType.java    |  2 +-
 .../ds3client/models/RestOperationType.java   |  2 +-
 .../models/S3DataReplicationRule.java         |  2 +-
 .../models/S3DataReplicationRuleList.java     |  2 +-
 .../models/S3InitialDataPlacementPolicy.java  |  2 +-
 .../ds3client/models/S3Object.java            |  2 +-
 ...3ObjectCachedNotificationRegistration.java |  2 +-
 ...ectCachedNotificationRegistrationList.java |  2 +-
 .../ds3client/models/S3ObjectList.java        |  2 +-
 .../S3ObjectLostNotificationRegistration.java |  2 +-
 ...bjectLostNotificationRegistrationList.java |  2 +-
 ...jectPersistedNotificationRegistration.java |  2 +-
 ...PersistedNotificationRegistrationList.java |  2 +-
 .../ds3client/models/S3ObjectToDelete.java    |  2 +-
 .../ds3client/models/S3ObjectType.java        |  2 +-
 .../ds3client/models/S3Region.java            |  2 +-
 .../ds3client/models/S3Target.java            |  2 +-
 .../ds3client/models/S3TargetBucketName.java  |  2 +-
 .../models/S3TargetBucketNameList.java        |  2 +-
 .../ds3client/models/S3TargetFailure.java     |  2 +-
 .../ds3client/models/S3TargetFailureList.java |  2 +-
 ...TargetFailureNotificationRegistration.java |  2 +-
 ...etFailureNotificationRegistrationList.java |  2 +-
 .../ds3client/models/S3TargetList.java        |  2 +-
 .../models/S3TargetReadPreference.java        |  2 +-
 .../models/S3TargetReadPreferenceList.java    |  2 +-
 .../ds3client/models/SpectraUser.java         |  2 +-
 .../ds3client/models/SpectraUserList.java     |  2 +-
 .../ds3client/models/StorageDomain.java       |  2 +-
 .../models/StorageDomainCapacitySummary.java  |  2 +-
 .../models/StorageDomainFailure.java          |  2 +-
 .../models/StorageDomainFailureList.java      |  2 +-
 ...DomainFailureNotificationRegistration.java |  2 +-
 ...inFailureNotificationRegistrationList.java |  2 +-
 .../models/StorageDomainFailureType.java      |  2 +-
 .../ds3client/models/StorageDomainList.java   |  2 +-
 .../ds3client/models/StorageDomainMember.java |  2 +-
 .../models/StorageDomainMemberList.java       |  2 +-
 .../models/StorageDomainMemberState.java      |  2 +-
 .../models/SuspectBlobAzureTarget.java        |  2 +-
 .../models/SuspectBlobAzureTargetList.java    |  2 +-
 .../models/SuspectBlobDs3Target.java          |  2 +-
 .../models/SuspectBlobDs3TargetList.java      |  2 +-
 .../ds3client/models/SuspectBlobPool.java     |  2 +-
 .../ds3client/models/SuspectBlobPoolList.java |  2 +-
 .../ds3client/models/SuspectBlobS3Target.java |  2 +-
 .../models/SuspectBlobS3TargetList.java       |  2 +-
 .../ds3client/models/SuspectBlobTape.java     |  2 +-
 .../ds3client/models/SuspectBlobTapeList.java |  2 +-
 .../ds3client/models/SystemFailure.java       |  2 +-
 .../ds3client/models/SystemFailureList.java   |  2 +-
 ...SystemFailureNotificationRegistration.java |  2 +-
 ...emFailureNotificationRegistrationList.java |  2 +-
 .../ds3client/models/SystemFailureType.java   |  2 +-
 .../ds3client/models/SystemInformation.java   |  2 +-
 .../spectralogic/ds3client/models/Tape.java   |  2 +-
 .../models/TapeDensityDirective.java          |  2 +-
 .../models/TapeDensityDirectiveList.java      |  2 +-
 .../ds3client/models/TapeDrive.java           |  2 +-
 .../ds3client/models/TapeDriveList.java       |  2 +-
 .../ds3client/models/TapeDriveState.java      |  2 +-
 .../ds3client/models/TapeDriveType.java       |  2 +-
 .../ds3client/models/TapeFailure.java         |  2 +-
 .../ds3client/models/TapeFailureList.java     |  2 +-
 .../TapeFailureNotificationRegistration.java  |  2 +-
 ...peFailureNotificationRegistrationList.java |  2 +-
 .../ds3client/models/TapeFailureType.java     |  2 +-
 .../ds3client/models/TapeLibrary.java         |  2 +-
 .../ds3client/models/TapeLibraryList.java     |  2 +-
 .../ds3client/models/TapeList.java            |  2 +-
 .../ds3client/models/TapePartition.java       |  2 +-
 .../models/TapePartitionFailure.java          |  2 +-
 .../models/TapePartitionFailureList.java      |  2 +-
 ...titionFailureNotificationRegistration.java |  2 +-
 ...onFailureNotificationRegistrationList.java |  2 +-
 .../models/TapePartitionFailureType.java      |  2 +-
 .../ds3client/models/TapePartitionList.java   |  2 +-
 .../ds3client/models/TapePartitionState.java  |  2 +-
 .../ds3client/models/TapeState.java           |  2 +-
 .../ds3client/models/TargetFailureType.java   |  2 +-
 .../models/TargetReadPreferenceType.java      |  2 +-
 .../ds3client/models/TargetState.java         |  2 +-
 .../models/UnavailableMediaUsagePolicy.java   |  2 +-
 .../spectralogic/ds3client/models/User.java   |  2 +-
 .../ds3client/models/VersioningLevel.java     |  2 +-
 .../ds3client/models/WriteOptimization.java   |  2 +-
 .../models/WritePreferenceLevel.java          |  2 +-
 .../ds3client/models/bulk/Ds3Object.java      |  2 +-
 .../ds3client/models/bulk/Ds3ObjectList.java  |  2 +-
 .../models/bulk/PartialDs3Object.java         |  2 +-
 .../ds3client/models/bulk/RequestType.java    |  2 +-
 .../models/common/CommonPrefixes.java         |  2 +-
 .../ds3client/models/common/Credentials.java  |  2 +-
 .../ds3client/models/common/Range.java        |  2 +-
 .../models/common/SignatureDetails.java       |  2 +-
 .../ds3client/models/delete/Delete.java       |  2 +-
 .../ds3client/models/delete/DeleteObject.java |  2 +-
 .../multipart/CompleteMultipartUpload.java    |  2 +-
 .../ds3client/models/multipart/Part.java      |  2 +-
 .../ds3client/models/package-info.java        |  2 +-
 .../ds3client/models/requestpayloads/Ids.java |  2 +-
 .../networking/ConnectionDetails.java         |  2 +-
 .../networking/FailedRequestException.java    |  2 +-
 .../FailedRequestUsingMgmtPortException.java  |  2 +-
 .../FailedToGetBucketException.java           |  2 +-
 .../HashGeneratingMatchHandler.java           |  2 +-
 .../ds3client/networking/HeadersImpl.java     |  2 +-
 .../ds3client/networking/HttpVerb.java        |  2 +-
 .../ds3client/networking/NetUtils.java        |  2 +-
 .../ds3client/networking/NetworkClient.java   |  2 +-
 .../networking/NetworkClientImpl.java         |  2 +-
 .../RequiresMarkSupportedException.java       |  2 +-
 .../ResponseProcessingException.java          |  2 +-
 .../networking/TooManyRedirectsException.java |  2 +-
 .../networking/TooManyRetriesException.java   |  2 +-
 .../ds3client/networking/WebResponse.java     |  2 +-
 .../ds3client/networking/WebResponseImpl.java |  2 +-
 .../ds3client/networking/package-info.java    |  2 +-
 .../spectralogic/ds3client/package-info.java  |  2 +-
 .../ds3client/serializer/Views.java           | 15 ++++++++++++++
 .../ds3client/serializer/XmlOutput.java       |  2 +-
 .../serializer/XmlProcessingException.java    |  2 +-
 .../ds3client/serializer/package-info.java    |  2 +-
 .../spectralogic/ds3client/utils/Builder.java |  2 +-
 .../utils/ByteArraySeekableByteChannel.java   |  2 +-
 .../ds3client/utils/DateFormatter.java        |  2 +-
 .../utils/EmptySeekableByteChannel.java       |  2 +-
 .../spectralogic/ds3client/utils/IOUtils.java |  2 +-
 .../ds3client/utils/InvalidMd5Exception.java  |  2 +-
 .../ds3client/utils/JobUtils.java             |  2 +-
 .../utils/LoggingSeekableByteChannel.java     |  2 +-
 .../ds3client/utils/MultiMap.java             |  2 +-
 .../ds3client/utils/MultiMapImpl.java         |  2 +-
 .../utils/NotImplementedException.java        |  2 +-
 .../ds3client/utils/PerformanceUtils.java     |  2 +-
 .../ds3client/utils/Predicate.java            |  2 +-
 .../ds3client/utils/PropertyUtils.java        |  2 +-
 .../ds3client/utils/ResponseUtils.java        |  2 +-
 .../ds3client/utils/SSLSetupException.java    |  2 +-
 .../utils/SafeStringManipulation.java         |  2 +-
 .../utils/SeekableByteChannelInputStream.java |  2 +-
 .../ds3client/utils/Signature.java            |  2 +-
 .../utils/collections/LazyIterable.java       |  2 +-
 .../utils/collections/StreamWrapper.java      |  2 +-
 .../ds3client/utils/hashing/CRC32CHasher.java |  2 +-
 .../ds3client/utils/hashing/CRC32Hasher.java  |  2 +-
 .../utils/hashing/ChecksumHasher.java         |  2 +-
 .../utils/hashing/ChecksumUtils.java          | 20 +++++++++----------
 .../ds3client/utils/hashing/Crc32c.java       |  2 +-
 .../ds3client/utils/hashing/DigestHasher.java |  2 +-
 .../ds3client/utils/hashing/Hasher.java       |  2 +-
 .../ds3client/utils/hashing/MD5Hasher.java    |  2 +-
 .../ds3client/utils/hashing/Md5Hash.java      |  2 +-
 .../ds3client/utils/hashing/SHA256Hasher.java |  2 +-
 .../ds3client/utils/hashing/SHA512Hasher.java |  2 +-
 .../ds3client/utils/package-info.java         |  2 +-
 .../ds3client/helpers/DeleteBucket.kt         |  2 +-
 .../ds3client/helpers/FileTreeWalker.kt       |  2 +-
 .../utils/collections/IterableExtensions.kt   |  2 +-
 .../utils/collections/WindowedIterator.kt     |  2 +-
 .../ds3client/ConnectionFixture.java          |  2 +-
 .../ds3client/Ds3ClientBuilder_Test.java      |  2 +-
 .../ds3client/Ds3Client_Test.java             |  2 +-
 .../spectralogic/ds3client/MockNetwork.java   |  2 +-
 .../spectralogic/ds3client/MockedHeaders.java |  2 +-
 .../ds3client/MockedWebResponse.java          |  2 +-
 .../commands/GetObjectRequest_Test.java       |  2 +-
 .../ds3client/commands/MetadataImpl_Test.java |  2 +-
 .../SeekableByteChannelInputStream_Test.java  |  2 +-
 .../interfaces/RequestHeadersImpl_Test.java   |  2 +-
 .../utils/ResponseParserUtils_Test.java       |  2 +-
 .../spectrads3/GetPutJobToReplicate_Test.java |  2 +-
 .../helpers/Ds3ClientHelpers_Test.java        |  2 +-
 .../helpers/ExceptionClassifier_Test.java     |  2 +-
 .../helpers/FileObjectGetter_Test.java        |  2 +-
 .../helpers/FileObjectPutter_Test.java        |  2 +-
 .../helpers/FolderNameFilter_Test.java        |  2 +-
 .../helpers/JobPartTrackerFactory_Test.java   |  2 +-
 .../helpers/JobPartTracker_Test.java          |  2 +-
 .../ds3client/helpers/Metadata_Test.java      |  2 +-
 .../helpers/ObjectPartComparator_Test.java    |  2 +-
 ...jectPartTrackerImpl_CompletePart_Test.java |  2 +-
 .../helpers/ObjectPartTrackerImpl_Test.java   |  2 +-
 .../ds3client/helpers/ObjectPart_Test.java    |  2 +-
 .../ds3client/helpers/RequestMatchers.java    |  2 +-
 .../ds3client/helpers/ResponseBuilders.java   |  2 +-
 .../ObjectInputStreamBuilder_Test.java        |  2 +-
 .../ObjectOutputStreamBuilder_Test.java       |  2 +-
 .../channelbuilders/StreamObjectGetter.java   |  2 +-
 .../channelbuilders/StreamObjectPutter.java   |  2 +-
 .../RangedSeekableByteChannel_Test.java       |  2 +-
 .../channels/WindowedChannelFactory_Test.java |  2 +-
 .../WindowedSeekableByteChannel_Test.java     |  2 +-
 .../SpectraS3PaginationLoader_Test.java       |  2 +-
 .../helpers/pagination/StubbedRequest.java    |  2 +-
 .../RangeHelperImpl_Test.java                 |  2 +-
 .../TransferStrategy_Test.java                |  2 +-
 .../util/PartialObjectHelpers_Test.java       |  2 +-
 .../models/ModelFunctionality_Test.java       |  2 +-
 .../ds3client/models/ModelParsing_Test.java   |  2 +-
 .../ds3client/models/ServiceList_Test.java    |  2 +-
 .../models/common/Credential_Test.java        |  2 +-
 .../FailedRequestException_Test.java          |  2 +-
 .../networking/HeadersImpl_Test.java          |  2 +-
 .../ds3client/serializer/XmlOutput_Test.java  |  2 +-
 .../ByteArraySeekableByteChannel_Test.java    |  2 +-
 .../ds3client/utils/DateFormatter_Test.java   |  2 +-
 .../ds3client/utils/Guard_Test.java           |  2 +-
 .../ds3client/utils/JobUtils_Test.java        |  2 +-
 .../ds3client/utils/NetUtils_Test.java        |  2 +-
 .../utils/SafeString_Manipulation_Test.java   |  2 +-
 .../ds3client/utils/Signature_Test.java       |  2 +-
 .../ds3client/utils/hashing/Md5Hash_Test.java |  2 +-
 .../utils/hashing/Sha256Hash_Test.java        |  2 +-
 .../utils/hashing/Sha512Hash_Test.java        |  2 +-
 .../ds3client/helpers/FileTreeWalker_Test.kt  |  2 +-
 .../collections/WindowedIterator_Test.kt      |  2 +-
 ds3-utils/build.gradle                        |  2 +-
 .../commands/interfaces/MetadataImpl.java     |  2 +-
 .../exceptions/AggregateException.java        | 20 +++++++++----------
 .../helpers/FailureEventListener.java         |  2 +-
 .../helpers/events/FailureEvent.java          |  2 +-
 .../ds3client/utils/FileUtils.java            |  2 +-
 .../spectralogic/ds3client/utils/Guard.java   |  2 +-
 .../utils/MetadataStringManipulation.java     |  2 +-
 .../ds3client/utils/Platform.java             |  2 +-
 .../ds3client/utils/ResourceUtils.java        |  2 +-
 .../ds3client/utils/StringExtensions.java     | 20 +++++++++----------
 .../exceptions/AggregateException_Test.java   | 20 +++++++++----------
 .../MetadataStringManipulation_Test.java      |  2 +-
 gradle.properties                             | 15 ++++++++++++++
 gradle/scripts/publish.gradle                 | 15 ++++++++++++++
 gradle/wrapper/gradle-wrapper.properties      | 15 ++++++++++++++
 settings.gradle                               | 20 +++++++++----------
 1702 files changed, 2272 insertions(+), 2182 deletions(-)

diff --git a/build.gradle b/build.gradle
index a89a6debf..c2af3936c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,16 +1,16 @@
 /*
- * ****************************************************************************
- *    Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
- *    Licensed under the Apache License, Version 2.0 (the "License"). You may not use
- *    this file except in compliance with the License. A copy of the License is located at
+ * ******************************************************************************
+ *   Copyright 2014-2019 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- *    or in the "license" file accompanying this file.
- *    This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- *    CONDITIONS OF ANY KIND, either express or implied. See the License for the
- *    specific language governing permissions and limitations under the License.
- *  ****************************************************************************
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
  */
 
 buildscript {
diff --git a/contract/5_0_x_contract.xml b/contract/5_0_x_contract.xml
index 156449ff5..5ee343c72 100644
--- a/contract/5_0_x_contract.xml
+++ b/contract/5_0_x_contract.xml
@@ -1,3 +1,18 @@
+
+