Skip to content

Commit 08bbe72

Browse files
ctruedenctrueden
committed
DownloadService: add the ability to cache the data
1 parent 71ef6d9 commit 08bbe72

6 files changed

Lines changed: 560 additions & 19 deletions

File tree

‎src/main/java/org/scijava/download/DefaultDownloadService.java‎

Lines changed: 112 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
package org.scijava.download;
3434

3535
import java.io.IOException;
36+
import java.util.Date;
3637

3738
import org.scijava.io.handle.DataHandle;
3839
import org.scijava.io.handle.DataHandleService;
@@ -63,24 +64,7 @@ public class DefaultDownloadService extends AbstractService implements
6364
@Override
6465
public Download download(final Location source, final Location destination) {
6566
final Task task = taskService.createTask("Download");
66-
final Download download = new Download() {
67-
68-
@Override
69-
public Location source() {
70-
return source;
71-
}
72-
73-
@Override
74-
public Location destination() {
75-
return destination;
76-
}
77-
78-
@Override
79-
public Task task() {
80-
return task;
81-
}
82-
};
83-
task.run(() -> {
67+
return new DefaultDownload(source, destination, task, () -> {
8468
try (final DataHandle<Location> in = dataHandleService.create(source);
8569
final DataHandle<Location> out = dataHandleService.create(
8670
destination))
@@ -95,7 +79,45 @@ public Task task() {
9579
throw new RuntimeException(exc);
9680
}
9781
});
98-
return download;
82+
}
83+
84+
@Override
85+
public Download download(final Location source, final Location destination,
86+
final LocationCache cache)
87+
{
88+
if (cache == null || !cache.canCache(source)) {
89+
// Caching this location is not supported.
90+
return download(source, destination);
91+
}
92+
93+
final Task task = taskService.createTask("Download");
94+
return new DefaultDownload(source, destination, task, () -> {
95+
final Location cached = cache.cachedLocation(source);
96+
try (
97+
final DataHandle<Location> sourceHandle = dataHandleService.create(source);
98+
final DataHandle<Location> cachedHandle = dataHandleService.create(cached);
99+
final DataHandle<Location> destHandle = dataHandleService.create(destination)
100+
)
101+
{
102+
if (isCachedHandleValid(source, cache, sourceHandle, cachedHandle)) {
103+
// The data is cached; download from the cached source instead.
104+
task.setStatusMessage("Retrieving " + source.getURI());
105+
copy(task, cachedHandle, destHandle);
106+
}
107+
else {
108+
// Data is not yet cached; write to the destination _and_ the cache.
109+
task.setStatusMessage("Downloading + caching " + source.getURI());
110+
copy(task, sourceHandle, //
111+
new MultiWriteHandle(cachedHandle, destHandle));
112+
}
113+
}
114+
catch (final IOException exc) {
115+
// TODO: Improve error handling:
116+
// 1. Consider a better exception handling design here.
117+
// 2. Retry at least a few times if something goes wrong.
118+
throw new RuntimeException(exc);
119+
}
120+
});
99121
}
100122

101123
// -- Helper methods --
@@ -124,4 +146,75 @@ private void copy(final Task task, final DataHandle<Location> in,
124146
if (length > 0) task.setProgressValue(task.getProgressValue() + r);
125147
}
126148
}
149+
150+
private boolean isCachedHandleValid(final Location source,
151+
final LocationCache cache, final DataHandle<Location> sourceHandle,
152+
final DataHandle<Location> cachedHandle) throws IOException
153+
{
154+
if (!cachedHandle.exists()) return false; // No cached data is present.
155+
156+
// Compare data lengths.
157+
final long sourceLen = sourceHandle.length();
158+
final long cachedLen = cachedHandle.length();
159+
if (sourceLen >= 0 && cachedLen >= 0 && sourceLen != cachedLen) {
160+
// Original and cached sources report different lengths; cache is invalid.
161+
return false;
162+
}
163+
164+
// Compare last modified timestamps.
165+
final Date sourceDate = sourceHandle.lastModified();
166+
final Date cachedDate = cachedHandle.lastModified();
167+
if (sourceDate != null && cachedDate != null && //
168+
sourceDate.after(cachedDate))
169+
{
170+
// Source was changed after cache was written; cache is invalid.
171+
return false;
172+
}
173+
174+
// Compare checksums.
175+
final String sourceChecksum = sourceHandle.checksum();
176+
final String cachedChecksum = cache.loadChecksum(source);
177+
if (sourceChecksum != null && cachedChecksum != null && //
178+
!sourceChecksum.equals(cachedChecksum))
179+
{
180+
// Checksums do not match; cache is invalid.
181+
return false;
182+
}
183+
184+
// Everything matched; we're all good.
185+
return true;
186+
}
187+
188+
// -- Helper classes --
189+
190+
private class DefaultDownload implements Download {
191+
192+
private Location source;
193+
private Location destination;
194+
private Task task;
195+
196+
private DefaultDownload(final Location source, final Location destination,
197+
final Task task, final Runnable r)
198+
{
199+
this.source = source;
200+
this.destination = destination;
201+
this.task = task;
202+
task.run(r);
203+
}
204+
205+
@Override
206+
public Location source() {
207+
return source;
208+
}
209+
210+
@Override
211+
public Location destination() {
212+
return destination;
213+
}
214+
215+
@Override
216+
public Task task() {
217+
return task;
218+
}
219+
}
127220
}

‎src/main/java/org/scijava/download/DiskLocationCache.java‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*-
2+
* #%L
3+
* SciJava Common shared library for SciJava software.
4+
* %%
5+
* Copyright (C) 2009 - 2017 Board of Regents of the University of
6+
* Wisconsin-Madison, Broad Institute of MIT and Harvard, Max Planck
7+
* Institute of Molecular Cell Biology and Genetics, University of
8+
* Konstanz, and KNIME GmbH.
9+
* %%
10+
* Redistribution and use in source and binary forms, with or without
11+
* modification, are permitted provided that the following conditions are met:
12+
*
13+
* 1. Redistributions of source code must retain the above copyright notice,
14+
* this list of conditions and the following disclaimer.
15+
* 2. Redistributions in binary form must reproduce the above copyright notice,
16+
* this list of conditions and the following disclaimer in the documentation
17+
* and/or other materials provided with the distribution.
18+
*
19+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
23+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29+
* POSSIBILITY OF SUCH DAMAGE.
30+
* #L%
31+
*/
32+
33+
package org.scijava.download;
34+
35+
import java.io.File;
36+
import java.io.IOException;
37+
38+
import org.scijava.io.location.FileLocation;
39+
import org.scijava.io.location.Location;
40+
import org.scijava.util.DigestUtils;
41+
import org.scijava.util.FileUtils;
42+
43+
/**
44+
* A file-based implementation of {@link LocationCache}.
45+
*
46+
* @author Curtis Rueden
47+
*/
48+
public class DiskLocationCache implements LocationCache {
49+
50+
private File baseDir = new File(System.getProperty("user.home") +
51+
File.separator + ".scijava" + File.separator + "cache" + File.separator);
52+
53+
private boolean cacheFileLocations;
54+
55+
// -- DiskLocationCache methods --
56+
57+
public File getBaseDirectory() {
58+
return baseDir;
59+
}
60+
61+
public void setBaseDirectory(final File baseDir) {
62+
if (!baseDir.isDirectory()) {
63+
throw new IllegalArgumentException("Not a directory: " + baseDir);
64+
}
65+
this.baseDir = baseDir;
66+
}
67+
68+
public boolean isFileLocationCachingEnabled() {
69+
return cacheFileLocations;
70+
}
71+
72+
public void setFileLocationCachingEnabled(final boolean enabled) {
73+
// NB: It is possible the input file is stored on a volume which is much
74+
// slower than the local disk cache, so we make this setting configurable.
75+
cacheFileLocations = enabled;
76+
}
77+
78+
// -- LocationCache methods --
79+
80+
@Override
81+
public boolean canCache(final Location source) {
82+
if (source instanceof FileLocation && !isFileLocationCachingEnabled()) {
83+
// The cache is not configured to cache files to other files.
84+
return false;
85+
}
86+
return source.getURI() != null;
87+
}
88+
89+
@Override
90+
public Location cachedLocation(final Location source) {
91+
if (!canCache(source)) {
92+
throw new IllegalArgumentException("Uncacheable source: " + source);
93+
}
94+
return new FileLocation(cachedData(source));
95+
}
96+
97+
@Override
98+
public String loadChecksum(final Location source) throws IOException {
99+
final File cachedChecksum = cachedChecksum(source);
100+
if (!cachedChecksum.exists()) return null;
101+
return DigestUtils.string(FileUtils.readFile(cachedChecksum));
102+
}
103+
104+
@Override
105+
public void saveChecksum(final Location source, final String checksum)
106+
throws IOException
107+
{
108+
final File cachedChecksum = cachedChecksum(source);
109+
FileUtils.writeFile(cachedChecksum, DigestUtils.bytes(checksum));
110+
}
111+
112+
// -- Helper methods --
113+
114+
private File cachedData(final Location source) {
115+
return cachedFile(source, ".data");
116+
}
117+
118+
private File cachedChecksum(final Location source) {
119+
return cachedFile(source, ".checksum");
120+
}
121+
122+
private File cachedFile(final Location source, final String suffix) {
123+
final String hexCode = Integer.toHexString(source.hashCode());
124+
return new File(getBaseDirectory(), hexCode + suffix);
125+
}
126+
}

‎src/main/java/org/scijava/download/DownloadService.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,16 @@ public interface DownloadService extends SciJavaService {
5050
* @param destination The location where the needed data should be stored.
5151
*/
5252
Download download(Location source, Location destination);
53+
54+
/**
55+
* Downloads data from the given source, storing it into the given
56+
* destination.
57+
*
58+
* @param source The location of the needed data.
59+
* @param destination The location where the needed data should be stored.
60+
* @param cache The cache from which already-downloaded data should be pulled
61+
* preferentially, and to which newly-downloaded data should be
62+
* stored for next time.
63+
*/
64+
Download download(Location source, Location destination, LocationCache cache);
5365
}

‎src/main/java/org/scijava/download/LocationCache.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*-
2+
* #%L
3+
* SciJava Common shared library for SciJava software.
4+
* %%
5+
* Copyright (C) 2009 - 2017 Board of Regents of the University of
6+
* Wisconsin-Madison, Broad Institute of MIT and Harvard, Max Planck
7+
* Institute of Molecular Cell Biology and Genetics, University of
8+
* Konstanz, and KNIME GmbH.
9+
* %%
10+
* Redistribution and use in source and binary forms, with or without
11+
* modification, are permitted provided that the following conditions are met:
12+
*
13+
* 1. Redistributions of source code must retain the above copyright notice,
14+
* this list of conditions and the following disclaimer.
15+
* 2. Redistributions in binary form must reproduce the above copyright notice,
16+
* this list of conditions and the following disclaimer in the documentation
17+
* and/or other materials provided with the distribution.
18+
*
19+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
23+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29+
* POSSIBILITY OF SUCH DAMAGE.
30+
* #L%
31+
*/
32+
33+
package org.scijava.download;
34+
35+
import java.io.IOException;
36+
37+
import org.scijava.io.handle.DataHandle;
38+
import org.scijava.io.location.Location;
39+
40+
/**
41+
* An object which knows how to convert a slow (typically remote)
42+
* {@link Location} to a faster (typically local) one.
43+
*
44+
* @author Curtis Rueden
45+
*/
46+
public interface LocationCache {
47+
48+
/** Gets whether the given location can be cached by this cache. */
49+
boolean canCache(Location source);
50+
51+
/**
52+
* Gets the cache location of a given data source.
53+
*
54+
* @return A {@link Location} where the source data is, or would be, cached.
55+
* @throws IllegalArgumentException if the given source cannot be cached (see
56+
* {@link #canCache}).
57+
*/
58+
Location cachedLocation(Location source);
59+
60+
/**
61+
* Loads the checksum value which corresponds to the cached location.
62+
*
63+
* @param source The source location for which the cached checksum is desired.
64+
* @return The loaded checksum, or null if one is not available.
65+
* @see DataHandle#checksum()
66+
* @throws IOException If something goes wrong accessing the checksum.
67+
*/
68+
String loadChecksum(Location source) throws IOException;
69+
70+
/**
71+
* Associates the given checksum value with the specified source location.
72+
*
73+
* @param source The source location for which the checksum should be cached.
74+
* @param checksum The checksum value to cache.
75+
* @see DataHandle#checksum()
76+
* @throws IOException If something goes wrong caching the checksum.
77+
*/
78+
void saveChecksum(Location source, String checksum) throws IOException;
79+
}

0 commit comments

Comments
 (0)