Skip to content

Commit faa55e9

Browse files
sadam21sadam21
committed
add batch upload support
1 parent b9712b0 commit faa55e9

11 files changed

Lines changed: 291 additions & 47 deletions

File tree

‎README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,18 @@ Upload data to datasets,..
9393
DatasetService datasetService = gd.getDatasetService();
9494
datasetService.loadDataset(project, "datasetId", new FileInputStream("data.csv")).get();
9595
```
96+
Upload data to datasets using batch upload ,..
97+
98+
```java
99+
DatasetService datasetService = gd.getDatasetService();
100+
final DatasetManifest personManifest = datasetService.getDatasetManifest(project, "dataset.person");
101+
personManifest.setSource(getClass().getResourceAsStream("/person.csv"));
102+
103+
final DatasetManifest cityManifest = datasetService.getDatasetManifest(project, "dataset.city");
104+
cityManifest.setSource(getClass().getResourceAsStream("/city.csv"));
105+
106+
datasetService.loadDatasets(project, personManifest, cityManifest).get();
107+
```
96108

97109
Update data in dataset
98110
```java

‎src/main/java/com/gooddata/dataset/DatasetException.java‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,49 @@
55

66
import com.gooddata.GoodDataException;
77

8+
import java.util.Collection;
9+
10+
import static java.util.Arrays.asList;
11+
812
/**
913
* Represents error in DatasetService
1014
*/
1115
public class DatasetException extends GoodDataException {
1216

13-
private final String dataset;
17+
private final Collection<String> datasets;
1418

1519
public DatasetException(String message, String dataset) {
1620
this(message, dataset, null);
1721
}
1822

1923
public DatasetException(String message, String dataset, Throwable cause) {
20-
super("Load dataset " + dataset + " failed: " + message, cause);
21-
this.dataset = dataset;
24+
this(message, asList(dataset), cause);
25+
}
26+
27+
public DatasetException(String message, Collection<String> datasets, Throwable cause) {
28+
super("Load datasets " + datasets + " failed: " + message, cause);
29+
this.datasets = datasets;
30+
}
31+
32+
public DatasetException(String message, Collection<String> datasets) {
33+
this(message, datasets, null);
34+
}
35+
36+
/**
37+
* Get datasets.
38+
* @return dataset names
39+
*/
40+
public Collection<String> getDatasets() {
41+
return datasets;
2242
}
2343

44+
/**
45+
* @return string representation of collection containing dataset names
46+
* @deprecated since this exception may contain more than one dataset, use {@link #getDatasets} instead
47+
* @see #getDatasets()
48+
*/
49+
@Deprecated
2450
public String getDataset() {
25-
return dataset;
51+
return datasets.toString();
2652
}
2753
}

‎src/main/java/com/gooddata/dataset/DatasetManifest.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,19 @@
66
import com.gooddata.util.BooleanIntegerDeserializer;
77
import com.gooddata.util.BooleanIntegerSerializer;
88
import org.codehaus.jackson.annotate.JsonCreator;
9+
import org.codehaus.jackson.annotate.JsonIgnore;
910
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
1011
import org.codehaus.jackson.annotate.JsonProperty;
1112
import org.codehaus.jackson.annotate.JsonTypeInfo;
1213
import org.codehaus.jackson.annotate.JsonTypeName;
1314
import org.codehaus.jackson.map.annotate.JsonDeserialize;
1415
import org.codehaus.jackson.map.annotate.JsonSerialize;
1516

17+
import java.io.InputStream;
1618
import java.util.List;
1719
import java.util.Map;
1820

21+
import static com.gooddata.util.Validate.notEmpty;
1922
import static com.gooddata.util.Validate.notNull;
2023

2124

@@ -33,11 +36,22 @@ public class DatasetManifest {
3336
private final String dataSet;
3437
private String file;
3538
private List<Part> parts;
39+
private InputStream source;
3640

3741
public DatasetManifest(String dataSet) {
3842
this.dataSet = dataSet;
3943
}
4044

45+
/**
46+
* Create dataset upload manifest.
47+
* @param dataSet dataset name
48+
* @param source source CSV
49+
*/
50+
public DatasetManifest(final String dataSet, final InputStream source) {
51+
this.source = notNull(source, "source");
52+
this.dataSet = notEmpty(dataSet, "dataSet");
53+
}
54+
4155
@JsonCreator
4256
public DatasetManifest(@JsonProperty("dataSet") String dataSet, @JsonProperty("file") String file,
4357
@JsonProperty("parts") List<Part> parts) {
@@ -63,7 +77,7 @@ public String getFile() {
6377
}
6478

6579
public void setFile(String file) {
66-
this.file = file;
80+
this.file = notEmpty(file, "file");
6781
}
6882

6983
/**
@@ -98,6 +112,16 @@ public void setMapping(final String columnName, final String populates) {
98112
throw new IllegalArgumentException("Dataset manifest parts doesn't contain populate value " + populates);
99113
}
100114

115+
@JsonIgnore
116+
public InputStream getSource() {
117+
return source;
118+
}
119+
120+
@JsonIgnore
121+
public void setSource(final InputStream source) {
122+
this.source = notNull(source, "source");
123+
}
124+
101125
@JsonIgnoreProperties(ignoreUnknown = true)
102126
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
103127
public static class Part {

‎src/main/java/com/gooddata/dataset/DatasetManifests.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved.
3+
*/
4+
5+
package com.gooddata.dataset;
6+
7+
import org.codehaus.jackson.annotate.JsonCreator;
8+
import org.codehaus.jackson.annotate.JsonProperty;
9+
10+
import java.util.Collection;
11+
12+
import static com.gooddata.util.Validate.notNull;
13+
14+
/**
15+
* Encapsulates list of {@link DatasetManifest}.
16+
*/
17+
class DatasetManifests {
18+
19+
private final Collection<DatasetManifest> manifests;
20+
21+
/**
22+
* Construct object.
23+
* @param manifests dataset upload manifests
24+
*/
25+
@JsonCreator
26+
public DatasetManifests(@JsonProperty("dataSetSLIManifestList") Collection<DatasetManifest> manifests) {
27+
this.manifests = notNull(manifests, "manifests");
28+
}
29+
30+
@JsonProperty("dataSetSLIManifestList")
31+
public Collection<DatasetManifest> getManifests() {
32+
return manifests;
33+
}
34+
}

‎src/main/java/com/gooddata/dataset/DatasetService.java‎

Lines changed: 103 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import static java.nio.charset.StandardCharsets.UTF_8;
3535
import static java.util.Arrays.asList;
3636
import static java.util.Collections.emptyList;
37+
import static org.apache.commons.lang.StringUtils.isEmpty;
3738

3839
/**
3940
* Service to work with datasets and manifests.
@@ -99,36 +100,115 @@ public FutureResult<Void> loadDataset(final Project project, final DatasetManife
99100
final ByteArrayInputStream inputStream = new ByteArrayInputStream(manifestJson.getBytes(UTF_8));
100101
dataStoreService.upload(dirPath.resolve(MANIFEST_FILE_NAME).toString(), inputStream);
101102

102-
final PullTask pullTask = restTemplate
103-
.postForObject(Pull.URI, new Pull(dirPath.toString()), PullTask.class, project.getId());
104-
return new FutureResult<>(this, new SimplePollHandler<Void>(pullTask.getUri(), Void.class) {
105-
@Override
106-
public boolean isFinished(ClientHttpResponse response) throws IOException {
107-
final PullTaskStatus status = extractData(response, PullTaskStatus.class);
108-
final boolean finished = status.isFinished();
109-
if (finished && !status.isSuccess()) {
110-
final String message = getErrorMessage(status, dirPath);
111-
throw new DatasetException(message, manifest.getDataSet());
112-
}
113-
return finished;
114-
}
115-
116-
@Override
117-
protected void onFinish() {
118-
try {
119-
dataStoreService.delete(dirPath.toString() + "/");
120-
} catch (DataStoreException ignored) {
121-
// todo log?
122-
}
123-
}
124-
});
103+
return pullLoad(project, dirPath, manifest.getDataSet());
125104
} catch (IOException e) {
126105
throw new DatasetException("Unable to serialize manifest", manifest.getDataSet(), e);
127106
} catch (DataStoreException | GoodDataRestException | RestClientException e) {
128107
throw new DatasetException("Unable to load", manifest.getDataSet(), e);
129108
}
130109
}
131110

111+
/**
112+
* Gets DatasetManifest (using {@link #getDatasetManifest(com.gooddata.project.Project, String)}
113+
* first and then calls {@link #loadDataset(com.gooddata.project.Project, DatasetManifest, java.io.InputStream)}
114+
*
115+
* @param project project to which dataset belongs
116+
* @param datasetId datasetId to obtain a manifest
117+
* @param dataset dataset to upload
118+
* @return {@link com.gooddata.FutureResult} of the task
119+
*/
120+
public FutureResult<Void> loadDataset(Project project, String datasetId, InputStream dataset) {
121+
notNull(project, "project");
122+
notEmpty(datasetId, "datasetId");
123+
notNull(dataset, "dataset");
124+
return loadDataset(project, getDatasetManifest(project, datasetId), dataset);
125+
}
126+
127+
public FutureResult<Void> loadDatasets(final Project project, DatasetManifest... datasets) {
128+
return loadDatasets(project, asList(datasets));
129+
}
130+
131+
/**
132+
* Loads datasets into platform. Uploads given datasets and their manifests to staging area and triggers ETL pull.
133+
* The call is asynchronous returning {@link com.gooddata.FutureResult} to let caller wait for results.
134+
* Uploaded files are deleted from staging area when finished.
135+
*
136+
* @param project project to which dataset belongs
137+
* @param datasets map dataset manifests
138+
* @return {@link com.gooddata.FutureResult} of the task, which can throw {@link com.gooddata.dataset.DatasetException}
139+
* in case the ETL pull task fails
140+
* @throws com.gooddata.dataset.DatasetException if there is a problem to serialize manifest or upload dataset
141+
* @see <a href="https://developer.gooddata.com/article/multiload-of-csv-data">batch upload reference</a>
142+
*/
143+
public FutureResult<Void> loadDatasets(final Project project, final Collection<DatasetManifest> datasets) {
144+
notNull(project, "project");
145+
validateUploadManifests(datasets);
146+
final List<String> datasetsNames = new ArrayList<>(datasets.size());
147+
try {
148+
final Path dirPath = Paths.get("/", project.getId() + "_" + RandomStringUtils.randomAlphabetic(3), "/");
149+
for (DatasetManifest datasetManifest : datasets) {
150+
datasetsNames.add(datasetManifest.getDataSet());
151+
dataStoreService.upload(dirPath.resolve(datasetManifest.getFile()).toString(), datasetManifest.getSource());
152+
}
153+
154+
final String manifestJson = mapper.writeValueAsString(new DatasetManifests(datasets));
155+
final ByteArrayInputStream inputStream = new ByteArrayInputStream(manifestJson.getBytes(UTF_8));
156+
dataStoreService.upload(dirPath.resolve(MANIFEST_FILE_NAME).toString(), inputStream);
157+
158+
return pullLoad(project, dirPath, datasetsNames);
159+
} catch (IOException e) {
160+
throw new DatasetException("Unable to serialize manifest", datasetsNames, e);
161+
} catch (DataStoreException | GoodDataRestException | RestClientException e) {
162+
throw new DatasetException("Unable to load", datasetsNames, e);
163+
}
164+
}
165+
166+
private void validateUploadManifests(final Collection<DatasetManifest> datasets) {
167+
notEmpty(datasets, "datasets");
168+
for (DatasetManifest datasetManifest : datasets) {
169+
if (datasetManifest.getSource() == null) {
170+
throw new IllegalArgumentException(format("Source for dataset '%s' is null", datasetManifest.getDataSet()));
171+
}
172+
if (datasetManifest.getFile() == null) {
173+
throw new IllegalArgumentException(format("File for dataset '%s' is null", datasetManifest.getDataSet()));
174+
}
175+
if (isEmpty(datasetManifest.getDataSet())) {
176+
throw new IllegalArgumentException("Dataset name is empty.");
177+
}
178+
}
179+
}
180+
181+
private FutureResult<Void> pullLoad(Project project, final Path dirPath, final String dataset) {
182+
return pullLoad(project, dirPath, asList(dataset));
183+
}
184+
185+
private FutureResult<Void> pullLoad(Project project, final Path dirPath, final Collection<String> datasets) {
186+
final PullTask pullTask = restTemplate
187+
.postForObject(Pull.URI, new Pull(dirPath.toString()), PullTask.class, project.getId());
188+
return new FutureResult<>(this, new SimplePollHandler<Void>(pullTask.getUri(), Void.class) {
189+
@Override
190+
public boolean isFinished(ClientHttpResponse response) throws IOException {
191+
final PullTaskStatus status = extractData(response, PullTaskStatus.class);
192+
final boolean finished = status.isFinished();
193+
if (finished && !status.isSuccess()) {
194+
final String message = getErrorMessage(status, dirPath);
195+
throw new DatasetException(message, datasets);
196+
}
197+
return finished;
198+
}
199+
200+
@Override
201+
protected void onFinish() {
202+
try {
203+
dataStoreService.delete(dirPath.toString() + "/");
204+
} catch (DataStoreException ignored) {
205+
// todo log?
206+
}
207+
}
208+
});
209+
210+
}
211+
132212
private String getErrorMessage(final PullTaskStatus status, final Path dirPath) {
133213
String message = "status: " + status.getStatus();
134214
try {
@@ -166,22 +246,6 @@ private <T> T download(final Path path, final Class<T> type) throws IOException
166246
}
167247
}
168248

169-
/**
170-
* Gets DatasetManifest (using {@link #getDatasetManifest(com.gooddata.project.Project, String)}
171-
* first and then calls {@link #loadDataset(com.gooddata.project.Project, DatasetManifest, java.io.InputStream)}
172-
*
173-
* @param project project to which dataset belongs
174-
* @param datasetId datasetId to obtain a manifest
175-
* @param dataset dataset to upload
176-
* @return {@link com.gooddata.FutureResult} of the task
177-
*/
178-
public FutureResult<Void> loadDataset(Project project, String datasetId, InputStream dataset) {
179-
notNull(project, "project");
180-
notEmpty(datasetId, "datasetId");
181-
notNull(dataset, "dataset");
182-
return loadDataset(project, getDatasetManifest(project, datasetId), dataset);
183-
}
184-
185249
/**
186250
* Lists datasets in project. Returns empty list in case there are no datasets.
187251
*

‎src/main/java/com/gooddata/util/Validate.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,22 @@ public static <T extends CharSequence> T notEmpty(T value, String argumentName)
3939
return value;
4040
}
4141

42+
/**
43+
* Throws IllegalArgumentException if the map is empty, otherwise returns the map.
44+
*
45+
* @param value input collection
46+
* @param argumentName the name of the input argument
47+
* @param <T> the type of map
48+
* @return map
49+
*/
50+
public static <T extends Collection> T notEmpty(T value, String argumentName) {
51+
notNull(value, argumentName);
52+
if (value.size() == 0) {
53+
throw new IllegalArgumentException(argumentName + " can't be empty");
54+
}
55+
return value;
56+
}
57+
4258
/**
4359
* Throws IllegalArgumentException if the collection contains null elements (or is null), otherwise returns
4460
* the collection.

‎src/test/java/com/gooddata/ShowcaseAT.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,19 @@ public void loadDataset() throws Exception {
211211
datasetService.loadDataset(project, manifest, getClass().getResourceAsStream("/person.csv")).get();
212212
}
213213

214-
@Test(groups = "dataset", dependsOnMethods = "loadDataset")
214+
@Test(groups = "dataset", dependsOnMethods = {"loadDataset"})
215+
public void loadDatasetBatch() throws Exception {
216+
final DatasetService datasetService = gd.getDatasetService();
217+
218+
final DatasetManifest personManifest = datasetService.getDatasetManifest(project, "dataset.person");
219+
personManifest.setSource(getClass().getResourceAsStream("/person.csv"));
220+
final DatasetManifest cityManifest = datasetService.getDatasetManifest(project, "dataset.city");
221+
cityManifest.setSource(getClass().getResourceAsStream("/city.csv"));
222+
223+
datasetService.loadDatasets(project, personManifest, cityManifest).get();
224+
}
225+
226+
@Test(groups = "dataset", dependsOnMethods = "loadDatasetBatch")
215227
public void updateData() {
216228
final DatasetService datasetService = gd.getDatasetService();
217229
datasetService.updateProjectData(project, "DELETE FROM {attr.person.name} WHERE {label.person.name} = \"not exists\";");

0 commit comments

Comments
 (0)