Skip to content

Commit f831092

Browse files
author
Ondrej Benkovsky
committed
parser for batch fail status json
1 parent 1547e16 commit f831092

8 files changed

Lines changed: 277 additions & 30 deletions

File tree

‎src/main/java/com/gooddata/dataset/BatchFailStatus.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved.
3+
*/
4+
package com.gooddata.dataset;
5+
6+
import com.fasterxml.jackson.annotation.JsonCreator;
7+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
8+
import com.fasterxml.jackson.annotation.JsonProperty;
9+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
10+
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
11+
import com.gooddata.util.GDDateTimeDeserializer;
12+
import com.gooddata.util.GDDateTimeSerializer;
13+
14+
import org.joda.time.DateTime;
15+
16+
import java.util.List;
17+
18+
/**
19+
* Batch fail status of dataset load.
20+
*/
21+
@JsonIgnoreProperties(ignoreUnknown = true)
22+
public class BatchFailStatus {
23+
24+
private final List<FailStatus> failStatuses;
25+
private final List<String> messages;
26+
private final String status;
27+
private final DateTime date;
28+
29+
@JsonCreator
30+
private BatchFailStatus(@JsonProperty("uploads") List<FailStatus> failStatuses, @JsonProperty("messages") List<String> messages,
31+
@JsonProperty("status") String status, @JsonProperty("date") @JsonDeserialize(using = GDDateTimeDeserializer.class) DateTime date) {
32+
this.failStatuses = failStatuses;
33+
this.messages = messages;
34+
this.status = status;
35+
this.date = date;
36+
}
37+
38+
public List<FailStatus> getFailStatuses() {
39+
return failStatuses;
40+
}
41+
42+
public List<String> getMessages() {
43+
return messages;
44+
}
45+
46+
public String getStatus() {
47+
return status;
48+
}
49+
50+
@JsonSerialize(using = GDDateTimeSerializer.class)
51+
public DateTime getDate() {
52+
return date;
53+
}
54+
}

‎src/main/java/com/gooddata/dataset/DatasetService.java‎

Lines changed: 50 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import static java.nio.charset.StandardCharsets.UTF_8;
3636
import static java.util.Arrays.asList;
3737
import static java.util.Collections.emptyList;
38+
import static java.util.Collections.singletonList;
3839
import static org.springframework.util.StringUtils.isEmpty;
3940

4041
/**
@@ -101,7 +102,7 @@ public FutureResult<Void> loadDataset(final Project project, final DatasetManife
101102
final ByteArrayInputStream inputStream = new ByteArrayInputStream(manifestJson.getBytes(UTF_8));
102103
dataStoreService.upload(dirPath.resolve(MANIFEST_FILE_NAME).toString(), inputStream);
103104

104-
return pullLoad(project, dirPath, manifest.getDataSet());
105+
return pullLoad(project, dirPath, manifest.getDataSet(), false);
105106
} catch (IOException e) {
106107
throw new DatasetException("Unable to serialize manifest", manifest.getDataSet(), e);
107108
} catch (DataStoreException | GoodDataRestException | RestClientException e) {
@@ -156,7 +157,7 @@ public FutureResult<Void> loadDatasets(final Project project, final Collection<D
156157
final ByteArrayInputStream inputStream = new ByteArrayInputStream(manifestJson.getBytes(UTF_8));
157158
dataStoreService.upload(dirPath.resolve(MANIFEST_FILE_NAME).toString(), inputStream);
158159

159-
return pullLoad(project, dirPath, datasetsNames);
160+
return pullLoad(project, dirPath, datasetsNames, true);
160161
} catch (IOException e) {
161162
throw new DatasetException("Unable to serialize manifest", datasetsNames, e);
162163
} catch (DataStoreException | GoodDataRestException | RestClientException e) {
@@ -179,11 +180,11 @@ private void validateUploadManifests(final Collection<DatasetManifest> datasets)
179180
}
180181
}
181182

182-
private FutureResult<Void> pullLoad(Project project, final Path dirPath, final String dataset) {
183-
return pullLoad(project, dirPath, asList(dataset));
183+
private FutureResult<Void> pullLoad(Project project, final Path dirPath, final String dataset, boolean isBatchUpload) {
184+
return pullLoad(project, dirPath, singletonList(dataset), isBatchUpload);
184185
}
185186

186-
private FutureResult<Void> pullLoad(Project project, final Path dirPath, final Collection<String> datasets) {
187+
private FutureResult<Void> pullLoad(Project project, final Path dirPath, final Collection<String> datasets, final boolean isBatchUpload) {
187188
final PullTask pullTask = restTemplate
188189
.postForObject(Pull.URI, new Pull(dirPath.toString()), PullTask.class, project.getId());
189190
return new PollResult<>(this, new SimplePollHandler<Void>(pullTask.getUri(), Void.class) {
@@ -192,7 +193,7 @@ public boolean isFinished(ClientHttpResponse response) throws IOException {
192193
final PullTaskStatus status = extractData(response, PullTaskStatus.class);
193194
final boolean finished = status.isFinished();
194195
if (finished && !status.isSuccess()) {
195-
final String message = getErrorMessage(status, dirPath);
196+
final String message = getErrorMessage(status, dirPath, isBatchUpload);
196197
throw new DatasetException(message, datasets);
197198
}
198199
return finished;
@@ -215,37 +216,58 @@ protected void onFinish() {
215216

216217
}
217218

218-
private String getErrorMessage(final PullTaskStatus status, final Path dirPath) {
219+
private String getErrorMessage(final PullTaskStatus status, final Path dirPath, boolean isBatchUpload) {
219220
String message = "status: " + status.getStatus();
220221
try {
221-
final FailStatus failStatus = download(dirPath.resolve(STATUS_FILE_NAME), FailStatus.class);
222-
if (failStatus != null) {
223-
final List<FailPart> errorParts = failStatus.getErrorParts();
224-
if (!errorParts.isEmpty()) {
225-
final List<String> errors = new ArrayList<>();
226-
for (FailPart part: errorParts) {
227-
if (part.getLogName() != null) {
228-
try {
229-
final String[] msg = download(dirPath.resolve(part.getLogName()), String[].class);
230-
errors.addAll(asList(msg));
231-
} catch (IOException | DataStoreException e) {
232-
if (part.getError() != null) {
233-
errors.add(part.getError().getFormattedMessage());
234-
}
235-
}
236-
}
237-
}
238-
message = errors.toString();
239-
} else if (failStatus.getError() != null) {
240-
message = failStatus.getError().getFormattedMessage();
241-
}
222+
Path statusFile = dirPath.resolve(STATUS_FILE_NAME);
223+
if (isBatchUpload) {
224+
final BatchFailStatus batchFailStatus = download(statusFile, BatchFailStatus.class);
225+
message = getBatchFailStatusErrorMsg(dirPath, message, batchFailStatus);
226+
} else {
227+
final FailStatus failStatus = download(statusFile, FailStatus.class);
228+
message = getFailStatusErrorMsg(dirPath, message, failStatus);
242229
}
243230
} catch (IOException | DataStoreException ignored) {
244231
// todo log?
245232
}
246233
return message;
247234
}
248235

236+
private String getBatchFailStatusErrorMsg(final Path dirPath, final String defaultMessage, final BatchFailStatus batchFailStatus) {
237+
if (batchFailStatus == null || batchFailStatus.getFailStatuses() == null) {
238+
return defaultMessage;
239+
}
240+
final List<String> messages = new ArrayList<>();
241+
for (FailStatus failStatus : batchFailStatus.getFailStatuses()) {
242+
messages.add(getFailStatusErrorMsg(dirPath, defaultMessage, failStatus));
243+
}
244+
return messages.isEmpty() ? defaultMessage : messages.toString();
245+
}
246+
247+
private String getFailStatusErrorMsg(final Path dirPath, final String defaultMessage, final FailStatus failStatus) {
248+
if (failStatus == null) {
249+
return defaultMessage;
250+
}
251+
final List<FailPart> errorParts = failStatus.getErrorParts();
252+
if (errorParts.isEmpty()){
253+
return (failStatus.getError() != null) ? failStatus.getError().getFormattedMessage() : defaultMessage;
254+
}
255+
final List<String> errors = new ArrayList<>();
256+
for (FailPart part : errorParts) {
257+
if (part.getLogName() != null) {
258+
try {
259+
final String[] msg = download(dirPath.resolve(part.getLogName()), String[].class);
260+
errors.addAll(asList(msg));
261+
} catch (IOException | DataStoreException e) {
262+
if (part.getError() != null) {
263+
errors.add(part.getError().getFormattedMessage());
264+
}
265+
}
266+
}
267+
}
268+
return errors.toString();
269+
}
270+
249271
private <T> T download(final Path path, final Class<T> type) throws IOException {
250272
try (final InputStream input = dataStoreService.download(path.toString())) {
251273
return mapper.readValue(input, type);
@@ -324,7 +346,6 @@ public void handlePollException(final GoodDataRestException e) {
324346
* @param project project to be updated
325347
* @param maqlDml update script to be executed in the project
326348
* @return poll result
327-
*
328349
* @see com.gooddata.model.ModelService#updateProjectModel
329350
*/
330351
public FutureResult<Void> updateProjectData(final Project project, final String maqlDml) {
@@ -362,5 +383,4 @@ public void handlePollException(final GoodDataRestException e) {
362383
}
363384
});
364385
}
365-
366386
}

‎src/test/java/com/gooddata/dataset/BatchFailStatusTest.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved.
3+
*/
4+
package com.gooddata.dataset;
5+
6+
import static org.hamcrest.MatcherAssert.assertThat;
7+
import static org.hamcrest.Matchers.containsInAnyOrder;
8+
import static org.hamcrest.Matchers.hasSize;
9+
import static org.hamcrest.Matchers.is;
10+
import static org.hamcrest.Matchers.notNullValue;
11+
12+
import com.fasterxml.jackson.databind.ObjectMapper;
13+
import org.hamcrest.Matchers;
14+
import org.joda.time.DateTime;
15+
import org.joda.time.DateTimeZone;
16+
import org.testng.annotations.Test;
17+
18+
import java.io.IOException;
19+
import java.io.InputStream;
20+
21+
public class BatchFailStatusTest {
22+
@Test
23+
public void testParser() throws Exception {
24+
final InputStream stream = getClass().getResourceAsStream("/dataset/batchFailStatus1.json");
25+
final BatchFailStatus value = new ObjectMapper().readValue(stream, BatchFailStatus.class);
26+
assertThat(value, is(notNullValue()));
27+
assertThat(value.getStatus(),is("ERROR"));
28+
assertThat(value.getMessages(),is(Matchers.<String>empty()));
29+
assertThat(value.getDate(),is(new DateTime(2016,2,1,10,12,9, DateTimeZone.UTC)));
30+
assertThat(value.getFailStatuses(),hasSize(2));
31+
}
32+
33+
@Test
34+
public void testParser2() throws IOException {
35+
final InputStream stream = getClass().getResourceAsStream("/dataset/batchFailStatus2.json");
36+
final BatchFailStatus value = new ObjectMapper().readValue(stream, BatchFailStatus.class);
37+
assertThat(value,is(notNullValue()));
38+
assertThat(value.getStatus(),is("ERROR"));
39+
assertThat(value.getMessages(),containsInAnyOrder("test1","test2"));
40+
assertThat(value.getDate(),is(new DateTime(2016,1,1,10,11,15, DateTimeZone.UTC)));
41+
assertThat(value.getFailStatuses(),is(Matchers.<FailStatus>empty()));
42+
}
43+
}

‎src/test/java/com/gooddata/dataset/DatasetServiceAT.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
package com.gooddata.dataset;
22

3+
import static org.hamcrest.CoreMatchers.equalTo;
4+
import static org.hamcrest.CoreMatchers.is;
5+
import static org.testng.Assert.fail;
6+
37
import com.gooddata.AbstractGoodDataAT;
8+
import static org.hamcrest.MatcherAssert.assertThat;
9+
410
import org.testng.annotations.Test;
511

612
/**
@@ -34,5 +40,32 @@ public void updateData() {
3440
datasetService.updateProjectData(project, "DELETE FROM {attr.person.name} WHERE {label.person.name} = \"not exists\";");
3541
}
3642

43+
@Test(groups = "dataset", dependsOnGroups = {"md", "datastore"})
44+
public void loadDatasetFail(){
45+
final DatasetService datasetService = gd.getDatasetService();
46+
final DatasetManifest manifest = datasetService.getDatasetManifest(project, "dataset.person");
47+
try {
48+
datasetService.loadDataset(project, manifest, getClass().getResourceAsStream("/corruptedPerson.csv")).get();
49+
fail();
50+
} catch (DatasetException ex){
51+
assertThat(ex.getMessage(),is(equalTo("Load datasets [dataset.person] failed: Number of columns doesn't corespond on line 3 in dataset.person.csv")));
52+
}
53+
}
54+
55+
@Test(groups = "dataset", dependsOnMethods = {"loadDataset"})
56+
public void loadDatasetBatchFail() throws Exception {
57+
final DatasetService datasetService = gd.getDatasetService();
3758

59+
final DatasetManifest personManifest = datasetService.getDatasetManifest(project, "dataset.person");
60+
personManifest.setSource(getClass().getResourceAsStream("/corruptedPerson.csv"));
61+
final DatasetManifest cityManifest = datasetService.getDatasetManifest(project, "dataset.city");
62+
cityManifest.setSource(getClass().getResourceAsStream("/city.csv"));
63+
64+
try {
65+
datasetService.loadDatasets(project, personManifest, cityManifest).get();
66+
fail();
67+
} catch (DatasetException ex){
68+
assertThat(ex.getMessage(),is(equalTo("Load datasets [dataset.person, dataset.city] failed: [Number of columns doesn't corespond on line 3 in dataset.person.csv, Number of columns doesn't corespond on line 3 in dataset.person.csv]")));
69+
}
70+
}
3871
}

‎src/test/java/com/gooddata/dataset/DatasetServiceIT.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,4 +280,54 @@ public void shouldFailUpdateProjectData() throws IOException {
280280

281281
gd.getDatasetService().updateProjectData(project, DML_MAQL).get();
282282
}
283+
284+
@Test
285+
public void shouldReadBatchErrorMessages() throws Exception {
286+
onRequest()
287+
.havingPathEqualTo("/gdc/md/PROJECT/etl/task/ID")
288+
.respond()
289+
.withStatus(200)
290+
.withBody(readFromResource("/dataset/pullTaskStatusError.json"));
291+
onRequest()
292+
.havingPath(containsString("upload_status.json"))
293+
.havingMethodEqualTo("GET")
294+
.respond()
295+
.withStatus(200)
296+
.withBody(readFromResource("/dataset/batchFailStatus1.json"));
297+
298+
final DatasetManifest manifest = MAPPER.readValue(readFromResource("/dataset/datasetManifest.json"), DatasetManifest.class);
299+
final InputStream source = new ByteArrayInputStream(new byte[]{});
300+
manifest.setSource(source);
301+
try {
302+
gd.getDatasetService().loadDatasets(project, manifest).get();
303+
fail("Exception should be thrown");
304+
} catch (DatasetException e) {
305+
assertThat(e.getMessage(), is("Load datasets [dataset.person] failed: [Manifest consist of columns that are not in single CSV file dataset.stats.csv: f_competitors.nm_name., Manifest consist of columns that are not in single CSV file dataset.stats.csv: f_competitors.nm_name.]"));
306+
}
307+
}
308+
309+
@Test
310+
public void shouldReadBatchErrorMessagesNoFailStatuses() throws Exception {
311+
onRequest()
312+
.havingPathEqualTo("/gdc/md/PROJECT/etl/task/ID")
313+
.respond()
314+
.withStatus(200)
315+
.withBody(readFromResource("/dataset/pullTaskStatusError.json"));
316+
onRequest()
317+
.havingPath(containsString("upload_status.json"))
318+
.havingMethodEqualTo("GET")
319+
.respond()
320+
.withStatus(200)
321+
.withBody(readFromResource("/dataset/batchFailStatus2.json"));
322+
323+
final DatasetManifest manifest = MAPPER.readValue(readFromResource("/dataset/datasetManifest.json"), DatasetManifest.class);
324+
final InputStream source = new ByteArrayInputStream(new byte[]{});
325+
manifest.setSource(source);
326+
try {
327+
gd.getDatasetService().loadDatasets(project, manifest).get();
328+
fail("Exception should be thrown");
329+
} catch (DatasetException e) {
330+
assertThat(e.getMessage(), is("Load datasets [dataset.person] failed: status: ERROR"));
331+
}
332+
}
283333
}

‎src/test/resources/corruptedPerson.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
f_person.f_shoesize,f_person.nm_name,d_person_role.nm_role,f_person.f_age,d_person_department.nm_department
2+
37,Jane,manager,35,HR
3+
42,developer,25,DevOps
4+
35,Sandy,recruiter,27
5+
40,Jonathan,developer,50,DevOps

‎src/test/resources/dataset/batchFailStatus1.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"uploads" : [
3+
{
4+
"date" : "2016-02-01 10:12:09",
5+
"status" : "ERROR",
6+
"error" : {
7+
"parameters" : [
8+
"dataset.stats.csv",
9+
"f_competitors.nm_name"
10+
],
11+
"component" : "GDC::SliToDli",
12+
"errorClass" : "GDC::Exception::User",
13+
"message" : "Manifest consist of columns that are not in single CSV file %s: %s."
14+
}
15+
},
16+
{
17+
"date" : "2016-02-01 10:12:09",
18+
"status" : "ERROR",
19+
"error" : {
20+
"parameters" : [
21+
"dataset.stats.csv",
22+
"f_competitors.nm_name"
23+
],
24+
"component" : "GDC::SliToDli",
25+
"errorClass" : "GDC::Exception::User",
26+
"message" : "Manifest consist of columns that are not in single CSV file %s: %s."
27+
}
28+
}
29+
],
30+
"messages" : [],
31+
"status" : "ERROR",
32+
"date" : "2016-02-01 10:12:09"
33+
}

‎src/test/resources/dataset/batchFailStatus2.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"uploads": [],
3+
"messages": [
4+
"test1",
5+
"test2"
6+
],
7+
"status": "ERROR",
8+
"date": "2016-01-01 10:11:15"
9+
}

0 commit comments

Comments
 (0)