Skip to content

Commit 3bf5111

Browse files
jimirocksjimirocks
committed
add etl pull capabilities
1 parent b4ffc23 commit 3bf5111

9 files changed

Lines changed: 163 additions & 9 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ InputStream stream = dataStoreService.download("/dir/file.txt");
5555
```java
5656
final DatasetService datasetService = gd.getDatasetService();
5757
final DatasetManifest manifest = datasetService.getDatasetManifest(project, "datasetId");
58+
datasetService.loadDataset(project, new FileInputStream("/person.csv"), manifest);
5859

5960
```
6061

‎src/main/java/com/gooddata/dataset/DatasetManifest.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import org.codehaus.jackson.annotate.JsonCreator;
44
import org.codehaus.jackson.annotate.JsonProperty;
5+
import org.codehaus.jackson.annotate.JsonPropertyOrder;
56
import org.codehaus.jackson.annotate.JsonTypeInfo;
67
import org.codehaus.jackson.annotate.JsonTypeName;
78
import java.util.List;
@@ -12,27 +13,28 @@
1213
*/
1314
@JsonTypeInfo(include = JsonTypeInfo.As.WRAPPER_OBJECT, use = JsonTypeInfo.Id.NAME)
1415
@JsonTypeName("dataSetSLIManifest")
16+
@JsonPropertyOrder({"parts", "dataSet", "file"})
1517
public class DatasetManifest {
1618

17-
public static final String URI = "/gdc/md/{projectId}/ldm/singleloadinterface/{id}/manifest";
19+
public static final String URI = "/gdc/md/{projectId}/ldm/singleloadinterface/{dataSet}/manifest";
1820

19-
private final String id;
21+
private final String dataSet;
2022
private String file;
2123
private List<Part> parts;
2224

23-
public DatasetManifest(String id) {
24-
this.id = id;
25+
public DatasetManifest(String dataSet) {
26+
this.dataSet = dataSet;
2527
}
2628

2729
@JsonCreator
28-
public DatasetManifest(@JsonProperty("dataSet") String id, @JsonProperty("file") String file, @JsonProperty("parts") List<Part> parts) {
29-
this.id = id;
30+
public DatasetManifest(@JsonProperty("dataSet") String dataSet, @JsonProperty("file") String file, @JsonProperty("parts") List<Part> parts) {
31+
this.dataSet = dataSet;
3032
this.file = file;
3133
this.parts = parts;
3234
}
3335

34-
public String getId() {
35-
return id;
36+
public String getDataSet() {
37+
return dataSet;
3638
}
3739

3840
public List<Part> getParts() {

‎src/main/java/com/gooddata/dataset/DatasetService.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,28 @@
44
package com.gooddata.dataset;
55

66
import com.gooddata.AbstractService;
7+
import com.gooddata.GoodDataException;
78
import com.gooddata.gdc.DataStoreService;
89
import com.gooddata.project.Project;
10+
import com.gooddata.task.PullTask;
11+
import com.gooddata.task.PullTaskStatus;
12+
import org.apache.commons.io.IOUtils;
13+
import org.apache.commons.lang.RandomStringUtils;
14+
import org.codehaus.jackson.map.ObjectMapper;
15+
import org.springframework.http.client.ClientHttpResponse;
16+
import org.springframework.web.client.HttpMessageConverterExtractor;
917
import org.springframework.web.client.RestTemplate;
18+
import java.io.IOException;
19+
import java.io.InputStream;
20+
import java.net.URI;
1021

1122
/**
1223
*/
1324
public class DatasetService extends AbstractService {
1425

26+
private static final String MANIFEST_FILE_NAME = "upload_info.json";
1527
private final DataStoreService dataStoreService;
28+
private final static ObjectMapper mapper = new ObjectMapper();
1629

1730
public DatasetService(RestTemplate restTemplate, DataStoreService dataStoreService) {
1831
super(restTemplate);
@@ -23,4 +36,34 @@ public DatasetManifest getDatasetManifest(Project project, String datasetId) {
2336
return restTemplate.getForObject(DatasetManifest.URI, DatasetManifest.class, project.getId(), datasetId);
2437
}
2538

39+
public void loadDataset(Project project, InputStream dataset, DatasetManifest manifest) {
40+
final String dirPath = "/" + project.getId() + "_" + RandomStringUtils.randomAlphabetic(3);
41+
try {
42+
dataStoreService.upload(dirPath + "/" + manifest.getFile(), dataset);
43+
String manifestJson = mapper.writeValueAsString(manifest);
44+
dataStoreService.upload(dirPath + "/" + MANIFEST_FILE_NAME, IOUtils.toInputStream(manifestJson));
45+
46+
final PullTask pullTask = restTemplate.postForObject(Pull.URI, new Pull(dirPath), PullTask.class, project.getId());
47+
PullTaskStatus taskStatus = poll(URI.create(pullTask.getUri()), new ConditionCallback() {
48+
@Override
49+
public boolean finished(ClientHttpResponse response) throws IOException {
50+
final PullTaskStatus status = new HttpMessageConverterExtractor<>(PullTaskStatus.class, restTemplate.getMessageConverters())
51+
.extractData(response);
52+
return status.isFinished();
53+
}
54+
}, PullTaskStatus.class);
55+
if (!taskStatus.isSuccess()) {
56+
throw new GoodDataException("ETL pull finished with status " + taskStatus.getStatus());
57+
}
58+
} catch (IOException e) {
59+
throw new GoodDataException("Unable to serialize manifest", e);
60+
} finally {
61+
// TODO delete
62+
}
63+
64+
}
65+
66+
public void loadDataset(Project project, InputStream dataset, String datasetId) {
67+
loadDataset(project, dataset, getDatasetManifest(project, datasetId));
68+
}
2669
}

‎src/main/java/com/gooddata/dataset/Pull.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package com.gooddata.dataset;
2+
3+
import org.codehaus.jackson.annotate.JsonProperty;
4+
5+
/**
6+
* TODO
7+
*/
8+
public class Pull {
9+
10+
public static final String URI = "/gdc/md/{projectId}/etl/pull";
11+
12+
@JsonProperty("pullIntegration")
13+
private final String remoteDir;
14+
15+
16+
public Pull(String remoteDir) {
17+
this.remoteDir = remoteDir;
18+
}
19+
20+
public String getRemoteDir() {
21+
return remoteDir;
22+
}
23+
}

‎src/main/java/com/gooddata/task/PullTask.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package com.gooddata.task;
2+
3+
import org.codehaus.jackson.annotate.JsonCreator;
4+
import org.codehaus.jackson.annotate.JsonProperty;
5+
import org.codehaus.jackson.annotate.JsonTypeInfo;
6+
import org.codehaus.jackson.annotate.JsonTypeName;
7+
8+
/**
9+
* TODO
10+
*/
11+
@JsonTypeInfo(include = JsonTypeInfo.As.WRAPPER_OBJECT, use = JsonTypeInfo.Id.NAME)
12+
@JsonTypeName("pullTask")
13+
public class PullTask {
14+
15+
public static final String URI = "/gdc/md/{projectId}/etl/task/{taskId}";
16+
17+
private final String uri;
18+
19+
@JsonCreator
20+
public PullTask(@JsonProperty("uri") String uri) {
21+
this.uri = uri;
22+
}
23+
24+
public String getUri() {
25+
return uri;
26+
}
27+
}

‎src/main/java/com/gooddata/task/PullTaskStatus.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package com.gooddata.task;
2+
3+
import org.codehaus.jackson.annotate.JsonCreator;
4+
import org.codehaus.jackson.annotate.JsonProperty;
5+
6+
/**
7+
* TODO
8+
*/
9+
public class PullTaskStatus {
10+
11+
private static final String OK = "OK";
12+
private static final String ERROR = "ERROR";
13+
private static final String WARNING = "WARNING";
14+
private final String status;
15+
16+
@JsonCreator
17+
public PullTaskStatus(@JsonProperty("taskStatus") String status) {
18+
this.status = status;
19+
}
20+
21+
public String getStatus() {
22+
return status;
23+
}
24+
25+
public boolean isSuccess() {
26+
return OK.equals(status);
27+
}
28+
29+
public boolean isFinished() {
30+
return OK.equals(status) || ERROR.equals(status) || WARNING.equals(status);
31+
}
32+
}

‎src/test/java/com/gooddata/DatasetExample.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package com.gooddata;
2+
3+
import com.gooddata.dataset.DatasetManifest;
4+
import com.gooddata.dataset.DatasetService;
5+
import com.gooddata.project.Project;
6+
7+
/**
8+
* TODO
9+
*/
10+
public class DatasetExample {
11+
12+
public static void main(String... args) {
13+
final GoodData gd = new GoodData("staging.getgooddata.com", "jiri.mikulasek@gooddata.com", "jindrisska");
14+
15+
final Project project = gd.getProjectService().getProjectById("amxhoeyj7oskijld63tajq0o9f4nhxy7");
16+
DatasetService datasetService = gd.getDatasetService();
17+
DatasetManifest manifest = datasetService.getDatasetManifest(project, "dataset.person");
18+
19+
datasetService.loadDataset(project, DatasetExample.class.getResourceAsStream("/person.csv"), manifest);
20+
}
21+
}

‎src/test/java/com/gooddata/dataset/DatasetManifestTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public class DatasetManifestTest {
6060
@Test
6161
public void testDeser() throws Exception {
6262
final DatasetManifest manifest = new ObjectMapper().readValue(SERIALIZED, DatasetManifest.class);
63-
assertEquals("dataset.person", manifest.getId());
63+
assertEquals("dataset.person", manifest.getDataSet());
6464
assertEquals("dataset.person.csv", manifest.getFile());
6565
assertEquals(5, manifest.getParts().size());
6666

‎src/test/resources/person.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
f_person.f_shoesize,f_person.nm_name,d_person_role.nm_xrole,f_person.f_age,d_person_department.nm_xdepartment
2+
37,Jane,manager,35,HR
3+
42,Jack,developer,25,DevOps
4+
35,Sandy,recruiter,27,HR
5+
40,Jonathan,developer,50,DevOps

0 commit comments

Comments
 (0)