Skip to content

Commit ceb26a1

Browse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat: [datalineage] add open lineage support (#10041)
* feat: add open lineage support fix: change `start_time` in message `.google.cloud.datacatalog.lineage.v1.LineageEvent` to `required` as intended by api PiperOrigin-RevId: 579762272 Source-Link: googleapis/googleapis@58878bd Source-Link: https://github.com/googleapis/googleapis-gen/commit/96a4d735b92f202fbdd2c62da2e21e63216bfa88 Copy-Tag: eyJwIjoiamF2YS1kYXRhbGluZWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiOTZhNGQ3MzViOTJmMjAyZmJkZDJjNjJkYTJlMjFlNjMyMTZiZmE4OCJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent cc0e248 commit ceb26a1

File tree

44 files changed

+4060
-475
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+4060
-475
lines changed

‎java-datalineage/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file:
2323
<dependency>
2424
<groupId>com.google.cloud</groupId>
2525
<artifactId>libraries-bom</artifactId>
26-
<version>26.23.0</version>
26+
<version>26.26.0</version>
2727
<type>pom</type>
2828
<scope>import</scope>
2929
</dependency>
@@ -201,7 +201,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
201201
[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-datalineage/java11.html
202202
[stability-image]: https://img.shields.io/badge/stability-preview-yellow
203203
[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-datalineage.svg
204-
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datalineage/0.16.0
204+
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datalineage/0.21.0
205205
[authentication]: https://github.com/googleapis/google-cloud-java#authentication
206206
[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
207207
[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles

‎java-datalineage/google-cloud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/LineageClient.java‎

Lines changed: 112 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.google.longrunning.Operation;
3535
import com.google.protobuf.Empty;
3636
import com.google.protobuf.FieldMask;
37+
import com.google.protobuf.Struct;
3738
import java.io.IOException;
3839
import java.util.List;
3940
import java.util.concurrent.TimeUnit;
@@ -56,9 +57,10 @@
5657
* // - It may require specifying regional endpoints when creating the service client as shown in
5758
* // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
5859
* try (LineageClient lineageClient = LineageClient.create()) {
59-
* LocationName parent = LocationName.of("[PROJECT]", "[LOCATION]");
60-
* Process process = Process.newBuilder().build();
61-
* Process response = lineageClient.createProcess(parent, process);
60+
* String parent = "parent-995424086";
61+
* Struct openLineage = Struct.newBuilder().build();
62+
* ProcessOpenLineageRunEventResponse response =
63+
* lineageClient.processOpenLineageRunEvent(parent, openLineage);
6264
* }
6365
* }</pre>
6466
*
@@ -203,6 +205,111 @@ public final OperationsClient getHttpJsonOperationsClient() {
203205
return httpJsonOperationsClient;
204206
}
205207

208+
// AUTO-GENERATED DOCUMENTATION AND METHOD.
209+
/**
210+
* Creates new lineage events together with their parents: process and run. Updates the process
211+
* and run if they already exist. Mapped from Open Lineage specification:
212+
* https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
213+
*
214+
* <p>Sample code:
215+
*
216+
* <pre>{@code
217+
* // This snippet has been automatically generated and should be regarded as a code template only.
218+
* // It will require modifications to work:
219+
* // - It may require correct/in-range values for request initialization.
220+
* // - It may require specifying regional endpoints when creating the service client as shown in
221+
* // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
222+
* try (LineageClient lineageClient = LineageClient.create()) {
223+
* String parent = "parent-995424086";
224+
* Struct openLineage = Struct.newBuilder().build();
225+
* ProcessOpenLineageRunEventResponse response =
226+
* lineageClient.processOpenLineageRunEvent(parent, openLineage);
227+
* }
228+
* }</pre>
229+
*
230+
* @param parent Required. The name of the project and its location that should own the process,
231+
* run, and lineage event.
232+
* @param openLineage Required. OpenLineage message following OpenLineage format:
233+
* https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json
234+
* @throws com.google.api.gax.rpc.ApiException if the remote call fails
235+
*/
236+
public final ProcessOpenLineageRunEventResponse processOpenLineageRunEvent(
237+
String parent, Struct openLineage) {
238+
ProcessOpenLineageRunEventRequest request =
239+
ProcessOpenLineageRunEventRequest.newBuilder()
240+
.setParent(parent)
241+
.setOpenLineage(openLineage)
242+
.build();
243+
return processOpenLineageRunEvent(request);
244+
}
245+
246+
// AUTO-GENERATED DOCUMENTATION AND METHOD.
247+
/**
248+
* Creates new lineage events together with their parents: process and run. Updates the process
249+
* and run if they already exist. Mapped from Open Lineage specification:
250+
* https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
251+
*
252+
* <p>Sample code:
253+
*
254+
* <pre>{@code
255+
* // This snippet has been automatically generated and should be regarded as a code template only.
256+
* // It will require modifications to work:
257+
* // - It may require correct/in-range values for request initialization.
258+
* // - It may require specifying regional endpoints when creating the service client as shown in
259+
* // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
260+
* try (LineageClient lineageClient = LineageClient.create()) {
261+
* ProcessOpenLineageRunEventRequest request =
262+
* ProcessOpenLineageRunEventRequest.newBuilder()
263+
* .setParent("parent-995424086")
264+
* .setOpenLineage(Struct.newBuilder().build())
265+
* .setRequestId("requestId693933066")
266+
* .build();
267+
* ProcessOpenLineageRunEventResponse response =
268+
* lineageClient.processOpenLineageRunEvent(request);
269+
* }
270+
* }</pre>
271+
*
272+
* @param request The request object containing all of the parameters for the API call.
273+
* @throws com.google.api.gax.rpc.ApiException if the remote call fails
274+
*/
275+
public final ProcessOpenLineageRunEventResponse processOpenLineageRunEvent(
276+
ProcessOpenLineageRunEventRequest request) {
277+
return processOpenLineageRunEventCallable().call(request);
278+
}
279+
280+
// AUTO-GENERATED DOCUMENTATION AND METHOD.
281+
/**
282+
* Creates new lineage events together with their parents: process and run. Updates the process
283+
* and run if they already exist. Mapped from Open Lineage specification:
284+
* https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
285+
*
286+
* <p>Sample code:
287+
*
288+
* <pre>{@code
289+
* // This snippet has been automatically generated and should be regarded as a code template only.
290+
* // It will require modifications to work:
291+
* // - It may require correct/in-range values for request initialization.
292+
* // - It may require specifying regional endpoints when creating the service client as shown in
293+
* // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
294+
* try (LineageClient lineageClient = LineageClient.create()) {
295+
* ProcessOpenLineageRunEventRequest request =
296+
* ProcessOpenLineageRunEventRequest.newBuilder()
297+
* .setParent("parent-995424086")
298+
* .setOpenLineage(Struct.newBuilder().build())
299+
* .setRequestId("requestId693933066")
300+
* .build();
301+
* ApiFuture<ProcessOpenLineageRunEventResponse> future =
302+
* lineageClient.processOpenLineageRunEventCallable().futureCall(request);
303+
* // Do something.
304+
* ProcessOpenLineageRunEventResponse response = future.get();
305+
* }
306+
* }</pre>
307+
*/
308+
public final UnaryCallable<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
309+
processOpenLineageRunEventCallable() {
310+
return stub.processOpenLineageRunEventCallable();
311+
}
312+
206313
// AUTO-GENERATED DOCUMENTATION AND METHOD.
207314
/**
208315
* Creates a new process.
@@ -993,6 +1100,7 @@ public final Run updateRun(Run run, FieldMask updateMask) {
9931100
* UpdateRunRequest.newBuilder()
9941101
* .setRun(Run.newBuilder().build())
9951102
* .setUpdateMask(FieldMask.newBuilder().build())
1103+
* .setAllowMissing(true)
9961104
* .build();
9971105
* Run response = lineageClient.updateRun(request);
9981106
* }
@@ -1022,6 +1130,7 @@ public final Run updateRun(UpdateRunRequest request) {
10221130
* UpdateRunRequest.newBuilder()
10231131
* .setRun(Run.newBuilder().build())
10241132
* .setUpdateMask(FieldMask.newBuilder().build())
1133+
* .setAllowMissing(true)
10251134
* .build();
10261135
* ApiFuture<Run> future = lineageClient.updateRunCallable().futureCall(request);
10271136
* // Do something.

‎java-datalineage/google-cloud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/LineageSettings.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
* <p>The builder of this class is recursive, so contained classes are themselves builders. When
5858
* build() is called, the tree of builders is called to create the complete settings object.
5959
*
60-
* <p>For example, to set the total timeout of createProcess to 30 seconds:
60+
* <p>For example, to set the total timeout of processOpenLineageRunEvent to 30 seconds:
6161
*
6262
* <pre>{@code
6363
* // This snippet has been automatically generated and should be regarded as a code template only.
@@ -67,10 +67,10 @@
6767
* // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
6868
* LineageSettings.Builder lineageSettingsBuilder = LineageSettings.newBuilder();
6969
* lineageSettingsBuilder
70-
* .createProcessSettings()
70+
* .processOpenLineageRunEventSettings()
7171
* .setRetrySettings(
7272
* lineageSettingsBuilder
73-
* .createProcessSettings()
73+
* .processOpenLineageRunEventSettings()
7474
* .getRetrySettings()
7575
* .toBuilder()
7676
* .setTotalTimeout(Duration.ofSeconds(30))
@@ -81,6 +81,12 @@
8181
@Generated("by gapic-generator-java")
8282
public class LineageSettings extends ClientSettings<LineageSettings> {
8383

84+
/** Returns the object with the settings used for calls to processOpenLineageRunEvent. */
85+
public UnaryCallSettings<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
86+
processOpenLineageRunEventSettings() {
87+
return ((LineageStubSettings) getStubSettings()).processOpenLineageRunEventSettings();
88+
}
89+
8490
/** Returns the object with the settings used for calls to createProcess. */
8591
public UnaryCallSettings<CreateProcessRequest, Process> createProcessSettings() {
8692
return ((LineageStubSettings) getStubSettings()).createProcessSettings();
@@ -296,6 +302,13 @@ public Builder applyToAllUnaryMethods(
296302
return this;
297303
}
298304

305+
/** Returns the builder for the settings used for calls to processOpenLineageRunEvent. */
306+
public UnaryCallSettings.Builder<
307+
ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
308+
processOpenLineageRunEventSettings() {
309+
return getStubSettingsBuilder().processOpenLineageRunEventSettings();
310+
}
311+
299312
/** Returns the builder for the settings used for calls to createProcess. */
300313
public UnaryCallSettings.Builder<CreateProcessRequest, Process> createProcessSettings() {
301314
return getStubSettingsBuilder().createProcessSettings();

‎java-datalineage/google-cloud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/gapic_metadata.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@
4949
"ListRuns": {
5050
"methods": ["listRuns", "listRuns", "listRuns", "listRunsPagedCallable", "listRunsCallable"]
5151
},
52+
"ProcessOpenLineageRunEvent": {
53+
"methods": ["processOpenLineageRunEvent", "processOpenLineageRunEvent", "processOpenLineageRunEventCallable"]
54+
},
5255
"SearchLinks": {
5356
"methods": ["searchLinks", "searchLinksPagedCallable", "searchLinksCallable"]
5457
},

‎java-datalineage/google-cloud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/package-info.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@
3535
* // - It may require specifying regional endpoints when creating the service client as shown in
3636
* // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
3737
* try (LineageClient lineageClient = LineageClient.create()) {
38-
* LocationName parent = LocationName.of("[PROJECT]", "[LOCATION]");
39-
* Process process = Process.newBuilder().build();
40-
* Process response = lineageClient.createProcess(parent, process);
38+
* String parent = "parent-995424086";
39+
* Struct openLineage = Struct.newBuilder().build();
40+
* ProcessOpenLineageRunEventResponse response =
41+
* lineageClient.processOpenLineageRunEvent(parent, openLineage);
4142
* }
4243
* }</pre>
4344
*/

‎java-datalineage/google-cloud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/stub/GrpcLineageStub.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@
5050
import com.google.cloud.datacatalog.lineage.v1.ListRunsResponse;
5151
import com.google.cloud.datacatalog.lineage.v1.OperationMetadata;
5252
import com.google.cloud.datacatalog.lineage.v1.Process;
53+
import com.google.cloud.datacatalog.lineage.v1.ProcessOpenLineageRunEventRequest;
54+
import com.google.cloud.datacatalog.lineage.v1.ProcessOpenLineageRunEventResponse;
5355
import com.google.cloud.datacatalog.lineage.v1.Run;
5456
import com.google.cloud.datacatalog.lineage.v1.SearchLinksRequest;
5557
import com.google.cloud.datacatalog.lineage.v1.SearchLinksResponse;
@@ -72,6 +74,20 @@
7274
*/
7375
@Generated("by gapic-generator-java")
7476
public class GrpcLineageStub extends LineageStub {
77+
private static final MethodDescriptor<
78+
ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
79+
processOpenLineageRunEventMethodDescriptor =
80+
MethodDescriptor
81+
.<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>newBuilder()
82+
.setType(MethodDescriptor.MethodType.UNARY)
83+
.setFullMethodName(
84+
"google.cloud.datacatalog.lineage.v1.Lineage/ProcessOpenLineageRunEvent")
85+
.setRequestMarshaller(
86+
ProtoUtils.marshaller(ProcessOpenLineageRunEventRequest.getDefaultInstance()))
87+
.setResponseMarshaller(
88+
ProtoUtils.marshaller(ProcessOpenLineageRunEventResponse.getDefaultInstance()))
89+
.build();
90+
7591
private static final MethodDescriptor<CreateProcessRequest, Process>
7692
createProcessMethodDescriptor =
7793
MethodDescriptor.<CreateProcessRequest, Process>newBuilder()
@@ -227,6 +243,8 @@ public class GrpcLineageStub extends LineageStub {
227243
ProtoUtils.marshaller(BatchSearchLinkProcessesResponse.getDefaultInstance()))
228244
.build();
229245

246+
private final UnaryCallable<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
247+
processOpenLineageRunEventCallable;
230248
private final UnaryCallable<CreateProcessRequest, Process> createProcessCallable;
231249
private final UnaryCallable<UpdateProcessRequest, Process> updateProcessCallable;
232250
private final UnaryCallable<GetProcessRequest, Process> getProcessCallable;
@@ -299,6 +317,18 @@ protected GrpcLineageStub(
299317
this.callableFactory = callableFactory;
300318
this.operationsStub = GrpcOperationsStub.create(clientContext, callableFactory);
301319

320+
GrpcCallSettings<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
321+
processOpenLineageRunEventTransportSettings =
322+
GrpcCallSettings
323+
.<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>newBuilder()
324+
.setMethodDescriptor(processOpenLineageRunEventMethodDescriptor)
325+
.setParamsExtractor(
326+
request -> {
327+
RequestParamsBuilder builder = RequestParamsBuilder.create();
328+
builder.add("parent", String.valueOf(request.getParent()));
329+
return builder.build();
330+
})
331+
.build();
302332
GrpcCallSettings<CreateProcessRequest, Process> createProcessTransportSettings =
303333
GrpcCallSettings.<CreateProcessRequest, Process>newBuilder()
304334
.setMethodDescriptor(createProcessMethodDescriptor)
@@ -463,6 +493,11 @@ protected GrpcLineageStub(
463493
})
464494
.build();
465495

496+
this.processOpenLineageRunEventCallable =
497+
callableFactory.createUnaryCallable(
498+
processOpenLineageRunEventTransportSettings,
499+
settings.processOpenLineageRunEventSettings(),
500+
clientContext);
466501
this.createProcessCallable =
467502
callableFactory.createUnaryCallable(
468503
createProcessTransportSettings, settings.createProcessSettings(), clientContext);
@@ -559,6 +594,12 @@ public GrpcOperationsStub getOperationsStub() {
559594
return operationsStub;
560595
}
561596

597+
@Override
598+
public UnaryCallable<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
599+
processOpenLineageRunEventCallable() {
600+
return processOpenLineageRunEventCallable;
601+
}
602+
562603
@Override
563604
public UnaryCallable<CreateProcessRequest, Process> createProcessCallable() {
564605
return createProcessCallable;

0 commit comments

Comments
 (0)