Skip to content

Commit 04a6665

Browse files
eaball35eaball35
authored
feat: write samples to file (pt3) (#980)
* feat: write to files * test: update test scripts * test: goldens * feat: sample src jar * feat: package tar with samples * refactor: sample class and file names should match * test: unit goldens * test: golden IT * refactor: keep samples seperate from gapic jar * test: ComposerTest * formatting * fix test * refactor: updateSample naming * refactor: include cause with GapicWriterException * ignore test files in snippetbot check * update composeSamples name * include javadoc comments * formatting
1 parent df5e9e0 commit 04a6665

1,017 files changed

Lines changed: 34473 additions & 54 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/snippet-bot.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ aggregateChecks: false
22
alwaysCreateStatusCheck: false
33
ignoreFiles:
44
- src/test/**
5+
- test/**

‎rules_java_gapic/java_gapic.bzl‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ def _java_gapic_postprocess_srcjar_impl(ctx):
2121
output_srcjar_name = ctx.label.name
2222
output_main = ctx.outputs.main
2323
output_test = ctx.outputs.test
24+
output_samples = ctx.outputs.samples
2425
output_resource_name = ctx.outputs.resource_name
2526
formatter = ctx.executable.formatter
2627

@@ -58,11 +59,16 @@ def _java_gapic_postprocess_srcjar_impl(ctx):
5859
cd $WORKING_DIR/{output_dir_path}/src/test/java
5960
zip -r $WORKING_DIR/{output_srcjar_name}-tests.srcjar ./
6061
62+
# Sample source files.
63+
cd $WORKING_DIR/{output_dir_path}/samples/snippets/generated/src/main/java
64+
zip -r $WORKING_DIR/{output_srcjar_name}-samples.srcjar ./
65+
6166
cd $WORKING_DIR
6267
6368
mv {output_srcjar_name}.srcjar {output_main}
6469
mv {output_srcjar_name}-resource-name.srcjar {output_resource_name}
6570
mv {output_srcjar_name}-tests.srcjar {output_test}
71+
mv {output_srcjar_name}-samples.srcjar {output_samples}
6672
""".format(
6773
gapic_srcjar = gapic_srcjar.path,
6874
output_srcjar_name = output_srcjar_name,
@@ -72,13 +78,14 @@ def _java_gapic_postprocess_srcjar_impl(ctx):
7278
output_main = output_main.path,
7379
output_resource_name = output_resource_name.path,
7480
output_test = output_test.path,
81+
output_samples = output_samples.path,
7582
)
7683

7784
ctx.actions.run_shell(
7885
inputs = [gapic_srcjar],
7986
tools = [formatter],
8087
command = script,
81-
outputs = [output_main, output_resource_name, output_test],
88+
outputs = [output_main, output_resource_name, output_test, output_samples],
8289
)
8390

8491
_java_gapic_postprocess_srcjar = rule(
@@ -94,10 +101,66 @@ _java_gapic_postprocess_srcjar = rule(
94101
"main": "%{name}.srcjar",
95102
"resource_name": "%{name}-resource-name.srcjar",
96103
"test": "%{name}-test.srcjar",
104+
"samples": "%{name}-samples.srcjar",
97105
},
98106
implementation = _java_gapic_postprocess_srcjar_impl,
99107
)
100108

109+
def _java_gapic_samples_srcjar_impl(ctx):
110+
gapic_srcjar = ctx.file.gapic_srcjar
111+
output_srcjar_name = ctx.label.name
112+
output_samples = ctx.outputs.samples
113+
formatter = ctx.executable.formatter
114+
115+
output_dir_name = ctx.label.name
116+
output_dir_path = "%s/%s" % (output_samples.dirname, output_dir_name)
117+
118+
script = """
119+
unzip -q {gapic_srcjar}
120+
# Sync'd to the output file name in Writer.java.
121+
unzip -q temp-codegen.srcjar -d {output_dir_path}
122+
# This may fail if there are spaces and/or too many files (exceed max length of command length).
123+
{formatter} --replace $(find {output_dir_path} -type f -printf "%p ")
124+
WORKING_DIR=`pwd`
125+
126+
# Sample source files.
127+
cd $WORKING_DIR/{output_dir_path}/samples/snippets/generated/src/main/java
128+
zip -r $WORKING_DIR/{output_srcjar_name}-samples.srcjar ./
129+
130+
cd $WORKING_DIR
131+
132+
mv {output_srcjar_name}-samples.srcjar {output_samples}
133+
""".format(
134+
gapic_srcjar = gapic_srcjar.path,
135+
output_srcjar_name = output_srcjar_name,
136+
formatter = formatter,
137+
output_dir_name = output_dir_name,
138+
output_dir_path = output_dir_path,
139+
output_samples = output_samples.path,
140+
)
141+
142+
ctx.actions.run_shell(
143+
inputs = [gapic_srcjar],
144+
tools = [formatter],
145+
command = script,
146+
outputs = [output_samples],
147+
)
148+
149+
_java_gapic_samples_srcjar = rule(
150+
attrs = {
151+
"gapic_srcjar": attr.label(mandatory = True, allow_single_file = True),
152+
"formatter": attr.label(
153+
default = Label("//:google_java_format_binary"),
154+
executable = True,
155+
cfg = "host",
156+
),
157+
},
158+
outputs = {
159+
"samples": "%{name}-samples.srcjar",
160+
},
161+
implementation = _java_gapic_samples_srcjar_impl,
162+
)
163+
101164
def _extract_common_proto_dep(dep):
102165
return dep[dep.index("/"):] if "//google" in dep else dep
103166

@@ -198,6 +261,12 @@ def java_gapic_library(
198261
**kwargs
199262
)
200263

264+
_java_gapic_samples_srcjar(
265+
name = "%s_samples" % name,
266+
gapic_srcjar = "%s.srcjar" % raw_srcjar_name,
267+
**kwargs
268+
)
269+
201270
resource_name_name = "%s_resource_name" % name
202271
resource_name_deps = [resource_name_name]
203272
native.java_library(

‎rules_java_gapic/java_gapic_pkg.bzl‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,18 +63,29 @@ def _gapic_pkg_tar_impl(ctx):
6363
for f in dep.files.to_list():
6464
deps.append(f)
6565

66+
samples =[]
67+
for s in ctx.attr.samples:
68+
for f in s.files.to_list():
69+
samples.append(f)
70+
6671
paths = _construct_package_dir_paths(
6772
ctx.attr.package_dir,
6873
ctx.outputs.pkg,
6974
ctx.label.name,
7075
)
7176

7277
script = """
78+
for s in {samples}; do
79+
mkdir -p {package_dir_path}/{tar_cd_suffix}/{tar_prefix}/samples/snippets/generated/
80+
unzip -q ./$s -d {package_dir_path}/{tar_cd_suffix}/{tar_prefix}/samples/snippets/generated/
81+
done
82+
7383
mkdir -p {package_dir_path}
7484
for dep in {deps}; do
7585
tar -xzpf $dep -C {package_dir_path}
7686
done
7787
cd {package_dir_path}/{tar_cd_suffix}
88+
7889
tar -zchpf {tar_prefix}/{package_dir}.tar.gz {tar_prefix}/*
7990
cd -
8091
mv {package_dir_path}/{package_dir}.tar.gz {pkg}
@@ -83,13 +94,14 @@ def _gapic_pkg_tar_impl(ctx):
8394
deps = " ".join(["'%s'" % d.path for d in deps]),
8495
package_dir_path = paths.package_dir_path,
8596
package_dir = paths.package_dir,
97+
samples = " ".join(["'%s'" % s.path for s in samples]),
8698
pkg = ctx.outputs.pkg.path,
8799
tar_cd_suffix = paths.tar_cd_suffix,
88100
tar_prefix = paths.tar_prefix,
89101
)
90102

91103
ctx.actions.run_shell(
92-
inputs = deps,
104+
inputs = deps + samples,
93105
command = script,
94106
outputs = [ctx.outputs.pkg],
95107
)
@@ -101,6 +113,7 @@ def _gapic_pkg_tar_impl(ctx):
101113
gapic_pkg_tar = rule(
102114
attrs = {
103115
"deps": attr.label_list(mandatory = True),
116+
"samples": attr.label_list(mandatory = False),
104117
"package_dir": attr.string(mandatory = False, default = ""),
105118
"extension": attr.string(mandatory = False, default = "tar.gz"),
106119
},
@@ -256,6 +269,10 @@ def _java_gapic_srcs_pkg_impl(ctx):
256269
# Remove empty files. If there are no resource names, one such file might have
257270
# been created. See java_gapic.bzl.
258271
rm $(find {package_dir_path}/src/main/java -size 0)
272+
273+
if [ -d {package_dir_path}/src/main/java/samples ]; then
274+
mv {package_dir_path}/src/main/java/samples {package_dir_path}
275+
fi
259276
done
260277
for proto_src in {proto_srcs}; do
261278
mkdir -p {package_dir_path}/src/main/proto
@@ -319,11 +336,14 @@ def java_gapic_assembly_gradle_pkg(
319336
client_test_deps = []
320337
grpc_deps = []
321338
proto_deps = []
339+
samples = []
322340

323341
processed_deps = {} #there is no proper Set in Starlark
324342
for dep in deps:
325343
# Use contains instead of endswith since microgenerator testing may use differently-named targets.
326-
if "_java_gapic" in dep:
344+
if "samples" in dep:
345+
samples.append(dep)
346+
elif "_java_gapic" in dep:
327347
_put_dep_in_a_bucket(dep, client_deps, processed_deps)
328348
_put_dep_in_a_bucket("%s_test" % dep, client_test_deps, processed_deps)
329349
_put_dep_in_a_bucket("%s_resource_name" % dep, proto_deps, processed_deps)
@@ -371,6 +391,7 @@ def java_gapic_assembly_gradle_pkg(
371391
name = name,
372392
assembly_name = package_dir,
373393
deps = proto_target_dep + grpc_target_dep + client_target_dep,
394+
samples = samples,
374395
)
375396

376397
def _java_gapic_gradle_pkg(
@@ -416,7 +437,7 @@ def _java_gapic_gradle_pkg(
416437
**kwargs
417438
)
418439

419-
def _java_gapic_assembly_gradle_pkg(name, assembly_name, deps, visibility = None):
440+
def _java_gapic_assembly_gradle_pkg(name, assembly_name, deps, samples = None, visibility = None):
420441
resource_target_name = "%s-resources" % assembly_name
421442
java_gapic_build_configs_pkg(
422443
name = resource_target_name,
@@ -434,6 +455,7 @@ def _java_gapic_assembly_gradle_pkg(name, assembly_name, deps, visibility = None
434455
Label("//rules_java_gapic:gradlew"),
435456
resource_target_name,
436457
] + deps,
458+
samples = samples,
437459
package_dir = assembly_name,
438460
visibility = visibility,
439461
)

‎scripts/diff_gen_and_golden.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ RAW_SRCJAR=$( find . -name '*_java_gapic_srcjar_raw.srcjar' )
88
mkdir unpacked src
99
cd unpacked
1010
unzip -q -c "../${RAW_SRCJAR}" temp-codegen.srcjar | jar x
11+
12+
mkdir -p ../samples/snippets/generated/
13+
cp -r samples/snippets/generated/**/* ../samples/snippets/generated
14+
1115
cp -r src/main/java/* ../src
1216
cp -r src/test/java/* ../src
1317
[ -d proto ] && cp -r proto/src/main/java/* ../src
@@ -17,7 +21,7 @@ cd ..
1721
find src -type f ! -name '*.java' -a ! -name '*gapic_metadata.json' -delete
1822
find src -type f -name 'PlaceholderFile.java' -delete
1923
find src -type d -empty -delete
20-
2124
# This will not print diff_output to the console unless `--test_output=all` option
2225
# is enabled, it only emits the comparison results to the test.log.
23-
diff -ru src test/integration/goldens/${API_NAME}
26+
diff -ru src test/integration/goldens/${API_NAME}/src
27+
diff -ru samples test/integration/goldens/${API_NAME}/samples

‎scripts/update_golden.sh‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,13 @@ cd ${BUILD_WORKSPACE_DIRECTORY}/test/integration/goldens/${API_NAME}
1717
find . -name '*.java' -delete
1818
find . -name 'gapic_metadata.json' -delete
1919

20-
cp -r ${UNPACK_DIR}/src/main/java/* .
21-
cp -r ${UNPACK_DIR}/src/test/java/* .
22-
[ -d ${UNPACK_DIR}/proto ] && cp -r ${UNPACK_DIR}/proto/src/main/java/* .
20+
mkdir -p ./src
21+
cp -r ${UNPACK_DIR}/src/main/java/* ./src
22+
cp -r ${UNPACK_DIR}/src/test/java/* ./src
23+
[ -d ${UNPACK_DIR}/proto ] && cp -r ${UNPACK_DIR}/proto/src/main/java/* ./src
24+
25+
mkdir -p ./samples/snippets/generated
26+
cp -r ${UNPACK_DIR}/samples/snippets/generated/**/* ./samples/snippets/generated
2327

2428
find . -name 'PlaceholderFile.java' -delete
2529
find . -type d -empty -delete

‎src/main/java/com/google/api/generator/gapic/composer/Composer.java‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@
3232
import com.google.api.generator.gapic.model.GapicClass;
3333
import com.google.api.generator.gapic.model.GapicContext;
3434
import com.google.api.generator.gapic.model.GapicPackageInfo;
35+
import com.google.api.generator.gapic.model.Sample;
3536
import com.google.api.generator.gapic.model.Service;
3637
import com.google.api.generator.gapic.model.Transport;
3738
import com.google.common.annotations.VisibleForTesting;
3839
import java.util.ArrayList;
40+
import java.util.Arrays;
3941
import java.util.List;
4042
import java.util.stream.Collectors;
4143

@@ -45,7 +47,8 @@ public static List<GapicClass> composeServiceClasses(GapicContext context) {
4547
clazzes.addAll(generateServiceClasses(context));
4648
clazzes.addAll(generateMockClasses(context, context.mixinServices()));
4749
clazzes.addAll(generateResourceNameHelperClasses(context));
48-
return addApacheLicense(clazzes);
50+
return addApacheLicense(
51+
prepareExecutableSamples(clazzes, context.gapicMetadata().getProtoPackage()));
4952
}
5053

5154
public static GapicPackageInfo composePackageInfo(GapicContext context) {
@@ -186,6 +189,42 @@ public static List<GapicClass> generateTestClasses(GapicContext context) {
186189
return clazzes;
187190
}
188191

192+
@VisibleForTesting
193+
static List<GapicClass> prepareExecutableSamples(List<GapicClass> clazzes, String protoPackage) {
194+
// parse protoPackage for apiVersion and apiShortName
195+
String[] pakkage = protoPackage.split("\\.");
196+
String apiVersion;
197+
String apiShortName;
198+
// e.g. v1, v2, v1beta1
199+
if (pakkage[pakkage.length - 1].matches("v[0-9].*")) {
200+
apiVersion = pakkage[pakkage.length - 1];
201+
apiShortName = pakkage[pakkage.length - 2];
202+
} else {
203+
apiVersion = "";
204+
apiShortName = pakkage[pakkage.length - 1];
205+
}
206+
// Include license header, apiShortName, and apiVersion
207+
return clazzes.stream()
208+
.map(
209+
gapicClass -> {
210+
List<Sample> samples =
211+
gapicClass.samples().stream()
212+
.map(
213+
sample -> addRegionTagAndHeaderToSample(sample, apiShortName, apiVersion))
214+
.collect(Collectors.toList());
215+
return gapicClass.withSamples(samples);
216+
})
217+
.collect(Collectors.toList());
218+
}
219+
220+
private static Sample addRegionTagAndHeaderToSample(
221+
Sample sample, String apiShortName, String apiVersion) {
222+
return sample
223+
.withHeader(Arrays.asList(CommentComposer.APACHE_LICENSE_COMMENT))
224+
.withRegionTag(
225+
sample.regionTag().withApiVersion(apiVersion).withApiShortName(apiShortName));
226+
}
227+
189228
@VisibleForTesting
190229
protected static List<GapicClass> addApacheLicense(List<GapicClass> gapicClassList) {
191230
return gapicClassList.stream()
@@ -197,7 +236,7 @@ protected static List<GapicClass> addApacheLicense(List<GapicClass> gapicClassLi
197236
.toBuilder()
198237
.setFileHeader(CommentComposer.APACHE_LICENSE_COMMENT)
199238
.build();
200-
return GapicClass.create(gapicClass.kind(), classWithHeader);
239+
return GapicClass.create(gapicClass.kind(), classWithHeader, gapicClass.samples());
201240
})
202241
.collect(Collectors.toList());
203242
}

0 commit comments

Comments
 (0)