Skip to content

Commit b0d79cd

Browse files
yifanzouyifanzou
committed
[BEAM-5341] create IT test for TfIdf example
1 parent 609a429 commit b0d79cd

2 files changed

Lines changed: 81 additions & 2 deletions

File tree

  • examples/java/src

‎examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -415,8 +415,7 @@ public void processElement(ProcessContext c) {
415415
}
416416
}
417417

418-
public static void main(String[] args) throws Exception {
419-
Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
418+
static void runTfIdf(Options options) throws Exception {
420419
Pipeline pipeline = Pipeline.create(options);
421420
pipeline.getCoderRegistry().registerCoderForClass(URI.class, StringDelegateCoder.of(URI.class));
422421

@@ -427,4 +426,10 @@ public static void main(String[] args) throws Exception {
427426

428427
pipeline.run().waitUntilFinish();
429428
}
429+
430+
public static void main(String[] args) throws Exception {
431+
Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
432+
433+
runTfIdf(options);
434+
}
430435
}

‎examples/java/src/test/java/org/apache/beam/examples/complete/TfIdfIT.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.beam.examples.complete;
20+
21+
import java.util.Date;
22+
import java.util.regex.Pattern;
23+
import org.apache.beam.examples.complete.TfIdf.Options;
24+
import org.apache.beam.sdk.io.FileSystems;
25+
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
26+
import org.apache.beam.sdk.options.PipelineOptionsFactory;
27+
import org.apache.beam.sdk.testing.FileChecksumMatcher;
28+
import org.apache.beam.sdk.testing.TestPipeline;
29+
import org.apache.beam.sdk.testing.TestPipelineOptions;
30+
import org.junit.BeforeClass;
31+
import org.junit.Test;
32+
import org.junit.runner.RunWith;
33+
import org.junit.runners.JUnit4;
34+
35+
/** Integration test for TfIdf example. */
36+
@RunWith(JUnit4.class)
37+
public class TfIdfIT {
38+
39+
private static final String DEFAULT_INPUT = "gs://apache-beam-samples/shakespeare/";
40+
private static final String EXPECTED_OUTPUT_CHECKSUM = "f072786dde51dc09fc74bf38ffbfc27dcfdf0b96";
41+
private static final Pattern DEFAULT_SHARD_TEMPLATE =
42+
Pattern.compile("(?x) \\S* (?<shardnum> \\d+) -of- (?<numshards> \\d+)\\.csv");
43+
44+
/**
45+
* Options for the TfIdf Integration Test.
46+
*
47+
* <p>Define expected output file checksum to verify TfIdf pipeline result with customized input.
48+
*/
49+
public interface TfIdfITOptions extends TestPipelineOptions, Options {}
50+
51+
@BeforeClass
52+
public static void setUp() {
53+
PipelineOptionsFactory.register(TfIdfITOptions.class);
54+
}
55+
56+
@Test
57+
public void testE2ETfIdf() throws Exception {
58+
TfIdfITOptions options = TestPipeline.testingPipelineOptions().as(TfIdfITOptions.class);
59+
options.setInput(DEFAULT_INPUT);
60+
options.setOutput(
61+
FileSystems.matchNewResource(options.getTempRoot(), true)
62+
.resolve(
63+
String.format("TfIdfIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
64+
StandardResolveOptions.RESOLVE_DIRECTORY)
65+
.resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY)
66+
.resolve("results", StandardResolveOptions.RESOLVE_FILE)
67+
.toString());
68+
options.setOnSuccessMatcher(
69+
new FileChecksumMatcher(
70+
EXPECTED_OUTPUT_CHECKSUM, options.getOutput() + "*-of-*.csv", DEFAULT_SHARD_TEMPLATE));
71+
72+
TfIdf.runTfIdf(options);
73+
}
74+
}

0 commit comments

Comments
 (0)