use reduce by key to implement word count

James Lee · James Lee · commit e4e9377c3925 · 2017-01-15T16:01:51.000Z
diff --git a/src/main/java/com/sparkTutorial/pairRdd/aggregation/reducebykey/WorldCount.java b/src/main/java/com/sparkTutorial/pairRdd/aggregation/reducebykey/WorldCount.java
@@ -0,0 +1,37 @@
+package com.sparkTutorial.pairRdd.aggregation.reducebykey;
+
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.api.java.function.PairFunction;
+import scala.Tuple2;
+
+import java.util.Arrays;
+import java.util.Map;
+
+public class WorldCount {
+
+    public static void main(String[] args) throws Exception {
+
+        Logger.getLogger("org").setLevel(Level.ERROR);
+        SparkConf conf = new SparkConf().setAppName("wordCounts").setMaster("local[3]");
+        JavaSparkContext sc = new JavaSparkContext(conf);
+
+        JavaRDD<String> lines = sc.textFile("in/word_count.text");
+        JavaRDD<String> wordRdd = lines.flatMap(line -> Arrays.asList(line.split(" ")).iterator());
+
+        JavaPairRDD<String, Integer> wordPairRdd = wordRdd.mapToPair((PairFunction<String, String, Integer>) word -> new Tuple2<>(word, 1));
+
+        JavaPairRDD<String, Integer> wordCounts = wordPairRdd.reduceByKey((Function2<Integer, Integer, Integer>) (x, y) -> x + y);
+
+        for (Map.Entry<String, Integer> wordCountPair : wordCounts.collectAsMap().entrySet()) {
+            System.out.println(wordCountPair.getKey() + " : " + wordCountPair.getValue());
+
+        }
+    }
+}
