您现在的位置是:主页 > Web前端技术 > Web前端技术

Spark Stream怎么使用开发技术

IDCBT2021-12-19服务器技术人已围观

简介这篇文章主要讲解了“Spark Stream怎么使用”,文中的讲解内容简单清晰,易于学习与理解,下面请大家跟着小编的思路慢慢深入,一起来研究和学习“Spark Stream怎么使用”吧! pom.xml d

这篇文章主要讲解了“Spark Stream怎么使用”,文中的讲解内容简单清晰,易于学习与理解,下面请大家跟着小编的思路慢慢深入,一起来研究和学习“Spark Stream怎么使用”吧!

pom.xml
<dependency>

        <groupId>org.apache.spark</groupId>

        <artifactId>spark-streaming_2.11</artifactId>

        <version>2.1.0</version>

    </dependency>

    import java.io.OutputStream;

    import java.net.ServerSocket;

    import java.net.Socket;

    import java.util.Arrays;

    import java.util.UUID;


    import org.apache.spark.SparkConf;

    import org.apache.spark.streaming.Durations;

    import org.apache.spark.streaming.api.java.JavaDStream;

    import org.apache.spark.streaming.api.java.JavaPairDStream;

    import org.apache.spark.streaming.api.java.JavaStreamingContext;


    import scala.Tuple2;


    public class SparkStreamTest {


        public static void main(String[] args) throws Exception {

            startSockerServer(9999);

            

            SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount");

            JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(1));

            JavaDStream<String> lines = jssc.socketTextStream("localhost", 9999);


            JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(x.split(" ")).iterator());

            JavaPairDStream<String, Integer> pairs = words.mapToPair(s -> new Tuple2<>(s, 1));

            JavaPairDStream<String, Integer> counts = pairs.reduceByKey((x, y) -> x + y);

            counts.print();

            

            jssc.start();

            jssc.awaitTermination();

            jssc.close();

        }


        private static void startSockerServer(int port) throws Exception {

            new Thread(){

    标签:

    很赞哦! ()

本栏推荐