flinkWordCount

2020-04-06 　　22:23:26

// 流处理
object FlinkStreamingWordCount {
  def main(args: Array[String]): Unit = {
    // 1.初始化流计算环境
    val streamEnv: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    streamEnv.setParallelism(4) //设置并行度


    // 2. 导入隐式转换
    import org.apache.flink.streaming.api.scala._

    // 3. 读取数据
    val stream: DataStream[String] = streamEnv.socketTextStream("xw1", 6666) // 设置端口和主机

    // 4. 转换和处理数据
    val result: DataStream[(String, Int)] = stream.flatMap(_.split("\\s+"))
      .map((_, 1))
      .keyBy(0)
      .sum(1)

    // 5. 打印结果
    result.print("res:")

    // 6. 启动流计算程序
    streamEnv.execute("myWordCount")

  }
}

// 批处理

object BatchWordCount {
  def main(args: Array[String]): Unit = {
    // 1.初始化批计算环境
    val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(4) //设置并行度

    // 2. 导入隐式转换
    import org.apache.flink.api.scala._


    // 3. 读取数据
    val dataPath: URL = getClass.getResource("/wc.txt")
    val data: DataSet[String] = env.readTextFile(dataPath.getPath)

    // 4. 转换和处理数据
    data.flatMap(_.split("\\s+"))
        .map((_,1))
        .groupBy(0)
        .sum(1)
        .print()
  }
}