自定义函数utf函数(1对1函数)
package SparkSQL.fun.registerfum
import org.apache.spark.SparkConf
import org.apache.spark.sql.{Dataset, SparkSession}
import scala.beans.BeanProperty
case class student(@BeanProperty var name: String, @BeanProperty var clz: String, @BeanProperty var age: Int, @BeanProperty var sex: String)
object registerfun1 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("func1").setMaster("local[*]")
val session = SparkSession.builder().config(conf).getOrCreate()
import session.implicits._
val dataset: Dataset[student] = session.createDataset(Array(
student("zs", "c001", 21, "男"),
student("ls", "c001", 22, "女"),
student("ww", "c001", 23, "男"),
student("ml", "c002", 20, "女"),
student("zb", "c002", 23, "男")
))
// 将每个age字段 +1
session.udf.register("age_auto", (age: Int, num: Int) => {
age + num
})
dataset.createTempView("student")
val frame = session.sql("select name, age_auto(age, 5), sex, clz from student")
frame.show()
session.stop()
}
}
本文来自博客园,作者:jsqup,转载请注明原文链接:https://www.cnblogs.com/jsqup/p/16659618.html