pyspark——Rdd与DataFrame相互转换

Rdd转DataFrame

?

1 2 3 4 5 6 7 8 9 10 11 12

from pyspark.sql.types import * from pyspark import SparkContext,SparkConf from  pyspark.sql import SparkSession   spark=SparkSession.builder.appName("boye").getOrCreate() sc = spark.sparkContext textFile = sc.textFile("file:///usr/local/test/urls") rdd = textFile.map(lambda x:x.split("\t")).filter(lambda x:len(x)==2) df = spark.createDataFrame(rdd,schema=["rowkey","url"]) df.write.format("json").mode("overwrite").save("file:///usr/local/test/outPut") #保存数据 df.write.save(path='/usr/local/test/csv', format='csv', mode='overwrite', sep='\t') #保存为csv文件 df.write.mode("overwrite").saveAsTable("ss") #永久保存

DataFrame转Rdd　　

from pyspark import SparkContext,SparkConf
from  pyspark.sql import SparkSession
spark=SparkSession.builder.appName("boye").getOrCreate()
sc = spark.sparkContext
df = spark.read.json("file:///usr/local/test/01.json")
rdd = df.select("name","age").limit(10).rdd
rdd = rdd.map(lambda d:"{}\t{}".format(d.name,d.age) )
rdd.saveAsTextFile("file:///usr/local/test/rdd_json")
#rdd.repartition(1).saveAsTextFile("file:///usr/local/test/rdd1")