pyspark list[dict]转pyspark df
数据处理把 list嵌套字段转成 pyspark dataframe
# coding=utf-8 from pyspark.sql import SparkSession from pyspark.sql.types import * import pandas as pd from pyspark.sql import Row class SparkContext: def __init__(self, name="cleaner"): self.spark = ( SparkSession.builder.appName(name) .config("hive.exec.dynamic.partition", True) .config("hive.exec.dynamic.partition.mode", "nonstrict") .enableHiveSupport() .getOrCreate() ) self.spark.sparkContext.setLogLevel("ERROR") def __enter__(self): return self.spark def __exit__(self, exc_type, exc_val, exc_tb): self.spark.stop() def main(): data = [{'ent_name': '百度', 'credit_code': '1234567890'}, {'ent_name': 'abc', 'credit_code': '121212222'} ] row_data = [Row(**row) for row in data] with SparkContext('test_df') as spark: df = spark.createDataFrame(row_data) df.show() if __name__ == '__main__': main()