from pyspark.sql import SparkSession
# from pyspark import SparkConf, SparkContext
def init_spark():
# conf = SparkConf().setAppName("normal spark") \
# .setMaster("local") \
# .set("spark.io.compression.codec", "org.apache.spark.io.SnappyCompressionCodec") \
# .set("spark.driver.extraLibraryPath", "D:\\app\\spark-2.4.7-bin-hadoop2.7\\jars") \
# .set("spark.driver.extraClassPath", "D:\\app\\spark-2.4.7-bin-hadoop2.7\\jars") \
# .set("spark.executor.extraLibraryPath", "D:\\app\\spark-2.4.7-bin-hadoop2.7\\jars") \
# .set("spark.executor.extraClassPath", "D:\\app\\spark-2.4.7-bin-hadoop2.7\\jars")
#
# sc = SparkContext(conf=conf)
spark = SparkSession.builder.appName("HelloWorld").getOrCreate()
sc = spark.sparkContext
return spark,sc
def main():
spark,sc = init_spark()
# nums = sc.parallelize([1,2,3,4])
rdd = sc.textFile("file:///D:\\data\\web_learning\\0118\\metadatas192.168.10.3_20210118_0000.snappy")
# rdd = sc.textFile("file:///D:\\data\\test.txt")
print("*"*88)
# print(nums.map(lambda x: x*x).collect())
print(rdd.collect())
print("*"*88)
if __name__ == '__main__':
main()
# System.setProperty("hadoop.home.dir", "D:\\app\\hadoop-2.7.3")
# > D:\app\spark - 2.4.7 - bin - hadoop2.7\bin\spark - submit - Djava.library.path = D:\app\hadoop-2.7.3\bin web_learning.py
