Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # Find path to PySpark.
- import collections
- import findspark
- findspark.init()
- # Import PySpark and initialize SparkContext object.
- import pyspark
- from pyspark.sql import SparkSession
- conf = pyspark.SparkConf().setMaster("local[*]").setAppName("RatingsHistogram")
- sc = pyspark.SparkContext(conf = conf)
- spark = SparkSession(sc)
- print('The Spark Version used is "{0}"n
- The python version used is "{1}"n
- The default parallelism is "{2}"n
- The default number of partitions is "{3}"'.format(sc.version,sc.pythonVer,sc.defaultParallelism,sc.defaultMinPartitions))
- # Find path to PySpark.
- import collections
- import findspark
- findspark.init()
- # Import PySpark and initialize SparkContext object.
- import pyspark
- from pyspark.sql import SparkSession
- spark = SparkSession.builder.appName('Basics').getOrCreate()
- print(spark.version)
Add Comment
Please, Sign In to add comment