Guest User

Untitled

a guest
Jun 25th, 2018
97
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.81 KB | None | 0 0
  1. # Find path to PySpark.
  2. import collections
  3. import findspark
  4. findspark.init()
  5.  
  6. # Import PySpark and initialize SparkContext object.
  7. import pyspark
  8. from pyspark.sql import SparkSession
  9.  
  10. conf = pyspark.SparkConf().setMaster("local[*]").setAppName("RatingsHistogram")
  11. sc = pyspark.SparkContext(conf = conf)
  12. spark = SparkSession(sc)
  13.  
  14. print('The Spark Version used is "{0}"n
  15. The python version used is "{1}"n
  16. The default parallelism is "{2}"n
  17. The default number of partitions is "{3}"'.format(sc.version,sc.pythonVer,sc.defaultParallelism,sc.defaultMinPartitions))
  18.  
  19. # Find path to PySpark.
  20. import collections
  21. import findspark
  22. findspark.init()
  23.  
  24. # Import PySpark and initialize SparkContext object.
  25. import pyspark
  26. from pyspark.sql import SparkSession
  27. spark = SparkSession.builder.appName('Basics').getOrCreate()
  28. print(spark.version)
Add Comment
Please, Sign In to add comment