Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- // boilerplate when writing python spark script
- import sys
- from pyspark import SparkContext, SparkConf
- if __name__ == "__main__":
- conf = (SparkConf()
- .setAppName("The App")
- .setMaster("local)
- .set("spark.executor.memory", "1g"))
- sc = SparkContext(conf=conf)
- // boilerplate is finished
- // finally, there is some useful work to do
- // usually, you want to use the arguments to run a script
- // bur, remember that first argument will be the script itself
- // sys.argv[0] = 'your_running_py_script.py'
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement