daily pastebin goal
81%
SHARE
TWEET

Untitled

a guest Mar 21st, 2019 58 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. : org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1.0 failed 1 times, most recent failure: Lost task 0.0 in stage 1.0 (TID 1, localhost, executor driver): java.lang.NoClassDefFoundError: com/twitter/jsr166e/LongAdder
  2.      
  3. import sys
  4. import uuid
  5. import time
  6. import os
  7.  
  8. os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.datastax.spark:spark-cassandra-connector_2.11:2.4.0 pyspark-shell'
  9.  
  10. try:
  11.     from pyspark import SparkContext
  12.     from pyspark import SparkConf
  13.     from pyspark.sql import SparkSession
  14.     from itertools import islice
  15.     from pyspark.sql import SQLContext
  16.     from pyspark.sql.types import *
  17.     from pyspark.sql import Row
  18.     from datetime import datetime
  19.  
  20. except ImportError as e:
  21.     print("error importing spark modules", e)
  22.     sys.exit(1)
  23.  
  24. conf = SparkConf().setAppName("Stand Alone Python Script").setMaster("local[*]")
  25.                         .setAll([('spark.executor.memory', '8g'),
  26.                         ('spark.executor.cores', '3'),
  27.                         ('spark.cores.max', '3'),
  28.                         ('spark.cassandra.connection.host', 'cassandra_ip'),
  29.                         ('spark.cassandra.auth.username', 'cassandra_user_name'),
  30.                         ('spark.cassandra.auth.password', 'cassandra_password'),
  31.                         ('spark.driver.memory','8g')])
  32.  
  33. sc = SparkContext(conf=conf)
  34. sql_context = SQLContext(sc)
  35.  
  36.  
  37.  
  38. consumer_complaints = sql_context.read.format("csv").option("header", "true").option("inferSchema", "false").load("in/Consumer_Complaints.csv")
  39. consumer_complaints.write
  40.     .format("org.apache.spark.sql.cassandra")
  41.     .mode('append')
  42.     .options(table="table_name", keyspace="space_name")
  43.     .save()
  44.  
  45.  
  46. sc.stop()
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top