Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- test.txt
- H,DatatypeCode,Description
- K,03099,Metric Expanded mple Size
- K,05307,Elapsed Day Factor
- val customSchema=StructType(StructField(head_record,StringType,false), StructField(data_typ_cd,StringType,false), StructField(data_typ_desc,StringType,false))
- var url =
- jdbc:hive2://xxhive.xxx.com:10000/devl_ccc_chx_batch;principal=hive/xxhive.xxx.com@internal.xxxglobal.com;mapred.job.queue.name=opp;AuthMech=3;SSL=1;password=xxxx;user=xxxxx;
- driverName=com.cloudera.hive.jdbc41.HS2Driver
- val schema = StructType(List())
- val sc = sparkSession.sparkContext
- var readDF = sparkSession.createDataFrame(sc.emptyRDD[Row], schema)
- readDF = sparkSession.read
- .format("com.databricks.spark.csv")
- .option("delimiter", ",")
- .option("header", "true")
- .schema(customSchema)
- .load("test.txt")
- readDF=readDF.filter(col("head_record")===lit("K")).drop("head_record")
- readDF=readDF.withColumn("cyc_dt",lit("201867").cast(IntegerType))
- .withColumn("proc_dt",unix_timestamp(lit(current_timestamp()),"yyyy-MM-dd HH:mm:ss").cast("timestamp"))
- val connectionProperties = new Properties()
- connectionProperties.put("user", xxxxxx)
- connectionProperties.put("password", xxxxxx)
- dataFrame.createOrReplaceTempView("tmpTable")
- sparkSession.sql("select * from tmpTable")
- .write
- .mode(SaveMode.Append) // <--- Append in existing table
- .option("driver", driverName)
- .option("header","false")
- .jdbc(url, "tableName", connectionProperties)
- *org.apache.hadoop.hive.ql.parse.SemanticException:Expression of type TOK_FUNCTION not supported in insert/values:27:1, org.apache.hadoop.hive.ql.parse.SemanticAnalyzer:unparseExprForValuesClause:SemanticAnalyzer.java:914, org.apache.hadoop.hive.ql.parse.SemanticAnalyzer:genValuesTempTable:SemanticAnalyzer.java:842], sqlState:42000, errorCode:10293, errorMessage:Error while compiling statement: FAILED: SemanticException [Error 10293]: Unable to create temp file for insert values Expression of type TOK_FUNCTION not supported in insert/values), Query: INSERT INTO TABLE `devl_ca9_chx_batch`.`chx_data_typs_stg_dcp` VALUES ( '03099' , 'Volumetric Expanded Sample Size' , 201867 , CAST( '2018-06-02 21:25:41.0' AS TIMESTAMP) ) , ( '03307' , 'Elapsed Day Factor' , 201867 , CAST( '2018-06-02 21:25:41.0' AS TIMESTAMP) ).rnt... 18 morernnDriver stacktrace:"}
Add Comment
Please, Sign In to add comment