Guest User

Untitled

a guest
Dec 13th, 2017
345
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.31 KB | None | 0 0
  1. SparkSession spark = SparkSession.builder().appName("Spark SQL Test")
  2. .master("local[*]").getOrCreate();
  3.  
  4. Properties connectionProperties = new Properties();
  5. connectionProperties.put("driver", "com.mysql.jdbc.Driver");
  6. connectionProperties.put("url", "jdbc:mysql://localhost:3306/test");
  7. connectionProperties.put("user", "root");
  8. connectionProperties.put("password", "password");
  9.  
  10. Dataset<Row> jdbcDF = spark.read().jdbc(connectionProperties.getProperty("url"), "family", connectionProperties);
  11. jdbcDF.show();
  12. jdbcDF.printSchema();
  13.  
  14. +------+----------+--------+-------+
  15. |EMP_ID|EMP_PASSWD|EMP_NAME|EMP_AGE|
  16. +------+----------+--------+-------+
  17. | jina| bbb| mother| 45|
  18. |joseph| aaa| father| 50|
  19. |julian| ccc| son | 20|
  20. +------+----------+--------+-------+
  21.  
  22.  
  23. root
  24. |-- EMP_ID: string (nullable = false)
  25. |-- EMP_PASSWD: string (nullable = false)
  26. |-- EMP_NAME: string (nullable = false)
  27. |-- EMP_AGE: integer (nullable = false)
  28.  
  29. Dataset<Row> appendSql = spark.sql("INSERT INTO family VALUES('jane' , 'ddd', 'daughter' , 15)");
  30.  
  31. +------+----------+--------+-------+
  32. |EMP_ID|EMP_PASSWD|EMP_NAME|EMP_AGE|
  33. +------+----------+--------+-------+
  34. | jane| ddd|daughter| 15|
  35. | jina| bbb| mother| 45|
  36. |joseph| aaa| father| 50|
  37. |julian| ccc| son| 20|
  38. +------+----------+--------+-------+
  39.  
  40. List<Object> appendData = Arrays.asList("julia", "eee", "grand mother", new Integer(70));
  41. spark.createDataFrame(appendData, Object.class).write().mode(SaveMode.Append).jdbc(connectionProperties.getProperty("url"), "family", connectionProperties);
  42.  
  43. java.sql.BatchUpdateException: Field 'EMP_ID' doesn't have a default value
  44. at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  45. at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  46. at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  47. at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  48. at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
  49. at com.mysql.jdbc.Util.getInstance(Util.java:408)
  50. at com.mysql.jdbc.SQLError.createBatchUpdateException(SQLError.java:1163)
  51. at com.mysql.jdbc.PreparedStatement.executeBatchSerially(PreparedStatement.java:1778)
  52. at com.mysql.jdbc.PreparedStatement.executeBatchInternal(PreparedStatement.java:1262)
  53. at com.mysql.jdbc.StatementImpl.executeBatch(StatementImpl.java:970)
  54. at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.savePartition(JdbcUtils.scala:641)
  55. at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$saveTable$1.apply(JdbcUtils.scala:782)
  56. at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$saveTable$1.apply(JdbcUtils.scala:782)
  57. at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  58. at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  59. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2062)
  60. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2062)
  61. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  62. at org.apache.spark.scheduler.Task.run(Task.scala:108)
  63. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)
  64. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  65. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  66. at java.lang.Thread.run(Thread.java:748)
  67. Caused by: java.sql.SQLException: Field 'EMP_ID' doesn't have a default value
  68. at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
  69. at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3973)
  70. at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3909)
  71. at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2527)
  72. at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2680)
  73. at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2484)
  74. at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:1858)
  75. at com.mysql.jdbc.PreparedStatement.executeUpdateInternal(PreparedStatement.java:2079)
  76. at com.mysql.jdbc.PreparedStatement.executeBatchSerially(PreparedStatement.java:1756)
  77. ... 15 more
Add Comment
Please, Sign In to add comment