SHARE
TWEET

Untitled

a guest Aug 14th, 2019 64 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. from pyspark.sql import SparkSession
  2.  
  3. spark = SparkSession
  4.     .builder
  5.     .appName("Python Spark SQL basic example")
  6.     .config("spark.some.config.option", "some-value")
  7.     .getOrCreate()
  8.  
  9. df_image = spark.read.format("image").load("car-images")
  10. df_image.printSchema()
  11.      
  12. root
  13.  |-- image: struct (nullable = true)
  14.  |    |-- origin: string (nullable = true)
  15.  |    |-- height: integer (nullable = true)
  16.  |    |-- width: integer (nullable = true)
  17.  |    |-- nChannels: integer (nullable = true)
  18.  |    |-- mode: integer (nullable = true)
  19.  |    |-- data: binary (nullable = true)
  20.      
  21. from mmlspark import ImageTransformer
  22. img_transformer = ImageTransformer().setOutputCol("flipped").flip()
  23. flipped_images = img_transformer.transform(df_image).select("flipped")
  24. flipped_images.printSchema()
  25.      
  26. root
  27.  |-- flipped: struct (nullable = true)
  28.  |    |-- origin: string (nullable = true)
  29.  |    |-- height: integer (nullable = false)
  30.  |    |-- width: integer (nullable = false)
  31.  |    |-- nChannels: integer (nullable = false)
  32.  |    |-- mode: integer (nullable = false)
  33.  |    |-- data: binary (nullable = false)
  34.      
  35. flipped_images.write.format("image").save("car-images-flipped")
  36.      
  37. ---------------------------------------------------------------------------
  38. Py4JJavaError                             Traceback (most recent call last)
  39. <ipython-input-4-7a8ced5adfe6> in <module>()
  40. ----> 1 flipped_images.write.format("image").save("car-images-flipped3")
  41.  
  42. ~/lib/spark/python/pyspark/sql/readwriter.py in save(self, path, format, mode, partitionBy, **options)
  43.     734             self._jwrite.save()
  44.     735         else:
  45. --> 736             self._jwrite.save(path)
  46.     737
  47.     738     @since(1.4)
  48.  
  49. ~/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py in __call__(self, *args)
  50.    1255         answer = self.gateway_client.send_command(command)
  51.    1256         return_value = get_return_value(
  52. -> 1257             answer, self.gateway_client, self.target_id, self.name)
  53.    1258
  54.    1259         for temp_arg in temp_args:
  55.  
  56. ~/lib/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
  57.      61     def deco(*a, **kw):
  58.      62         try:
  59. ---> 63             return f(*a, **kw)
  60.      64         except py4j.protocol.Py4JJavaError as e:
  61.      65             s = e.java_exception.toString()
  62.  
  63. ~/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
  64.     326                 raise Py4JJavaError(
  65.     327                     "An error occurred while calling {0}{1}{2}.n".
  66. --> 328                     format(target_id, ".", name), value)
  67.     329             else:
  68.     330                 raise Py4JError(
  69.  
  70. Py4JJavaError: An error occurred while calling o104.save.
  71. : java.lang.UnsupportedOperationException: Write is not supported for image data source
  72.     at org.apache.spark.ml.source.image.ImageFileFormat.prepareWrite(ImageFileFormat.scala:48)
  73.     at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:103)
  74.     at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:159)
  75.     at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:104)
  76.     at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:102)
  77.     at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:122)
  78.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
  79.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
  80.     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
  81.     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  82.     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
  83.     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
  84.     at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
  85.     at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
  86.     at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:668)
  87.     at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:668)
  88.     at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78)
  89.     at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125)
  90.     at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73)
  91.     at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:668)
  92.     at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:276)
  93.     at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:270)
  94.     at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:228)
  95.     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  96.     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  97.     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  98.     at java.lang.reflect.Method.invoke(Method.java:498)
  99.     at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
  100.     at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
  101.     at py4j.Gateway.invoke(Gateway.java:282)
  102.     at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
  103.     at py4j.commands.CallCommand.execute(CallCommand.java:79)
  104.     at py4j.GatewayConnection.run(GatewayConnection.java:238)
  105.     at java.lang.Thread.run(Thread.java:748)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!
 
Top