Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- dataFile = ("s3n://<AWS Access Key ID>:"
- "<AWS Secret Access Key>"
- "@<mybucket>/data.txt")
- Traceback (most recent call last):
- File "/home/adas/spark/SimpleApp.py", line 23, in <module>
- numAs = logData.filter(lambda s: 'a' in s).count()
- File "/home/adas/spark/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1041, in count
- File "/home/adas/spark/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1032, in sum
- File "/home/adas/spark/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 906, in fold
- File "/home/adas/spark/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 809, in collect
- File "/home/adas/spark/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py", line 1133, in __call__
- File "/home/adas/spark/spark/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py", line 319, in get_return_value
- py4j.protocol.Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.
- : java.lang.IllegalArgumentException: AWS Secret Access Key must be specified as the password of a s3n URL, or by setting the fs.s3n.awsSecretAccessKey property.
- at org.apache.hadoop.fs.s3.S3Credentials.initialize(S3Credentials.java:86)
- at org.apache.hadoop.fs.s3native.Jets3tNativeFileSystemStore.initialize(Jets3tNativeFileSystemStore.java:80)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
- at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
- at org.apache.hadoop.fs.s3native.$Proxy16.initialize(Unknown Source)
- at org.apache.hadoop.fs.s3native.NativeS3FileSystem.initialize(NativeS3FileSystem.java:334)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2669)
- at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94)
- at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703)
- at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373)
- at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
- at org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:258)
- at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:229)
- at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:315)
- at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:202)
- at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
- at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
- at scala.Option.getOrElse(Option.scala:121)
- at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
- at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
- at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
- at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
- at scala.Option.getOrElse(Option.scala:121)
- at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
- at org.apache.spark.api.python.PythonRDD.getPartitions(PythonRDD.scala:53)
- at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
- at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
- at scala.Option.getOrElse(Option.scala:121)
- at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
- at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)
- at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:935)
- at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
- at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
- at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
- at org.apache.spark.rdd.RDD.collect(RDD.scala:934)
- at org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:453)
- at org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
- at py4j.Gateway.invoke(Gateway.java:280)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:214)
- at java.lang.Thread.run(Thread.java:745)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement