Advertisement
Guest User

Untitled

a guest
Aug 23rd, 2017
100
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.75 KB | None | 0 0
  1. Failed with exception java.io.IOException:org.apache.avro.AvroRuntimeException: java.io.IOException: Block size invalid or too large for this implementation: -40
  2.  
  3. TwitterAgent.sources = Twitter
  4. TwitterAgent.channels = MemChannel
  5. TwitterAgent.sinks = HDFS
  6.  
  7. TwitterAgent.sources.Twitter.type = org.apache.flume.source.twitter.TwitterSource
  8. TwitterAgent.sources.Twitter.consumerKey = xxx
  9. TwitterAgent.sources.Twitter.consumerSecret = xxx
  10. TwitterAgent.sources.Twitter.accessToken = xxx
  11. TwitterAgent.sources.Twitter.accessTokenSecret = xxx
  12.  
  13. TwitterAgent.sinks.HDFS.type = hdfs
  14. TwitterAgent.sinks.HDFS.hdfs.path = hdfs://sandbox.hortonworks.com:8020/user/flume/twitter_data/
  15. TwitterAgent.sinks.HDFS.hdfs.fileType = DataStream
  16. TwitterAgent.sinks.HDFS.hdfs.writeFormat = Text
  17. TwitterAgent.sinks.HDFS.hdfs.batchSize = 1000
  18. TwitterAgent.sinks.HDFS.hdfs.rollSize = 0
  19. TwitterAgent.sinks.HDFS.hdfs.rollCount = 10000
  20. TwitterAgent.sinks.HDFS.serializer = Text
  21.  
  22. TwitterAgent.channels.MemChannel.type = memory
  23. TwitterAgent.channels.MemChannel.capacity = 10000
  24. TwitterAgent.channels.MemChannel.transactionCapacity = 1000
  25.  
  26. TwitterAgent.sources.Twitter.channels = MemChannel
  27. TwitterAgent.sinks.HDFS.channel = MemChannel
  28.  
  29. java -jar avro-tools-1.7.7.jar getschema FlumeData.1503479843633 > twitter.avsc
  30.  
  31. CREATE TABLE tweetsavro
  32. ROW FORMAT SERDE
  33. 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
  34. STORED AS INPUTFORMAT
  35. 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
  36. OUTPUTFORMAT
  37. 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
  38. TBLPROPERTIES ('avro.schema.url'='hdfs://sandbox.hortonworks.com:8020/user/flume/twitter.avsc') ;
  39. LOAD DATA INPATH 'hdfs://sandbox.hortonworks.com:8020/user/flume/twitter_data/FlumeData.*' OVERWRITE INTO TABLE tweetsavro;
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement