Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- Failed with exception java.io.IOException:org.apache.avro.AvroRuntimeException: java.io.IOException: Block size invalid or too large for this implementation: -40
- TwitterAgent.sources = Twitter
- TwitterAgent.channels = MemChannel
- TwitterAgent.sinks = HDFS
- TwitterAgent.sources.Twitter.type = org.apache.flume.source.twitter.TwitterSource
- TwitterAgent.sources.Twitter.consumerKey = xxx
- TwitterAgent.sources.Twitter.consumerSecret = xxx
- TwitterAgent.sources.Twitter.accessToken = xxx
- TwitterAgent.sources.Twitter.accessTokenSecret = xxx
- TwitterAgent.sinks.HDFS.type = hdfs
- TwitterAgent.sinks.HDFS.hdfs.path = hdfs://sandbox.hortonworks.com:8020/user/flume/twitter_data/
- TwitterAgent.sinks.HDFS.hdfs.fileType = DataStream
- TwitterAgent.sinks.HDFS.hdfs.writeFormat = Text
- TwitterAgent.sinks.HDFS.hdfs.batchSize = 1000
- TwitterAgent.sinks.HDFS.hdfs.rollSize = 0
- TwitterAgent.sinks.HDFS.hdfs.rollCount = 10000
- TwitterAgent.sinks.HDFS.serializer = Text
- TwitterAgent.channels.MemChannel.type = memory
- TwitterAgent.channels.MemChannel.capacity = 10000
- TwitterAgent.channels.MemChannel.transactionCapacity = 1000
- TwitterAgent.sources.Twitter.channels = MemChannel
- TwitterAgent.sinks.HDFS.channel = MemChannel
- java -jar avro-tools-1.7.7.jar getschema FlumeData.1503479843633 > twitter.avsc
- CREATE TABLE tweetsavro
- ROW FORMAT SERDE
- 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
- STORED AS INPUTFORMAT
- 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
- OUTPUTFORMAT
- 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
- TBLPROPERTIES ('avro.schema.url'='hdfs://sandbox.hortonworks.com:8020/user/flume/twitter.avsc') ;
- LOAD DATA INPATH 'hdfs://sandbox.hortonworks.com:8020/user/flume/twitter_data/FlumeData.*' OVERWRITE INTO TABLE tweetsavro;
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement