Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import java.io.BufferedInputStream;
- import java.io.File;
- import java.io.FileInputStream;
- import java.io.FileNotFoundException;
- import java.io.IOException;
- import net.peerindex.pisae.thrift.SocialBit;
- import org.apache.hadoop.io.SequenceFile;
- import org.apache.thrift.TException;
- import org.apache.thrift.protocol.TCompactProtocol;
- import org.apache.thrift.transport.TIOStreamTransport;
- public class ThriftFileReader {
- public static final String path=
- "/Users/Jpeerindex/Development/workspace/sandbox"+
- "my_hdfs_job_output/";
- public static boolean hasNext(BufferedInputStream bufferedIn) throws Exception{
- bufferedIn.mark(1);
- int val = bufferedIn.read();
- bufferedIn.reset();
- return val != -1;
- }
- public static void main(String[] args) throws Exception{
- int i = 127;
- for(File f : new File(path).listFiles()){
- BufferedInputStream in = new BufferedInputStream(new FileInputStream(f), 2048);
- TCompactProtocol p = new TCompactProtocol(new TIOStreamTransport(in));
- //System.out.println("File " + f.getAbsolutePath() + "\n\t"+p.getTransport().peek());
- if(f.getName().endsWith(".crc")){
- ;
- }
- else{
- try {
- //System.out.println("Try skip "+i);
- in.skip(i);
- while(hasNext(in)){
- SocialBit b =new SocialBit();
- b.read(p);
- if(b.getVerb()!=null)
- System.out.println("bit " + b);
- }
- p.getTransport().close();
- System.out.println("Done");
- } catch (Throwable e) {
- System.err.println("fail " + i + " " + e.getMessage());
- }
- }
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment