Guest User

sequence files java bare bones reader

a guest
Sep 25th, 2012
140
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.60 KB | None | 0 0
  1.  
  2.  
  3.  
  4. import java.io.BufferedInputStream;
  5. import java.io.File;
  6. import java.io.FileInputStream;
  7. import java.io.FileNotFoundException;
  8. import java.io.IOException;
  9.  
  10. import net.peerindex.pisae.thrift.SocialBit;
  11.  
  12. import org.apache.hadoop.io.SequenceFile;
  13. import org.apache.thrift.TException;
  14. import org.apache.thrift.protocol.TCompactProtocol;
  15. import org.apache.thrift.transport.TIOStreamTransport;
  16.  
  17. public class ThriftFileReader {
  18.  
  19. public static final String path=
  20. "/Users/Jpeerindex/Development/workspace/sandbox"+
  21. "my_hdfs_job_output/";
  22.  
  23. public static boolean hasNext(BufferedInputStream bufferedIn) throws Exception{
  24. bufferedIn.mark(1);
  25. int val = bufferedIn.read();
  26. bufferedIn.reset();
  27. return val != -1;
  28. }
  29.  
  30. public static void main(String[] args) throws Exception{
  31. int i = 127;
  32. for(File f : new File(path).listFiles()){
  33. BufferedInputStream in = new BufferedInputStream(new FileInputStream(f), 2048);
  34. TCompactProtocol p = new TCompactProtocol(new TIOStreamTransport(in));
  35.  
  36. //System.out.println("File " + f.getAbsolutePath() + "\n\t"+p.getTransport().peek());
  37. if(f.getName().endsWith(".crc")){
  38. ;
  39. }
  40. else{
  41. try {
  42. //System.out.println("Try skip "+i);
  43. in.skip(i);
  44. while(hasNext(in)){
  45. SocialBit b =new SocialBit();
  46. b.read(p);
  47. if(b.getVerb()!=null)
  48. System.out.println("bit " + b);
  49. }
  50. p.getTransport().close();
  51. System.out.println("Done");
  52. } catch (Throwable e) {
  53. System.err.println("fail " + i + " " + e.getMessage());
  54. }
  55. }
  56. }
  57. }
  58. }
Advertisement
Add Comment
Please, Sign In to add comment