Advertisement
Guest User

Untitled

a guest
Mar 1st, 2017
79
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.59 KB | None | 0 0
  1. // Example - MINIMAL IMPLEMENTATION - of simple ORC file writing.
  2. // Non-distributed (not require YARN) and can run in off-cluster process.
  3. // Requires that the host has appropriate HDFS and ORC client libs installed, plus connectivity to the cluster.
  4.  
  5. package org.dkaiser.orctest;
  6.  
  7. import java.io.IOException;
  8.  
  9. import org.apache.hadoop.conf.Configuration;
  10. import org.apache.hadoop.fs.FileSystem;
  11. import org.apache.hadoop.fs.Path;
  12. import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
  13. import org.apache.hadoop.hive.ql.io.orc.OrcFile;
  14. import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
  15. import org.apache.hadoop.hive.ql.io.orc.Writer;
  16. import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
  17. import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
  18.  
  19. public class Driver {
  20.  
  21. public static class Row {
  22. Integer int1;
  23. Long long1;
  24. String firstName;
  25. String lastName;
  26. String city;
  27. String state;
  28. String zip9;
  29.  
  30. public Row(int val, long l, String fn, String ln, String c, String s, String z) {
  31. this.int1 = val;
  32. this.long1 = l;
  33. this.firstName = fn;
  34. this.lastName = ln;
  35. this.city = c;
  36. this.state = s;
  37. this.zip9 = z;
  38. }
  39. }
  40.  
  41. public static void main(String[] args) {
  42.  
  43. Configuration conf = new Configuration(false);
  44. conf.set("fs.defaultFS", "hdfs://sandbox.hortonworks.com:8020");
  45.  
  46. ObjectInspector inspector;
  47.  
  48. synchronized (Driver.class) {
  49. inspector = ObjectInspectorFactory.getReflectionObjectInspector(
  50. Row.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
  51. }
  52.  
  53. try {
  54. FileSystem fs = FileSystem.get(conf);
  55. Path path = new Path("/tmp/file.orc");
  56. if (fs.exists(path)) {
  57. fs.delete(path, true);
  58. }
  59.  
  60. Writer writer = OrcFile.createWriter(path,
  61. OrcFile.writerOptions(conf)
  62. .fileSystem(fs)
  63. .inspector(inspector)
  64. .blockPadding(false)
  65. .compress(CompressionKind.NONE)
  66. .encodingStrategy(EncodingStrategy.SPEED)
  67. // .stripeSize(100000)
  68. // .bufferSize(10000)
  69. );
  70.  
  71. writer.addRow(new Row(111, 1111L, "John", "Doe", "Buffalo", "NY", "12345-6789"));
  72. writer.addRow(new Row(112, 2222L, "Henry", "Ford", "Detroit", "MI", "12345-6789"));
  73. writer.addRow(new Row(113, 3333L, "Walter", "Cronkite", "New York", "NY", "12345-6789"));
  74. writer.addRow(new Row(114, 4444L, "Al", "Gore", "Nashville", "TN", "12345-6789"));
  75. writer.close();
  76. } catch (IOException ex) {
  77. System.err.println("Error: Failure writing values to OrcFile.");
  78. ex.printStackTrace();
  79. }
  80. }
  81. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement