Advertisement
Guest User

Untitled

a guest
Jul 30th, 2015
327
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 5.42 KB | None | 0 0
  1. package com
  2.  
  3. import au.com.bytecode.opencsv.CSVParser;
  4. import com.sleepycat.je.*;
  5.  
  6. import java.io.*;
  7. import java.nio.charset.Charset;
  8.  
  9. public class Main {
  10.  
  11. static Environment myDbEnvironment = null;
  12. static Database myDatabase = null;
  13. static String csvFileName = "/Users/jianfeng/tmp/blades-all.csv";
  14. static String queryFile = "/Users/jianfeng/tmp/blade-stats-stream.csv";
  15. static String outputFile = "/Users/jianfeng/tmp/bdb.out";
  16. static CSVParser parser = new CSVParser(',');
  17. static int KEY_FIELDS = 1;
  18. static int STREAM_KEY_FIELDS = 19;
  19.  
  20. public static void main(String[] args) throws IOException, ClassNotFoundException {
  21.  
  22. open();
  23.  
  24. insertRecord();
  25.  
  26. queryRecord();
  27.  
  28. close();
  29.  
  30. }
  31.  
  32. private static void queryRecord() throws IOException, ClassNotFoundException {
  33. PrintWriter writer = new PrintWriter(outputFile, "UTF-8");
  34. InputStream fis = new FileInputStream(queryFile);
  35. InputStreamReader isr = new InputStreamReader(fis, Charset.forName("UTF-8"));
  36. BufferedReader br = new BufferedReader(isr);
  37.  
  38. String line;
  39. int count = 0;
  40. DatabaseEntry theData = new DatabaseEntry();
  41. int invteral = 4000;
  42. long start = System.currentTimeMillis();
  43. long mark = System.currentTimeMillis();
  44. while ((line = br.readLine()) != null) {
  45. if (line.startsWith("#")) {
  46. continue;
  47. }
  48. String[] fields = null;
  49. try {
  50. fields = parser.parseLine(line);
  51. } catch (IOException io) {
  52. continue;
  53. }
  54. count++;
  55. if (count == 1) {
  56.  
  57. System.err.println("first : " + (System.currentTimeMillis() - start) + " ms");
  58. }
  59. if (count % invteral == 0) {
  60. long cur = System.currentTimeMillis();
  61. System.err.println("line " + invteral + " time:" + (cur - mark) + " speed:" + (int)(invteral / ((cur - mark) / 1000.0)) + " record/sec");
  62. }
  63. if (myDatabase.get(null, new DatabaseEntry(fields[STREAM_KEY_FIELDS].getBytes()), theData, LockMode.DEFAULT)
  64. == OperationStatus.SUCCESS) {
  65. String[] data = readValue(theData.getData());
  66. StringBuilder builder = new StringBuilder();
  67. builder.append(line);
  68. for (String field : data) {
  69. builder.append(field);
  70. }
  71. writer.println(builder.toString());
  72. } else {
  73. writer.println(line + ",null");
  74. }
  75. }
  76.  
  77. }
  78.  
  79. static byte[] writeValue(String[] value) throws IOException {
  80. final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  81. final ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteArrayOutputStream);
  82. objectOutputStream.writeObject(value);
  83. objectOutputStream.flush();
  84. objectOutputStream.close();
  85. return byteArrayOutputStream.toByteArray();
  86. }
  87.  
  88. static String[] readValue(byte[] data) throws IOException, ClassNotFoundException {
  89. final ByteArrayInputStream byteArrayInputStream =
  90. new ByteArrayInputStream(data);
  91. final ObjectInputStream objectInputStream =
  92. new ObjectInputStream(byteArrayInputStream);
  93.  
  94. final String[] stringArray2 = (String[]) objectInputStream.readObject();
  95.  
  96. objectInputStream.close();
  97. return stringArray2;
  98. }
  99.  
  100. private static void insertRecord() throws IOException {
  101. try {
  102. long begin = System.currentTimeMillis();
  103. InputStream fis = new FileInputStream(csvFileName);
  104. InputStreamReader isr = new InputStreamReader(fis, Charset.forName("UTF-8"));
  105. BufferedReader br = new BufferedReader(isr);
  106.  
  107. String line;
  108. int count = 0;
  109. while ((line = br.readLine()) != null) {
  110. if (line.startsWith("#")) {
  111. continue;
  112. }
  113. try {
  114. String[] fields = parser.parseLine(line);
  115. DatabaseEntry key = new DatabaseEntry(fields[KEY_FIELDS].getBytes());
  116. DatabaseEntry data = new DatabaseEntry(writeValue(fields));
  117. myDatabase.put(null, key, data);
  118. count++;
  119. } catch (IOException e) {
  120. continue;
  121. }
  122. }
  123. System.err.println("Finished the load file: count:" + count + " time:" + (System.currentTimeMillis() - begin) + " ms");
  124.  
  125. } catch (Exception e) {
  126. throw e;
  127. }
  128. }
  129.  
  130. private static void close() {
  131. try {
  132. if (myDatabase != null) {
  133. myDatabase.close();
  134. }
  135.  
  136. if (myDbEnvironment != null) {
  137. myDbEnvironment.close();
  138. }
  139. } catch (DatabaseException dbe) {
  140. throw dbe;
  141. }
  142. }
  143.  
  144. private static void open() {
  145. try {
  146. EnvironmentConfig envConfig = new EnvironmentConfig();
  147. envConfig.setAllowCreate(true);
  148. myDbEnvironment = new Environment(new File("/tmp/bdb"), envConfig);
  149.  
  150.  
  151. DatabaseConfig dbConfig = new DatabaseConfig();
  152. dbConfig.setAllowCreate(true);
  153. myDatabase = myDbEnvironment.openDatabase(null, "lookup", dbConfig);
  154. } catch (DatabaseException dbe) {
  155. throw dbe;
  156. }
  157.  
  158. }
  159. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement