Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package com
- import au.com.bytecode.opencsv.CSVParser;
- import com.sleepycat.je.*;
- import java.io.*;
- import java.nio.charset.Charset;
- public class Main {
- static Environment myDbEnvironment = null;
- static Database myDatabase = null;
- static String csvFileName = "/Users/jianfeng/tmp/blades-all.csv";
- static String queryFile = "/Users/jianfeng/tmp/blade-stats-stream.csv";
- static String outputFile = "/Users/jianfeng/tmp/bdb.out";
- static CSVParser parser = new CSVParser(',');
- static int KEY_FIELDS = 1;
- static int STREAM_KEY_FIELDS = 19;
- public static void main(String[] args) throws IOException, ClassNotFoundException {
- open();
- insertRecord();
- queryRecord();
- close();
- }
- private static void queryRecord() throws IOException, ClassNotFoundException {
- PrintWriter writer = new PrintWriter(outputFile, "UTF-8");
- InputStream fis = new FileInputStream(queryFile);
- InputStreamReader isr = new InputStreamReader(fis, Charset.forName("UTF-8"));
- BufferedReader br = new BufferedReader(isr);
- String line;
- int count = 0;
- DatabaseEntry theData = new DatabaseEntry();
- int invteral = 4000;
- long start = System.currentTimeMillis();
- long mark = System.currentTimeMillis();
- while ((line = br.readLine()) != null) {
- if (line.startsWith("#")) {
- continue;
- }
- String[] fields = null;
- try {
- fields = parser.parseLine(line);
- } catch (IOException io) {
- continue;
- }
- count++;
- if (count == 1) {
- System.err.println("first : " + (System.currentTimeMillis() - start) + " ms");
- }
- if (count % invteral == 0) {
- long cur = System.currentTimeMillis();
- System.err.println("line " + invteral + " time:" + (cur - mark) + " speed:" + (int)(invteral / ((cur - mark) / 1000.0)) + " record/sec");
- }
- if (myDatabase.get(null, new DatabaseEntry(fields[STREAM_KEY_FIELDS].getBytes()), theData, LockMode.DEFAULT)
- == OperationStatus.SUCCESS) {
- String[] data = readValue(theData.getData());
- StringBuilder builder = new StringBuilder();
- builder.append(line);
- for (String field : data) {
- builder.append(field);
- }
- writer.println(builder.toString());
- } else {
- writer.println(line + ",null");
- }
- }
- }
- static byte[] writeValue(String[] value) throws IOException {
- final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- final ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteArrayOutputStream);
- objectOutputStream.writeObject(value);
- objectOutputStream.flush();
- objectOutputStream.close();
- return byteArrayOutputStream.toByteArray();
- }
- static String[] readValue(byte[] data) throws IOException, ClassNotFoundException {
- final ByteArrayInputStream byteArrayInputStream =
- new ByteArrayInputStream(data);
- final ObjectInputStream objectInputStream =
- new ObjectInputStream(byteArrayInputStream);
- final String[] stringArray2 = (String[]) objectInputStream.readObject();
- objectInputStream.close();
- return stringArray2;
- }
- private static void insertRecord() throws IOException {
- try {
- long begin = System.currentTimeMillis();
- InputStream fis = new FileInputStream(csvFileName);
- InputStreamReader isr = new InputStreamReader(fis, Charset.forName("UTF-8"));
- BufferedReader br = new BufferedReader(isr);
- String line;
- int count = 0;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("#")) {
- continue;
- }
- try {
- String[] fields = parser.parseLine(line);
- DatabaseEntry key = new DatabaseEntry(fields[KEY_FIELDS].getBytes());
- DatabaseEntry data = new DatabaseEntry(writeValue(fields));
- myDatabase.put(null, key, data);
- count++;
- } catch (IOException e) {
- continue;
- }
- }
- System.err.println("Finished the load file: count:" + count + " time:" + (System.currentTimeMillis() - begin) + " ms");
- } catch (Exception e) {
- throw e;
- }
- }
- private static void close() {
- try {
- if (myDatabase != null) {
- myDatabase.close();
- }
- if (myDbEnvironment != null) {
- myDbEnvironment.close();
- }
- } catch (DatabaseException dbe) {
- throw dbe;
- }
- }
- private static void open() {
- try {
- EnvironmentConfig envConfig = new EnvironmentConfig();
- envConfig.setAllowCreate(true);
- myDbEnvironment = new Environment(new File("/tmp/bdb"), envConfig);
- DatabaseConfig dbConfig = new DatabaseConfig();
- dbConfig.setAllowCreate(true);
- myDatabase = myDbEnvironment.openDatabase(null, "lookup", dbConfig);
- } catch (DatabaseException dbe) {
- throw dbe;
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement