Advertisement
Guest User

Untitled

a guest
Jul 19th, 2017
94
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.20 KB | None | 0 0
  1. /*
  2. * Copyright Stuart Rex King 2010
  3. *
  4. */
  5.  
  6. package com.wenzani.mongo;
  7.  
  8. import com.mongodb.*;
  9. import com.mongodb.util.JSON;
  10. import org.apache.commons.lang.exception.ExceptionUtils;
  11. import org.codehaus.jackson.map.ObjectMapper;
  12.  
  13. import java.io.IOException;
  14. import java.net.UnknownHostException;
  15. import java.sql.*;
  16. import java.util.ArrayList;
  17. import java.util.List;
  18.  
  19. public class PlacesMain {
  20.  
  21. private static final String PLACE_SELECT = "SELECT * FROM places LIMIT %d, %d;";
  22. private static final String TAGS_SELECT = "SELECT tag FROM tags where place_id = ?;";
  23. private static final String CATEGORIES_SELECT = "SELECT category FROM categories where place_id = ?;";
  24. private static final String WENZANI = "wenzani";
  25. private static final String PLACES = "places";
  26. private final ObjectMapper objectMapper = new ObjectMapper();
  27.  
  28. public static void main(String[] args) {
  29. PlacesMain placesMain = new PlacesMain();
  30. placesMain.createIndexes();
  31. placesMain.run();
  32. }
  33.  
  34. private void createIndexes() {
  35. Mongo mongo = null;
  36. try {
  37. mongo = new Mongo();
  38. DB db = mongo.getDB(WENZANI);
  39. DBCollection collection = db.getCollection(PLACES);
  40. collection.ensureIndex(new BasicDBObject("lat_lng", "2d"));
  41. } catch (UnknownHostException e) {
  42. System.out.println(ExceptionUtils.getFullStackTrace(e));
  43. }
  44.  
  45. mongo.close();
  46. }
  47.  
  48. public void run() {
  49. insertPlace();
  50. }
  51.  
  52. public void insertPlace() {
  53. Connection connection = null;
  54. try {
  55. Class.forName("com.mysql.jdbc.Driver");
  56. connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/places?user=places&password=places");
  57. for (int i = 0; i < 4769976; i = i + 500) {
  58. int start = i;
  59. int end = 499;
  60. String sql = String.format(PLACE_SELECT, start, end);
  61. Statement statement = connection.createStatement();
  62. ResultSet resultSet = statement.executeQuery(sql);
  63. List<Place> places = new ArrayList<Place>();
  64. while (resultSet.next()) {
  65. int id = resultSet.getInt(1);
  66. Place place = new Place();
  67. place.setSource_id(resultSet.getInt(2));
  68. place.setTitle(resultSet.getString(3));
  69.  
  70. List<Double> latLng = new ArrayList<Double>();
  71. latLng.add(Double.valueOf(resultSet.getString(4)));
  72. latLng.add(Double.valueOf(resultSet.getString(5)));
  73. place.setLat_long(latLng);
  74.  
  75. place.setType(resultSet.getString(6));
  76. place.setStreetAddress(resultSet.getString(7));
  77. place.setLocality(resultSet.getString(8));
  78. place.setRegion(resultSet.getString(9));
  79. place.setPostalCode(resultSet.getString(10));
  80. place.setPhoneNumber(resultSet.getString(11));
  81. place.setSource(resultSet.getString(12));
  82.  
  83. List<String> tags = getTags(id, connection);
  84. place.setTags(tags);
  85.  
  86. List<String> categories = getCategories(id, connection);
  87. place.setCategories(categories);
  88.  
  89. System.out.println(place);
  90. places.add(place);
  91. }
  92. resultSet.close();
  93. statement.close();
  94. addPlacesToMongo(places);
  95. }
  96. } catch (Exception e) {
  97. System.out.println(e);
  98. } finally {
  99. if (null != connection) {
  100. try {
  101. connection.close();
  102. } catch (SQLException e) {
  103. System.out.println(e);
  104. }
  105. }
  106. }
  107. }
  108.  
  109. private void addPlacesToMongo(List<Place> places) {
  110. try {
  111. Mongo mongo = new Mongo();
  112. DB db = mongo.getDB(WENZANI);
  113. DBCollection collection = db.getCollection(PLACES);
  114.  
  115. List<DBObject> dbObjects = createObject(places);
  116. collection.insert(dbObjects);
  117.  
  118. mongo.close();
  119. } catch (UnknownHostException e) {
  120. System.out.println(ExceptionUtils.getFullStackTrace(e));
  121. }
  122. }
  123.  
  124. private List<DBObject> createObject(List<Place> places) {
  125. List<DBObject> dbObjects = new ArrayList<DBObject>();
  126.  
  127. for (Place place : places) {
  128. try {
  129. String json = objectMapper.writeValueAsString(place);
  130. DBObject dbObject = (DBObject) JSON.parse(json);
  131. dbObjects.add(dbObject);
  132. } catch (IOException e) {
  133. System.out.println(ExceptionUtils.getFullStackTrace(e));
  134. }
  135. }
  136.  
  137. return dbObjects;
  138. }
  139.  
  140. private List<String> getCategories(int id, Connection connection) {
  141. List<String> categories = new ArrayList<String>();
  142. try {
  143. PreparedStatement preparedStatement = connection.prepareStatement(CATEGORIES_SELECT);
  144. preparedStatement.setInt(1, id);
  145. ResultSet resultSet = preparedStatement.executeQuery();
  146.  
  147. while (resultSet.next()) {
  148. categories.add(resultSet.getString(1));
  149. }
  150. resultSet.close();
  151. preparedStatement.close();
  152. } catch (SQLException e) {
  153. System.out.println(e);
  154. }
  155. return categories;
  156. }
  157.  
  158. private List<String> getTags(int id, Connection connection) {
  159. List<String> tags = new ArrayList<String>();
  160. try {
  161. PreparedStatement preparedStatement = connection.prepareStatement(TAGS_SELECT);
  162. preparedStatement.setInt(1, id);
  163. ResultSet resultSet = preparedStatement.executeQuery();
  164.  
  165. while (resultSet.next()) {
  166. tags.add(resultSet.getString(1));
  167. }
  168. resultSet.close();
  169.  
  170. preparedStatement.close();
  171. } catch (SQLException e) {
  172. System.out.println(e);
  173. }
  174. return tags;
  175. }
  176.  
  177. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement