Guest User

Untitled

a guest
Sep 15th, 2018
87
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 16.48 KB | None | 0 0
  1. geotools SEVERE: The following locker still has a lock read on file
  2. [root@website-qc filespool]# /usr/bin/java -jar /opt/mcmap/library/Application/geotools/mcgeotools.jar -t publisher -i 1/194/Namibia_SCLB12.shp -rid 12 -s
  3. app get cmd option cast to int: 12
  4. app passing region id to runconvert: 12
  5. runconvert rid passed: 12
  6. Jul 9, 2012 4:23:48 PM org.geotools.data.db2.DB2DataStoreFactory isAvailable
  7. INFO: DB2 driver found: true
  8. Database connection acquired
  9. Coordinates length: 5390
  10. Coordinates length: 5358
  11. Coordinates length: 9932
  12. Jul 9, 2012 4:28:14 PM org.geotools.data.shapefile.ShpFiles logCurrentLockers
  13. SEVERE: The following locker still has a lock� read on file:/opt/mcmap/public/filespool/1/194/Namibia_SCLB12.prj by org.geotools.data.shapefile.prj.PrjFileReader
  14. it was created with the following stack trace
  15. org.geotools.data.shapefile.ShpFilesLocker$Trace: Locking file:/opt/mcmap/public/filespool/1/194/Namibia_SCLB12.prj for read by org.geotools.data.shapefile.prj.PrjFileReader in thread main
  16. at org.geotools.data.shapefile.ShpFilesLocker.setTraceException(ShpFilesLocker.java:72)
  17. at org.geotools.data.shapefile.ShpFilesLocker.<init>(ShpFilesLocker.java:36)
  18. at org.geotools.data.shapefile.ShpFiles.acquireRead(ShpFiles.java:365)
  19. at org.geotools.data.shapefile.ShpFiles.getReadChannel(ShpFiles.java:813)
  20. at org.geotools.data.shapefile.prj.PrjFileReader.<init>(PrjFileReader.java:66)
  21. at com.domain.mcgeotools.convert.Converter.getCoordinateSystem(Converter.java:521)
  22. at com.domain.mcgeotools.convert.Converter.runConvert(Converter.java:106)
  23. at com.domain.mcgeotools.App.main(App.java:158)
  24. Coordinates length: 5741
  25. Coordinates length: 5374
  26. Coordinates length: 4193
  27. Coordinates length: 14161
  28. Coordinates length: 5375
  29. Coordinates length: 4212
  30.  
  31. package com.domain.mcgeotools.convert;
  32. import com.vividsolutions.jts.geom.Coordinate;
  33. import com.vividsolutions.jts.geom.Geometry;
  34. import java.io.BufferedWriter;
  35. import java.io.IOException;
  36. import java.io.File;
  37. import java.net.ConnectException;
  38. import java.net.MalformedURLException;
  39. import java.util.Properties;
  40. import java.util.Collection;
  41. import java.util.HashMap;
  42. import java.util.Iterator;
  43. import java.util.Map;
  44. import java.util.logging.Level;
  45. import java.util.logging.Logger;
  46. import java.sql.*;
  47. import org.geotools.data.DataStore;
  48. import org.geotools.data.DataStoreFinder;
  49. import org.geotools.data.FeatureSource;
  50. import org.geotools.data.shapefile.*;
  51. import org.geotools.data.shapefile.prj.PrjFileReader;
  52. import org.geotools.feature.FeatureCollection;
  53. import org.geotools.feature.FeatureIterator;
  54. import org.geotools.feature.simple.SimpleFeatureImpl;
  55. import org.opengis.feature.Property;
  56. import org.opengis.referencing.crs.CoordinateReferenceSystem;
  57.  
  58. public class Converter {
  59.  
  60. protected String filePath = null;
  61. private Properties configFile = null;
  62. private String[] acceptedCoordinateSytems = {
  63. "Lat Long for MAPINFO type 0 Datum",
  64. "GCS_WGS_1984"
  65. };
  66. private HashMap sqlStatements = new HashMap();
  67. private int sqlCounter = 0;
  68.  
  69.  
  70. public Converter() throws IOException {
  71.  
  72. /**
  73. * We load the configuration details from the properties file and use the config. The config property
  74. * item stores things such as database connection details etc.
  75. */
  76. this.configFile = new Properties();
  77. this.configFile.load(this.getClass().getClassLoader().getResourceAsStream("mcgeotools-config.properties"));
  78. }
  79.  
  80. protected void setFilePath(String filePath) {
  81. this.filePath = filePath;
  82. }
  83.  
  84. protected String getFilePath() {
  85. return this.filePath;
  86. }
  87.  
  88. /**
  89. * Converts the passed shape files and place the data into the RDBMS
  90. *
  91. * @param filePath
  92. * @param verbose
  93. * @param shrink
  94. * @return
  95. * @throws Exception
  96. */
  97. public int runConvert(String filePath, int regionId, boolean verbose, boolean shrink, String type) throws Exception {
  98. System.out.println("runconvert rid passed: " + regionId);
  99. ShapeFile shapeFile = new ShapeFile();
  100. if (!shapeFile.isShapeFile(filePath)) {
  101. System.out.println(filePath);
  102. throw new Exception("Not a shape file");
  103. }
  104. //we get the coordinate type from the shape file
  105. String coordinateSystem = null;
  106. try {
  107. coordinateSystem = this.getCoordinateSystem(filePath);
  108. if (verbose) {
  109. //Print to console.
  110. System.out.println("Coordinate System: " + coordinateSystem + " for " + filePath);
  111. }
  112.  
  113. } catch (Exception cse) {
  114. Logger.getLogger(Converter.class.getName()).log(Level.SEVERE, null, cse);
  115. }
  116.  
  117. boolean acceptedSystem = false;
  118.  
  119. for (int i = 0; i < this.acceptedCoordinateSytems.length; i++) {
  120. if (this.acceptedCoordinateSytems[i].equalsIgnoreCase(coordinateSystem)) {
  121. acceptedSystem = true;
  122. }
  123.  
  124. }
  125.  
  126. if (!acceptedSystem) {
  127. throw new Exception("Not an excepted Coordinate System.");
  128. }
  129.  
  130. //we now load the shapefile into the application
  131. File file = new File(filePath);
  132.  
  133. if (file == null) {
  134. throw new Exception("Failed to open " + filePath);
  135. }
  136.  
  137. Map connect = new HashMap();
  138. connect.put("url", file.toURI().toURL());
  139.  
  140. DataStore dataStore = DataStoreFinder.getDataStore(connect);
  141. String[] typeNames = dataStore.getTypeNames();
  142. String typeName = typeNames[0];
  143.  
  144. FeatureSource featureSource = dataStore.getFeatureSource(typeName);
  145. FeatureCollection collection = featureSource.getFeatures();
  146. FeatureIterator iterator = collection.features();
  147.  
  148. //database connection
  149. Connection dbConn = null;
  150.  
  151. int surveyDbId = 0;
  152.  
  153. String localhostname = java.net.InetAddress.getLocalHost().getHostName();
  154.  
  155. try {
  156.  
  157. String dbUser = "", dbPwd = "", dbUrl = "";
  158.  
  159. dbUser = this.configFile.getProperty("DB_USER");
  160. dbPwd = this.configFile.getProperty("DB_PASSWORD");
  161. dbUrl = "jdbc:" + this.configFile.getProperty("DB_CONNSTR");
  162. dbUrl = "jdbc:mysql://"+ localhostname +"/mcdatabase";
  163.  
  164. // If publishing overite connection strings
  165. if(type.equals("publish")){
  166. dbUser = this.configFile.getProperty("DB_PB_USER");
  167. dbPwd = this.configFile.getProperty("DB_PB_PASSWORD");
  168. dbUrl = "jdbc:" + this.configFile.getProperty("DB_PB_CONNSTR");
  169. }
  170.  
  171. Class.forName("com.mysql.jdbc.Driver").newInstance();
  172. dbConn = DriverManager.getConnection(dbUrl, dbUser, dbPwd);
  173.  
  174. System.out.println("Database connection acquired");
  175.  
  176. //switch off auto commits so that we run this as one transaction.
  177. dbConn.setAutoCommit(false);
  178.  
  179. this.sqlStatements.put(this.sqlCounter, "START TRANSACTION");
  180. this.sqlCounter++;
  181.  
  182. PreparedStatement surveyStmt = null;
  183. ResultSet rs = null;
  184.  
  185. if(type.equals("convert")){
  186.  
  187. //insert data in to the survey table and get the last row ID.
  188. surveyStmt = dbConn.prepareStatement("INSERT INTO surveys (name, guid, date_added, region_id, shrink) VALUES( ?, md5(now()), now(), ?, ? )");
  189. surveyStmt.setString(1, typeName.replaceAll("%20", " "));
  190. surveyStmt.setInt(2, regionId);
  191. surveyStmt.setBoolean(3, shrink);
  192. surveyStmt.executeUpdate();
  193. rs = surveyStmt.getGeneratedKeys();
  194. rs.first();
  195. surveyDbId = rs.getInt(1);
  196.  
  197. this.sqlStatements.put(this.sqlCounter, "INSERT INTO surveys (name, guid, date_added, region_id, shrink) VALUES( '"+ typeName.replaceAll("%20", " ") +"', md5(now()), now(), "+ regionId +", "+ shrink +")");
  198.  
  199.  
  200. }else if(type.equals("publish")){
  201. //insert data in to the survey table and get the last row ID.
  202. surveyStmt = dbConn.prepareStatement("INSERT INTO surveys (name, guid, published, date_added, region_id, shrink) VALUES( ?, md5(now()), 1, now(), ?, ? )");
  203. surveyStmt.setString(1, typeName.replaceAll("%20", " "));
  204. surveyStmt.setInt(2, regionId);
  205. surveyStmt.setBoolean(3, shrink);
  206. surveyStmt.executeUpdate();
  207. rs = surveyStmt.getGeneratedKeys();
  208. rs.first();
  209. surveyDbId = rs.getInt(1);
  210.  
  211. this.sqlStatements.put(this.sqlCounter, "INSERT INTO surveys (name, guid, published, date_added, region_id, shrink) VALUES( '"+ typeName.replaceAll("%20", " ") +"', md5(now()), 1, now(), "+ regionId +", "+ shrink +")");
  212.  
  213. }
  214. this.sqlCounter++;
  215.  
  216. rs.close();
  217. surveyStmt.close();
  218.  
  219. //Total lat/long values to work center point.
  220. double totalLat = 0.0;
  221. double totalLong = 0.0;
  222. int totalLongLats = 0;
  223.  
  224.  
  225. while (iterator.hasNext()) {
  226.  
  227. SimpleFeatureImpl feature = (SimpleFeatureImpl) iterator.next();
  228. Geometry sourceGeometry = (Geometry) feature.getDefaultGeometry();
  229.  
  230. //we now need to enter a link between the survey and the line data.
  231. PreparedStatement lineStmt = null;
  232. String lineStmtSQL = "INSERT INTO `lines` (guid, date_added, survey_id) VALUES (md5(now()), now(), ?)";
  233. lineStmt = dbConn.prepareCall(lineStmtSQL);
  234. lineStmt.setInt(1, surveyDbId);
  235. lineStmt.executeUpdate();
  236. ResultSet lineRS = lineStmt.getGeneratedKeys();
  237. lineRS.first();
  238. int lineDbId = lineRS.getInt(1);
  239.  
  240. this.sqlStatements.put(this.sqlCounter, "INSERT INTO `lines` (guid, date_added, survey_id) VALUES (md5(now()), now(), " +
  241. surveyDbId +")");
  242. this.sqlCounter++;
  243.  
  244.  
  245. lineRS.close();
  246. lineStmt.close();
  247.  
  248. //we now look for the properties of the line that we are about to process.
  249. Collection<Property> coll = feature.getProperties();
  250. Iterator colIterator = coll.iterator();
  251.  
  252. //we now look at inserting the line properties into the database.
  253. PreparedStatement linePropStmt = null;
  254. String linePropSQL = "INSERT INTO line_properties(`key`, `value`, `line_id`) VALUES(?, ?, ?)";
  255. linePropStmt = dbConn.prepareStatement(linePropSQL);
  256.  
  257. while (colIterator.hasNext()) {
  258.  
  259. Property geoProp = (Property) colIterator.next();
  260. if (verbose) {
  261. System.out.println("prop name: " + geoProp.getName() + " Value: " + geoProp.getValue());
  262. }
  263.  
  264. if (geoProp.getValue() != null && !geoProp.getName().toString().equals("the_geom")) {
  265. //we then prepare the statment and execute for each property.
  266. linePropStmt.setString(1, geoProp.getName().toString());
  267. linePropStmt.setString(2, geoProp.getValue().toString());
  268. linePropStmt.setInt(3, lineDbId);
  269.  
  270. linePropStmt.executeUpdate();
  271.  
  272. //add to sql command hashmap
  273. this.sqlStatements.put(this.sqlCounter, "INSERT INTO line_properties(`key`, `value`, `line_id`) VALUES('" +
  274. geoProp.getName().toString()+"', '"+
  275. geoProp.getValue().toString()+"', "+lineDbId+")");
  276. this.sqlCounter++;
  277. }
  278.  
  279. }
  280.  
  281. //close the prepared statement now that we've finished with it.
  282. linePropStmt.close();
  283.  
  284.  
  285. //finally we populate the line coordinates into the database.
  286. Coordinate[] coordinates = sourceGeometry.getCoordinates();
  287. String lineName = feature.getID();
  288. if (verbose) {
  289. System.out.println(lineName);
  290. }
  291.  
  292. PreparedStatement coordStmt = null;
  293. String coordSQL = "INSERT INTO coordinates(`x`, `y`, line_id, sequence) VALUES(?, ?, ?, ?)";
  294. coordStmt = dbConn.prepareStatement(coordSQL);
  295.  
  296. PreparedStatement lineStringStmt = null;
  297. String longLatSQL = "UPDATE `lines` SET longlat_string = ? WHERE line_id = ?";
  298. lineStringStmt = dbConn.prepareStatement(longLatSQL);
  299.  
  300. /**
  301. * Used to build a full long lat string for fast lookups
  302. * when being used in KML/Google Maps
  303. */
  304. String longLatString = "";
  305. boolean first = true;
  306. int shotCount = 0;
  307.  
  308. System.out.println("Coordinates length: "+ coordinates.length);
  309.  
  310. for (int i = 0; i < coordinates.length; i++) {
  311.  
  312. if (verbose) {
  313. System.out.println("x: " + coordinates[i].x + " - Y: " + coordinates[i].y);
  314. }
  315.  
  316. //add to the long lat counter to find center point
  317. totalLat += coordinates[i].x;
  318. totalLong += coordinates[i].y;
  319. totalLongLats++;
  320.  
  321. coordStmt.setString(1, Double.toString(coordinates[i].x));
  322. coordStmt.setString(2, Double.toString(coordinates[i].y));
  323. coordStmt.setInt(3, lineDbId);
  324. coordStmt.setInt(4, i);
  325.  
  326. coordStmt.executeUpdate();
  327.  
  328. this.sqlStatements.put(this.sqlCounter, "INSERT INTO coordinates(`x`, `y`, line_id, sequence) VALUES('"+
  329. Double.toString(coordinates[i].x) +"', '"+
  330. Double.toString(coordinates[i].y) +"', "+lineDbId+", "+i+")");
  331. this.sqlCounter++;
  332.  
  333.  
  334. //now we build/update a string for the full coordinates.
  335. if (first) {
  336.  
  337. longLatString = Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  338.  
  339. first = false;
  340.  
  341. } else {
  342.  
  343. // Not implemented function fully, to strip long lats to start and end points only
  344. if(shrink){
  345.  
  346. if(i == (coordinates.length -1)){
  347. longLatString += ",0 " + Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  348. }
  349.  
  350. }else{
  351. /* // testing this if
  352. if( coordinates.length < 10 && shotCount == 2 ){
  353. longLatString += ",0 " + Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  354. shotCount = 0;
  355.  
  356. }else */
  357. //if( coordinates.length >= 10 && coordinates.length < 30 && shotCount == 4 ){
  358. if( coordinates.length < 30 && shotCount == 4 ){
  359. longLatString += ",0 " + Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  360. shotCount = 0;
  361.  
  362. }else if( coordinates.length >= 30 && coordinates.length < 100 && shotCount == 9 ){
  363. longLatString += ",0 " + Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  364. shotCount = 0;
  365.  
  366. }else if( coordinates.length >= 100 && coordinates.length < 1000 && shotCount == 49 ){
  367. longLatString += ",0 " + Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  368. shotCount = 0;
  369.  
  370. }else if( coordinates.length >= 1000 && coordinates.length < 10000 && shotCount == 99 ){
  371. longLatString += ",0 " + Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  372. shotCount = 0;
  373.  
  374. }else if( coordinates.length >= 10000 && shotCount == 199 ){
  375. longLatString += ",0 " + Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  376. shotCount = 0;
  377. }
  378.  
  379. // Make sure to get last co-ordinate of line
  380. if( i == (coordinates.length-1) ){ // -1 since coutn starts from zero
  381. longLatString += ",0 " + Double.toString(coordinates[i].x) + "," + Double.toString(coordinates[i].y);
  382. System.out.println("Last coordinate stored: " + longLatString);
  383. shotCount = 0;
  384. }
  385. }
  386.  
  387. }
  388.  
  389. shotCount ++;
  390. }
  391.  
  392. /**
  393. * Update the line table to include the precreated line string.
  394. */
  395. lineStringStmt.setString(1, longLatString);
  396. lineStringStmt.setInt(2, lineDbId);
  397. lineStringStmt.executeUpdate();
  398.  
  399. this.sqlStatements.put(this.sqlCounter, "UPDATE `lines` SET longlat_string = '"+longLatString+"' WHERE line_id = " + lineDbId);
  400. this.sqlCounter++;
  401.  
  402. lineStringStmt.close();
  403. coordStmt.close();
  404.  
  405.  
  406. }
  407.  
  408. //finally we update the center point
  409. PreparedStatement centerPointStmt = null;
  410. String centerPointSQL = "UPDATE `surveys` SET center_point = ? WHERE survey_id = ?";
  411. centerPointStmt = dbConn.prepareStatement(centerPointSQL);
  412.  
  413. double centerLat = totalLat/totalLongLats;
  414. double centerLong = totalLong/totalLongLats;
  415.  
  416. String centerPoint = Double.toString(centerLat) + ", " + Double.toString(centerLong);
  417.  
  418. centerPointStmt.setString(1, centerPoint);
  419. centerPointStmt.setInt(2, surveyDbId);
  420. centerPointStmt.executeUpdate();
  421. centerPointStmt.close();
  422.  
  423. this.sqlStatements.put(this.sqlCounter, "UPDATE `surveys` SET center_point = '"+centerPoint+"' WHERE survey_id = "+surveyDbId);
  424. this.sqlCounter++;
  425.  
  426. //we now commit the transaction
  427. dbConn.commit();
  428. //now we can close the database connection.
  429. dbConn.close();
  430.  
  431. this.sqlStatements.put(this.sqlCounter,"COMMIT");
  432. this.sqlCounter++;
  433.  
  434. BufferedWriter out = new BufferedWriter(new java.io.FileWriter(this.configFile.getProperty("SQL_FILE").toString()));
  435.  
  436. for(int i = 0; i < this.sqlStatements.size(); i++){
  437.  
  438. out.write(this.sqlStatements.get(i) + ";" + "n");
  439.  
  440. }
  441.  
  442. //close file connection
  443. out.close();
  444. System.out.println("End transaction");
  445.  
  446. } catch (Exception ex) {
  447. System.out.println();
  448. System.out.println("Message: "+ex.getMessage());
  449. if(ex.getMessage().trim().contains("Communications link failure")){
  450. System.out.println("Check there is a mysql connection and retry");
  451. System.out.println();
  452. System.exit(0);
  453.  
  454. }else{
  455.  
  456. Logger.getLogger(Converter.class.getName()).log(Level.SEVERE, null, ex);
  457.  
  458. //we've errored so we need to toll back the connection.
  459. dbConn.rollback();
  460.  
  461. //we now insert a message into the audit log.
  462.  
  463. String auditSQL = "INSERT INTO auditlog (username, date_added, message, guid, type)"
  464. + "VALUES('mcgeotools', now(), ?, md5(now()), ?)";
  465.  
  466. PreparedStatement auditStmt = dbConn.prepareStatement(auditSQL);
  467.  
  468. //we want to know which survey failed to import and we need the stacktrace message.
  469. String auditMessage = "Error whilst populating Survey " + typeName + " with error message : " + ex.getMessage();
  470.  
  471. auditStmt.setString(1, auditMessage);
  472. auditStmt.setString(2, "error"); //error
  473.  
  474. auditStmt.execute();
  475.  
  476. dbConn.commit();
  477. }
  478.  
  479. } finally {
  480.  
  481. //close the iterator
  482. iterator.close();
  483.  
  484. //commit any transaction and close the connection to the database if it is open.
  485. if (dbConn != null && !dbConn.isClosed()) {
  486. try {
  487. dbConn.commit();
  488. dbConn.close();
  489. } catch (Exception dbex) {
  490. System.out.println("Communications failure, check there is a connection to mysql");
  491. Logger.getLogger(Converter.class.getName()).log(Level.SEVERE, null, dbex);
  492. }
  493. }
  494.  
  495. //close file connection
  496.  
  497. }
  498.  
  499.  
  500. return surveyDbId;
  501. }
  502.  
  503. /**
  504. * Get the coordinate system that is used in the shape file
  505. *
  506. * @param String shpFilePath
  507. * @return String
  508. */
  509. private String getCoordinateSystem(String shpFilePath) throws MalformedURLException, IOException {
  510. String coordType = null;
  511. ShpFiles shapeFiles = new ShpFiles(shpFilePath);
  512. PrjFileReader fileReader = new PrjFileReader(shapeFiles);
  513. CoordinateReferenceSystem coordSystem = fileReader.getCoodinateSystem();
  514. coordType = coordSystem.getName().toString();
  515. return coordType;
  516. }
  517. }
Add Comment
Please, Sign In to add comment