API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Feb 10th, 2016
76
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 16.59 KB | None | 0 0
raw download clone embed print report
  1. package com.cisco.installbase.hiveconnector;
  2.  
  3. import java.util.Date;
  4.  
  5. public class MainApp {
  6.  
  7. private static final String hiveDB = ReadProperties.getInstance().getProperty("hive_db");
  8. private static final String logTable = ReadProperties.getInstance().getProperty("IB_log_table");
  9. private static final String dataGovernanceLogTable = ReadProperties.getInstance().getProperty("SR_DG_table");
  10. private static final String dataGovernanceMasterTable = ReadProperties.getInstance()
  11. .getProperty("SR_DG_master_table");
  12.  
  13. private static final String count_xxccs_ds_sahdr_core = "select count(*) from " + hiveDB + "."
  14. + "xxccs_ds_sahdr_core";
  15. private static final String count_mtl_system_items_b = "select count(*) from " + hiveDB + "."
  16. + "mtl_system_items_b";
  17. private static final String count_xxccs_scdc_product_profile = "select count(*) from " + hiveDB + "."
  18. + "xxccs_scdc_product_profile";
  19. private static final String count_xxccs_ds_cvdprdline_detail = "select count(*) from " + hiveDB + "."
  20. + "xxccs_ds_cvdprdline_detail";
  21. private static final String count_xxccs_ds_instance_detail = "select count(*) from " + hiveDB + "."
  22. + "xxccs_ds_instance_detail";
  23.  
  24. private static int currentJobID = 0;
  25. private static Date startTime = null;
  26. private static Date stopTime = null;
  27. private static int runTime = 0;
  28.  
  29. static CommonDBUtilities commonDB = new CommonDBUtilities();
  30. static ShellUtilities shellUtilities = new ShellUtilities();
  31. static SqoopUtility sqoop = new SqoopUtility();
  32.  
  33. public static void main(String[] args) {
  34.  
  35. MainApp.startTimeLogger();
  36. System.out.println("Started the Job");
  37.  
  38. }
  39.  
  40. public static void startTimeLogger() {
  41. // getting the Job ID and the start time for the log table
  42.  
  43. if (Constants.isFlag()) {
  44. currentJobID = commonDB.getMaximumJobID();
  45. startTime = commonDB.getTime();
  46. MainApp.importTables();
  47. System.out.println("executing startTimeLogger");
  48. } else {
  49. MainApp.onFailure();
  50. JobMailer.PostMail("IB Load Failed", "Load failed while logging method name startTimeLogger()");
  51. System.out.println("executing startTimeLogger failed");
  52. }
  53. }
  54.  
  55. public static void importTables() {
  56. // Delete target directory before running the sqoop imports
  57.  
  58. if (Constants.isFlag()) {
  59. shellUtilities.DeleteDirectory(Constants.getMtlSystems());
  60. shellUtilities.DeleteDirectory(Constants.getProductLine());
  61. shellUtilities.DeleteDirectory(Constants.getInstanceDetail());
  62. shellUtilities.DeleteDirectory(Constants.getProductLine());
  63. shellUtilities.DeleteDirectory(Constants.getHeaderCore());
  64.  
  65. // Run the sqoop imports to load the data from oracle to hive
  66.  
  67. sqoop.runSqoop();
  68. MainApp.getCounts();
  69. System.out.println("executing importTables");
  70. } else {
  71. MainApp.onFailure();
  72. JobMailer.PostMail("IB Load Failed", "Load failed while running sqoop import method name importTables()");
  73. System.out.println("executing importTables failed");
  74. }
  75.  
  76. }
  77.  
  78. public static void getCounts() {
  79.  
  80. // Get the record counts for all the IB tables pulled
  81.  
  82. if (Constants.isFlag()) {
  83. commonDB.getCounts(count_xxccs_ds_instance_detail);
  84. commonDB.getCounts(count_xxccs_ds_cvdprdline_detail);
  85. commonDB.getCounts(count_xxccs_scdc_product_profile);
  86. commonDB.getCounts(count_mtl_system_items_b);
  87. commonDB.getCounts(count_xxccs_ds_sahdr_core);
  88. MainApp.stopTimeLogger();
  89. System.out.println("executing getCounts");
  90. } else {
  91. MainApp.onFailure();
  92. JobMailer.PostMail("IB Load Failed", "Load failed while getting counts method name getCounts()");
  93. System.out.println("executing getCounts failed");
  94. }
  95. }
  96.  
  97. public static void stopTimeLogger() {
  98. // Get the stop time or end time
  99. if (Constants.isFlag()) {
  100. stopTime = commonDB.getTime();
  101. MainApp.runTimeLogger();
  102. System.out.println("executing stopTimeLogger");
  103. } else {
  104. MainApp.onFailure();
  105. JobMailer.PostMail("IB Load Failed", "Load failed while end logging method name stopTimeLogger()");
  106. System.out.println("executing stopTimeLogger failed");
  107. }
  108. }
  109.  
  110. public static void runTimeLogger() {
  111. // Get the run time or total time taken
  112. if (Constants.isFlag()) {
  113. runTime = (int) (stopTime.getTime() - startTime.getTime()) / 1000 * 60 * 60 * 24;
  114. MainApp.onSuccess();
  115. MainApp.logGovernance();
  116. System.out.println("executing runTimeLogger");
  117. } else {
  118. MainApp.onFailure();
  119. JobMailer.PostMail("IB Load Failed", "Load failed while runtime logging method name runTimeLogger()");
  120. System.out.println("executing runTimeLogger failed");
  121. }
  122. }
  123.  
  124. public static void logGovernance() {
  125. // IB Data governance
  126.  
  127. if (Constants.isFlag()) {
  128. String dataGovernance = "Insert into table " + hiveDB + "." + dataGovernanceLogTable
  129. + " select Data_Asset_Reference,File_Name,Origin_System,Transfer_System," + startTime
  130. + ",Column_Reference,Element_Reference,Rule_Priority,Delete_By_Date,Classification,Geographic_Inclusion,Geographic_Restriction,Group_Inclusion,Group_Restriction,Reserved from "
  131. + hiveDB + "." + dataGovernanceMasterTable;
  132. commonDB.InsertToTable(dataGovernance);
  133. System.out.println("executing logGovernance");
  134. } else {
  135. MainApp.onFailure();
  136. JobMailer.PostMail("IB Load Failed",
  137. "Load failed while inserting into datagovernance method name logGovernance()");
  138. System.out.println("executing logGovernance failed");
  139. }
  140. }
  141.  
  142. public static void onFailure() {
  143. // Write to log on Failure
  144. String insertOnFailure = "insert into table " + hiveDB + "." + logTable + " select " + currentJobID + ","
  145. + stopTime + "," + runTime + "," + "FAILED from " + hiveDB + "." + "dual" + " limit 1; ";
  146. commonDB.InsertToTable(insertOnFailure);
  147. JobMailer.PostMail("IB Load Failed", "Load failed");
  148. System.out.println("executing onFailure");
  149. }
  150.  
  151. public static void onSuccess() {
  152. // Write to log on Success
  153. String insertOnSuccess = "insert into table " + hiveDB + "." + logTable + " select " + currentJobID + ","
  154. + stopTime + "," + runTime + "," + "SUCCESS from " + hiveDB + "." + "dual" + " limit 1; ";
  155. commonDB.InsertToTable(insertOnSuccess);
  156. JobMailer.PostMail("IB Load Successfully completed", "Load completed");
  157. System.out.println("executing onSuccess");
  158. }
  159.  
  160. }
  161.  
  162. package com.cisco.installbase.hiveconnector;
  163.  
  164. import java.io.IOException;
  165. import java.io.InputStream;
  166. import java.util.Properties;
  167. import java.util.Set;
  168.  
  169. public class ReadProperties {
  170.  
  171. private final Properties props = new Properties();
  172.  
  173. private ReadProperties() {
  174. InputStream in = this.getClass().getClassLoader().getResourceAsStream("config.properties");
  175. try {
  176. props.load(in);
  177. Constants.setFlag(true);
  178. } catch (IOException e) {
  179. e.printStackTrace();
  180. Constants.setFlag(false);
  181. }
  182. }
  183.  
  184. private static class PropHolder {
  185. private static final ReadProperties INSTANCE = new ReadProperties();
  186. }
  187.  
  188. public static ReadProperties getInstance() {
  189. return PropHolder.INSTANCE;
  190. }
  191.  
  192. public String getProperty(String key) {
  193. return props.getProperty(key);
  194. }
  195.  
  196. public Set<String> getAllPropertyNames() {
  197. return props.stringPropertyNames();
  198. }
  199.  
  200. public boolean containsKey(String key) {
  201. return props.containsKey(key);
  202. }
  203. }
  204.  
  205. package com.cisco.installbase.hiveconnector;
  206.  
  207. import java.sql.Connection;
  208. import java.sql.PreparedStatement;
  209. import java.sql.ResultSet;
  210. import java.sql.SQLException;
  211. import java.sql.Statement;
  212. import java.util.Date;
  213.  
  214. public class CommonDBUtilities {
  215.  
  216. static final String hiveDB = ReadProperties.getInstance().getProperty("hive_db");
  217. static final String logTable = ReadProperties.getInstance().getProperty("IB_log_table");
  218.  
  219. Connection con = CreateConnection.getInstance();
  220. Statement stm = null;
  221. ResultSet rs = null;
  222. PreparedStatement pstat = null;
  223.  
  224. String sql1 = "select max(job_id)+1 from " + hiveDB + "." + logTable;
  225.  
  226. public Date getTime() {
  227. return new Date();
  228. }
  229.  
  230. public int getMaximumJobID() {
  231. int maximumID = -1;
  232. try {
  233. System.out.println(con);
  234. stm = con.createStatement();
  235. rs = stm.executeQuery(sql1);
  236. maximumID = rs.getInt(0);
  237. Constants.setFlag(true);
  238. } catch (SQLException e) {
  239. e.printStackTrace();
  240. Constants.setFlag(false);
  241. }
  242. return maximumID;
  243. }
  244.  
  245. public int getCounts(String query) {
  246. int count = 0;
  247. try {
  248. stm = con.createStatement();
  249. rs = stm.executeQuery(query);
  250. count = rs.getInt(0);
  251. Constants.setFlag(true);
  252. } catch (SQLException e) {
  253. e.printStackTrace();
  254. Constants.setFlag(false);
  255. }
  256.  
  257. return count;
  258. }
  259.  
  260. public void InsertToTable(String insertquery) {
  261. try {
  262. pstat = con.prepareStatement(insertquery);
  263. Constants.setFlag(true);
  264. } catch (SQLException e) {
  265. e.printStackTrace();
  266. Constants.setFlag(false);
  267. }
  268. }
  269. }
  270.  
  271. CreateConnection.java (Singleton class which creates a connection object to the database)
  272.  
  273. package com.cisco.installbase.hiveconnector;
  274.  
  275. import java.sql.DriverManager;
  276. import java.sql.SQLException;
  277. import java.sql.Connection;
  278.  
  279. public class CreateConnection {
  280.  
  281. private static Connection instance = null;
  282. static final String drivername = "org.apache.hive.jdbc.HiveDriver";
  283.  
  284. private CreateConnection() {
  285.  
  286. try {
  287. Class.forName(drivername);
  288. // instance =
  289. // DriverManager.getConnection("jdbc:hive2://hddev-c01-edge-01:20000/",
  290. // "phodisvc", "B1GD4T4dev");
  291. // for hive 1 use this ------> instance =
  292. // DriverManager.getConnection("thrift://hddev-c01-edge-02:9083");
  293. instance = DriverManager.getConnection("jdbc:hive://hddev-c01-edge-01:9083/");
  294. System.out.println("get instance" + instance);
  295. Constants.setFlag(true);
  296. } catch (ClassNotFoundException e) {
  297. e.printStackTrace();
  298. Constants.setFlag(false);
  299. } catch (SQLException e) {
  300. e.printStackTrace();
  301. Constants.setFlag(false);
  302. } catch (Exception e) {
  303. e.printStackTrace();
  304. }
  305. }
  306.  
  307. public static Connection getInstance() {
  308. if (instance == null) {
  309. instance = (Connection) new CreateConnection();
  310. }
  311. Constants.setFlag(true);
  312. return instance;
  313. }
  314. }
  315.  
  316. package com.cisco.installbase.hiveconnector;
  317.  
  318. import java.io.IOException;
  319. //import java.net.URI;
  320. //import java.net.URISyntaxException;
  321.  
  322. import org.apache.hadoop.conf.Configuration;
  323. import org.apache.hadoop.fs.FileSystem;
  324. import org.apache.hadoop.fs.Path;
  325.  
  326. public class ShellUtilities {
  327.  
  328. String target_dir = ReadProperties.getInstance().getProperty("target_dir");
  329. String tablename = "";
  330. // String maprfsURI = "maprfs://hdnprd-c01-r01-01:7222";
  331.  
  332. Configuration conf = new Configuration();
  333. FileSystem fs = null;
  334. String dir = " ";
  335.  
  336. public void DeleteDirectory(String tablename) {
  337. String fullpath = target_dir + tablename;
  338. try {
  339. // fs = FileSystem.get(new URI(maprfsURI), conf);
  340. fs = FileSystem.get(conf);
  341. Constants.setFlag(true);
  342. } catch (IOException e) {
  343. e.printStackTrace();
  344. Constants.setFlag(false);
  345. }
  346. // } catch (URISyntaxException e) {
  347. // e.printStackTrace();
  348. // Constants.setFlag(false);
  349. // }
  350. Path directory = new Path(fullpath);
  351.  
  352. try {
  353. if (fs.exists(directory)) {
  354. fs.delete(directory, true);
  355. }
  356. Constants.setFlag(true);
  357. } catch (IOException e) {
  358. e.printStackTrace();
  359. Constants.setFlag(false);
  360. }
  361.  
  362. }
  363. }
  364.  
  365. public class SqoopUtility {
  366.  
  367. private static final String connectionString = "'" + ReadProperties.getInstance().getProperty("dburl") + "'";
  368. @SuppressWarnings("unused")
  369. private static final String edgeNode = ReadProperties.getInstance().getProperty("EDGE_HIVE_CONN");
  370. private static final String targetDir = ReadProperties.getInstance().getProperty("target_dir") + "/";
  371. private static final String userName = ReadProperties.getInstance().getProperty("user_name");
  372. private static final String password = ReadProperties.getInstance().getProperty("password");
  373. private static final String sqoopEdgeNode = ReadProperties.getInstance().getProperty("SQOOP_EDGE_CONN");
  374. private static final String hiveDB = ReadProperties.getInstance().getProperty("hive_db");
  375.  
  376. String[] command = { "sh", "/apps/pentaho_nfs/installbase/input/poc/parallel.sh", sqoopEdgeNode, connectionString,
  377. userName, password, targetDir, hiveDB };
  378. ProcessBuilder processBuilder = null;
  379. @SuppressWarnings("unused")
  380. private Process spawnProcess = null;
  381.  
  382. public void runSqoop() {
  383. processBuilder = new ProcessBuilder(command);
  384. try {
  385. spawnProcess = processBuilder.start();
  386. Constants.setFlag(true);
  387. } catch (IOException e) {
  388. e.printStackTrace();
  389. Constants.setFlag(false);
  390. }
  391. }
  392. }
  393.  
  394. package com.cisco.installbase.hiveconnector;
  395.  
  396. public class Constants {
  397.  
  398. private static boolean flag = false;
  399.  
  400. public static boolean isFlag() {
  401. return flag;
  402. }
  403.  
  404. public static void setFlag(boolean flag) {
  405. Constants.flag = flag;
  406. }
  407.  
  408. private static String mtlSystems = "MTL_SYSTEM_ITEMS_B";
  409. private static String productLine = "XXCCS_DS_CVDPRDLINE_DETAIL";
  410. private static String instanceDetail = "XXCCS_DS_INSTANCE_DETAIL";
  411. private static String productProfile = "XXCCS_SCDC_PRODUCT_PROFILE";
  412. private static String headerCore = "XXCCS_DS_SAHDR_CORE";
  413.  
  414. public static String getMtlSystems() {
  415. return mtlSystems;
  416. }
  417.  
  418. public static String getProductLine() {
  419. return productLine;
  420. }
  421.  
  422. public static String getInstanceDetail() {
  423. return instanceDetail;
  424. }
  425.  
  426. public static String getProductProfile() {
  427. return productProfile;
  428. }
  429.  
  430. public static String getHeaderCore() {
  431. return headerCore;
  432. }
  433. }
  434.  
  435. // private static boolean startLogTableFlag = false;
  436. // private static boolean ingestionFlag = false;
  437. // private static boolean recordCountFlag = false;
  438. // private static boolean stopLogTableFlag = false;
  439. // private static boolean runtimeLogTableFlag = false;
  440. // private static boolean writeToLogTableFlag = false;
  441. // public static boolean isStartLogTableFlag() {
  442. // return startLogTableFlag;
  443. // }
  444. //
  445. // public static boolean isIngestionFlag() {
  446. // return ingestionFlag;
  447. // }
  448. //
  449. // public static boolean isRecordCountFlag() {
  450. // return recordCountFlag;
  451. // }
  452. //
  453. // public static boolean isStopLogTableFlag() {
  454. // return stopLogTableFlag;
  455. // }
  456. //
  457. // public static boolean isRuntimeLogTableFlag() {
  458. // return runtimeLogTableFlag;
  459. // }
  460. //
  461. // public static boolean isWriteToLogTableFlag() {
  462. // return writeToLogTableFlag;
  463. // }
  464. //
  465. // public static void setStartLogTableFlag(boolean startLogTableFlag) {
  466. // Constants.startLogTableFlag = startLogTableFlag;
  467. // }
  468. //
  469. // public static void setIngestionFlag(boolean ingestionFlag) {
  470. // Constants.ingestionFlag = ingestionFlag;
  471. // }
  472. //
  473. // public static void setRecordCountFlag(boolean recordCountFlag) {
  474. // Constants.recordCountFlag = recordCountFlag;
  475. // }
  476. //
  477. // public static void setStopLogTableFlag(boolean stopLogTableFlag) {
  478. // Constants.stopLogTableFlag = stopLogTableFlag;
  479. // }
  480. //
  481. // public static void setRuntimeLogTableFlag(boolean runtimeLogTableFlag) {
  482. // Constants.runtimeLogTableFlag = runtimeLogTableFlag;
  483. // }
  484. //
  485. // public static void setWriteToLogTableFlag(boolean writeToLogTableFlag) {
  486. // Constants.writeToLogTableFlag = writeToLogTableFlag;
  487. // }
  488.  
  489. // IB Tables getters
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!