Advertisement
Guest User

Untitled

a guest
Feb 24th, 2019
127
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 12.72 KB | None | 0 0
  1. import org.jsoup.Jsoup;
  2. import org.jsoup.nodes.Document;
  3. import org.jsoup.nodes.Element;
  4. import org.jsoup.select.Elements;
  5.  
  6. import java.io.IOException;
  7. import java.sql.*;
  8. import java.text.NumberFormat;
  9. import java.text.ParseException;
  10. import java.util.HashSet;
  11. import java.util.Set;
  12.  
  13. public class BestBuy_PopulateProducts {
  14.  
  15. public static void main(String[] args) throws IOException, ParseException, SQLException {
  16.  
  17. // fetch data from Best Buy
  18. final String TABLET_CATEGORY = "tablet";
  19. final String DESKTOP_CATEGORY = "desktop";
  20. final String LAPTOP_CATEGORY = "laptop";
  21. Product[] tabletsList = getProductsByCategory(TABLET_CATEGORY);
  22. Product[] desktopsList = getProductsByCategory(DESKTOP_CATEGORY);
  23. Product[] laptopsList = getProductsByCategory(LAPTOP_CATEGORY);
  24. logProducts(tabletsList, desktopsList, laptopsList);
  25.  
  26. // insert into tables
  27. Connection connection = null;
  28. Statement statement = null;
  29. try {
  30. connection = initializeConnection();
  31. statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
  32.  
  33. // do not insert if `products` not empty
  34. if (isProductTableEmpty(statement)) {
  35. // insert into `company`
  36. insertIntoCompanyTable(tabletsList, statement);
  37. insertIntoCompanyTable(desktopsList, statement);
  38. insertIntoCompanyTable(laptopsList, statement);
  39.  
  40. // insert into `product`
  41. insertIntoProduct(tabletsList, statement);
  42. insertIntoProduct(desktopsList, statement);
  43. insertIntoProduct(laptopsList, statement);
  44. }
  45. } catch (Exception e) {
  46. e.printStackTrace();
  47. } finally {
  48. if (statement != null) statement.close();
  49. if (connection != null) connection.close();
  50. }
  51. }
  52.  
  53. public static boolean isProductTableEmpty(Statement statement) throws SQLException {
  54. final String TABLENAME = "product";
  55. String selectQuery = "SELECT COUNT(*) AS total FROM " + TABLENAME + ";";
  56. ResultSet resultSet = getSelection(selectQuery, statement);
  57.  
  58. int count = 0;
  59. if (resultSet.next())
  60. count = resultSet.getInt("total");
  61. return count == 0;
  62. }
  63.  
  64. public static void insertIntoProduct(Product[] productsList, Statement statement) {
  65. final String TABLENAME = "product";
  66. for (Product product: productsList) {
  67. String insertQuery = "INSERT INTO " + TABLENAME +
  68. " (product_ID, product_name, price, description, category, discount, cname) VALUES(" +
  69. "\'" + product.product_ID + "\', " +
  70. "\'" + product.product_name + "\', " +
  71. product.price + ", " +
  72. "\'" + product.description + "\', " +
  73. "\'" + product.category + "\', " +
  74. product.discount + ", " +
  75. "\'" + product.cname + "\') ON CONFLICT DO NOTHING;";
  76. runQuery(insertQuery, statement);
  77. }
  78. }
  79.  
  80. public static Set insertIntoCompanyTable(Product[] productsList, Statement statement) {
  81. final String TABLENAME = "company";
  82. Set<String> companyNames = new HashSet<String>();
  83. for (Product product: productsList)
  84. companyNames.add(product.cname);
  85. for (String company: companyNames) {
  86. String insertQuery = "INSERT INTO " + TABLENAME + " (cname) VALUES(\'" + company + "\') ON CONFLICT DO NOTHING;";
  87. runQuery(insertQuery, statement);
  88. }
  89. return companyNames;
  90. }
  91.  
  92. /**
  93. * Use this for SELECTION queries
  94. */
  95. public static ResultSet getSelection(String sqlSelect, Statement statement) {
  96. ResultSet resultSet = null;
  97. try {
  98. log(sqlSelect);
  99. resultSet = statement.executeQuery(sqlSelect);
  100. } catch (SQLException e) {
  101. logSqlException(e);
  102. }
  103. return resultSet;
  104. }
  105.  
  106. public static void runQueries(String[] sqlQueries, Statement statement) {
  107. for (String sqlQuery: sqlQueries)
  108. runQuery(sqlQuery, statement);
  109. }
  110.  
  111. /**
  112. * Run SQL query, return true if successful
  113. */
  114. public static boolean runQuery(String sqlQuery, Statement statement) {
  115. try {
  116. log(sqlQuery);
  117. statement.executeQuery(sqlQuery);
  118. } catch (SQLException e) {
  119. logSqlException(e);
  120. return false;
  121. }
  122. return true;
  123. }
  124.  
  125. /**
  126. * Register & return the connection
  127. * @return
  128. */
  129. public static Connection initializeConnection() {
  130. final String DATABASE_URL = System.getenv("COMP421_URL");
  131. final String DATABASE_USER = System.getenv("COMP421_USER");
  132. final String DATABASE_PASS = System.getenv("COMP421_PASS");
  133. Connection connection = null;
  134.  
  135. try {
  136. DriverManager.registerDriver(new org.postgresql.Driver());
  137. connection = DriverManager.getConnection(
  138. DATABASE_URL,
  139. DATABASE_USER,
  140. DATABASE_PASS
  141. );
  142. log("CONNECTION SUCCESSFUL");
  143. } catch (Exception e) {
  144. e.printStackTrace();
  145. log("TERMINATING PROGRAM");
  146. System.exit(0);
  147. }
  148. return connection;
  149. }
  150.  
  151. /**
  152. * Given a category:
  153. * - tablet / desktop / laptop
  154. *
  155. * Fetch from the BestBuy website:
  156. * - 1st page of products
  157. */
  158. public static Product[] getProductsByCategory(String category) throws IOException, ParseException {
  159. Document document = getProductsPageByCategory(category);
  160. Elements LISTING_ITEMS = document.getElementsByClass("listing-item");
  161.  
  162. // get category - from arguments
  163. int i = 0, limit = 20;
  164. Product[] productsList = new Product[limit]; // limit to 20 for now - do not overpopulate relation
  165. for (Element item: LISTING_ITEMS) {
  166. double price = getPriceFromItem(item);
  167. String productName = getProductNameFromItem(item);
  168. String companyName = productName.split(" ", 2)[0]; // get company name - from product name
  169. int sku = getSkuFromItem(item); // get sku (id)
  170. double discount = getDiscountFromElement(item);
  171.  
  172. if (!validateProduct(price, companyName, sku)) continue;
  173.  
  174. // description - there is no description to fetch from the dom
  175. // for now, just mention if its on sale or not
  176. String description = (discount > 0.0) ? "On sale!" : "";
  177.  
  178. productsList[i] = new Product(sku, productName, price, description, category, companyName, discount);
  179. if (++i == limit) break;
  180. }
  181. return productsList;
  182. }
  183.  
  184. public static boolean validateProduct(double price, String companyName, int sku) {
  185. return price > 0.0 && !companyName.toLowerCase().equals("refurbished") && sku > 0;
  186. }
  187.  
  188. public static double getDiscountFromElement(Element item) throws ParseException {
  189. double discount = 0.0;
  190. if (item != null) {
  191. Element discountWrapperElement = item.getElementsByClass("prod-saving").first(); // todo: null check first()
  192. if (discountWrapperElement != null) {
  193. Element discountElement = discountWrapperElement.select("span").last(); // todo: null check last()
  194. if (discountElement != null) {
  195. String discountS = discountElement.text();
  196. if (discountS != null) {
  197. NumberFormat format = NumberFormat.getCurrencyInstance();
  198. Number number = format.parse(discountS);
  199. discount = number.doubleValue();
  200. }
  201. }
  202. }
  203. }
  204. return discount;
  205. }
  206.  
  207. public static int getSkuFromItem(Element item) {
  208. int sku = 0;
  209. if (item != null) {
  210. Element skuWrapperElement = item.getElementsByClass("compare-checkbox").first(); // todo: null check first()
  211. if (skuWrapperElement != null) {
  212. Element skuInputElement = skuWrapperElement.select("input").first(); // todo: null check first()
  213. if (skuInputElement != null) {
  214. String skuS = skuInputElement.attr("data-sku");
  215. if (skuS != null) {
  216. sku = Integer.parseInt(skuS);
  217. }
  218. }
  219. }
  220. }
  221. return sku;
  222. }
  223.  
  224. public static String getProductNameFromItem(Element item) {
  225. String productName = "";
  226. if (item != null) {
  227. Element productNameWrapperElement = item.getElementsByClass("prod-title").first(); // todo: null check first()
  228. if (productNameWrapperElement != null) {
  229. Element productNameElement = productNameWrapperElement.select("a").first(); // todo: null check first()
  230. if (productNameElement != null) {
  231. productName = productNameElement.text();
  232. productName = productName.length() >= 70 ? productName.substring(0, 66) + "..." : productName;
  233. }
  234. }
  235. }
  236. return productName;
  237. }
  238.  
  239. public static double getPriceFromItem(Element item) throws ParseException {
  240. double priceD = 0.0;
  241. if (item != null) {
  242. Element priceElement = item.getElementsByClass("amount").first(); // todo: null check first()
  243. if (priceElement != null) {
  244. String price = priceElement.text();
  245. if (price != null) {
  246. NumberFormat format = NumberFormat.getCurrencyInstance();
  247. Number number = format.parse(price);
  248. priceD = number.doubleValue();
  249. }
  250. }
  251. }
  252. return priceD;
  253. }
  254.  
  255. /**
  256. * Unused - use if we want to display images on the GUI
  257. */
  258. public static String getImageFromItem(Element item) {
  259. String image = "";
  260. if (item != null) {
  261. Element thumbnailElement = item.select("img").first(); // todo: null check first()
  262. if (thumbnailElement != null) {
  263. image = thumbnailElement.absUrl("src");
  264. }
  265. }
  266. return image;
  267. }
  268.  
  269. public static Document getProductsPageByCategory(String category) throws IOException {
  270. final String BASE_URL = "https://www.bestbuy.ca/en-ca/category/";
  271. final String productsPageUrl = BASE_URL + getCategoryUrl(category);
  272. log("URL: " + productsPageUrl);
  273. return Jsoup.connect(productsPageUrl).get();
  274. }
  275.  
  276. /**
  277. * tablets: https://www.bestbuy.ca/en-ca/category/tablets-ipads/30297.aspx?
  278. * desktops: https://www.bestbuy.ca/en-ca/category/desktop-computers/20213.aspx?
  279. * laptops: https://www.bestbuy.ca/en-ca/category/laptops-macbooks/20352.aspx?
  280. */
  281. public static String getCategoryUrl(String category) {
  282. String categoryUrl = "";
  283. switch (category) {
  284. case "tablet":
  285. categoryUrl = "tablets-ipads/30297.aspx?";
  286. break;
  287. case "desktop":
  288. categoryUrl = "desktop-computers/20213.aspx?";
  289. break;
  290. case "laptop":
  291. categoryUrl = "laptops-macbooks/20352.aspx?";
  292. break;
  293. }
  294. return categoryUrl;
  295. }
  296.  
  297. public static void logProducts(Product[] tabletsList, Product[] desktopsList, Product[] laptopsList) {
  298. log("TABLETS: ");
  299. for (Product tablet: tabletsList)
  300. log(tablet.toString());
  301.  
  302. log("DESKTOPS: ");
  303. for (Product desktop: desktopsList)
  304. log(desktop.toString());
  305.  
  306. log("LAPTOPS: ");
  307. for (Product laptop: laptopsList)
  308. log(laptop.toString());
  309. }
  310.  
  311. /**
  312. * Print resultSet
  313. */
  314. public static void logResultSet(ResultSet resultSet) {
  315. if (resultSet == null) return;
  316. try {
  317. ResultSetMetaData RSMD = resultSet.getMetaData();
  318. final int NUMBER_COLUMNS = RSMD.getColumnCount();
  319. for (int i = 1; i <= NUMBER_COLUMNS; i++)
  320. System.out.print(RSMD.getColumnName(i) + "\t\t");
  321.  
  322. while (resultSet.next()) {
  323. log();
  324. for (int i = 1; i <= NUMBER_COLUMNS; i++)
  325. System.out.print(resultSet.getString(i) + "\t\t");
  326. }
  327. log();
  328. } catch (SQLException e) {
  329. logSqlException(e);
  330. }
  331. }
  332.  
  333. public static void logSqlException(SQLException e) {
  334. if (!ignoreSqlWarnings(e)) {
  335. log("ERROR CODE:"); log(e.getErrorCode());
  336. log("SQL STATE:"); log(e.getSQLState());
  337. }
  338. }
  339.  
  340. public static boolean ignoreSqlWarnings(SQLException e) {
  341. switch (e.getSQLState()) {
  342. // no data
  343. case "02000":
  344. case "02001":
  345. return true;
  346. }
  347. return false;
  348. }
  349.  
  350. public static void log() { System.out.println(); }
  351. public static void log(String s) { System.out.println(s); }
  352. public static void log(int n) { System.out.println(n); }
  353.  
  354. }
  355.  
  356. class Product {
  357. public int product_ID;
  358. public String product_name;
  359. public double price;
  360. public double discount;
  361. public String description; // similar to toString()
  362. public String category;
  363. public String cname; // company name
  364.  
  365. public Product(int product_ID, String product_name,
  366. double price, String description,
  367. String category, String cname, double discount) {
  368. this.product_ID = product_ID;
  369. this.product_name = product_name;
  370. this.price = price;
  371. this.description = description;
  372. this.category = category;
  373. this.cname = cname;
  374. this.discount = discount;
  375. }
  376.  
  377. @Override
  378. public String toString() { return "\tPRODUCT: " + product_name + " COSTS $" + price; }
  379. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement