GenuineSounds

CoolROMDownloader

Sep 15th, 2014
381
0
Never
1
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 7.70 KB | None | 0 0
  1. import java.io.BufferedOutputStream;
  2. import java.io.File;
  3. import java.io.FileOutputStream;
  4. import java.io.InputStream;
  5. import java.io.OutputStream;
  6. import java.io.OutputStreamWriter;
  7. import java.io.Serializable;
  8. import java.lang.reflect.Type;
  9. import java.net.HttpURLConnection;
  10. import java.net.URL;
  11. import java.util.ArrayList;
  12. import java.util.Arrays;
  13. import java.util.HashMap;
  14. import java.util.List;
  15. import java.util.Map;
  16. import java.util.regex.Matcher;
  17. import java.util.regex.Pattern;
  18.  
  19. import org.jsoup.Jsoup;
  20. import org.jsoup.nodes.Document;
  21. import org.jsoup.nodes.Element;
  22. import org.jsoup.select.Elements;
  23.  
  24. import com.google.gson.Gson;
  25. import com.google.gson.GsonBuilder;
  26. import com.google.gson.reflect.TypeToken;
  27.  
  28. /**
  29.  * This program requires Java 8 since it is using the new Lambda expressions and Streams.
  30.  * @author Brian Wiegand
  31.  * @since Java 8
  32.  */
  33. public class CoolROMDownloader {
  34.  
  35.     public static final File currentDir = new File(".");
  36.     /**
  37.      *  The list of the acceptable consoles.
  38.      *  You can add/remove as many as you like, but at least one is required.
  39.      */
  40.     //  public static final List<String> CONSOLES = Arrays.asList("cps1", "cps2");
  41.     public static final List<String> CONSOLES = Arrays.asList("atari2600", "atari5200", "atari7800", "atarijaguar", "atarilynx",
  42.         "c64", "cps1", "cps2", "gba", "gbc", "mame", "namcosystem22", "neogeo", "neogeocd", "neogeopocket", "nes", "n64",
  43.         "nds", "gcn", "segacd", "dc", "gamegear", "genesis", "mastersystem", "model2", "saturn", "psx", "ps2", "snes");
  44.     /**
  45.      *  The list of the acceptable pages.
  46.      *  You can add/remove as many as you like, but at least one is required.
  47.      */
  48.     public static final List<String> LETTERS =
  49.         Arrays.asList("0", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
  50.                 "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
  51.     /**
  52.      *  The list of the acceptable regions.
  53.      *  You can add/remove as many as you like, but at least one is required.
  54.      */
  55.     public static final List<String> REGIONS = Arrays.asList("USA");
  56.     /**
  57.      * A thread-safe list we use to hold all the Game objects we create.
  58.      */
  59.     public static Map<String, ArrayList<Game>> map = new HashMap<String, ArrayList<Game>>();
  60.  
  61.     /**
  62.      * Main program entry point.
  63.      */
  64.     public static void main(String[] args) {
  65.         for (String string : CONSOLES) {
  66.             map.putIfAbsent(string, new ArrayList<Game>());
  67.         }
  68.         // Brand new Streams and Lambda expressions with parallelism built-in.
  69.         CONSOLES.stream().parallel().forEach(console -> {
  70.             LETTERS.stream().parallel().forEach(letter -> {
  71. //              This isn't needed unless you're downloading for multiple regions.
  72. //              REGIONS.stream().parallel().forEach(region -> {
  73. //                  tryIt(console, letter, region);
  74. //              });
  75.                 tryIt(console, letter, "USA");
  76.             });
  77.         });
  78.         // Quickly sort the list, and gather info.
  79.         for (ArrayList<Game> list : map.values()) {
  80.             list.sort((o1, o2) -> Integer.compare(o1.id, o2.id));
  81.             list.stream().parallel().forEach(g -> {
  82.                 g.findLink();
  83.                 g.getInfo();
  84.             });
  85.         }
  86.         // Save the list as JSON.
  87.         Type mapType = new TypeToken<Map<String, ArrayList<Game>>>() {}.getType();
  88.         GsonBuilder builder = new GsonBuilder();
  89.         builder.setPrettyPrinting();
  90.         Gson gson = builder.create();
  91.         String out = gson.toJson(map, mapType);
  92.         File dir = new File(currentDir, "json");
  93.         File myFile = new File(dir, "out.json");
  94.         try {
  95.             myFile.createNewFile();
  96.             FileOutputStream fOut = new FileOutputStream(myFile);
  97.             OutputStreamWriter myOutWriter = new OutputStreamWriter(fOut);
  98.             myOutWriter.append(out);
  99.             myOutWriter.close();
  100.             fOut.close();
  101.         }
  102.         catch (Exception e) {}
  103.     }
  104.  
  105.     /**
  106.      * Take the console, letter, and region: Try and connect to that page, if so then proceed to find each game
  107.      * in parallel and {@code Main::processLink} on each of the results.
  108.      */
  109.     public static void tryIt(String console, String letter, String region) {
  110.         try {
  111.             // Get the HTML of the page.
  112.             Document doc = Jsoup.connect("http://coolrom.com/roms/" + console + "/" + letter + "/").get();
  113.             // Region is important since it's the easiest way to quickly find each game on the page.
  114.             doc.getElementsByClass(region).parallelStream().forEach(CoolROMDownloader::processLink);
  115.         }
  116.         catch (Exception e) {}
  117.     }
  118.  
  119.     /**
  120.      * Basically this will create and add a Game object to the thread-safe list.
  121.      */
  122.     public static void processLink(Element region) {
  123.         // Find the game by the fact that it is the only link inside the Region.
  124.         Elements gameElement = region.getElementsByAttribute("href");
  125.         // Create the Game Object whereby text() is the game's name, and attr("href") is the page link.
  126.         Game game = new Game(gameElement.text()).parseURL("http://coolrom.com" + gameElement.attr("href"));
  127.         // Add the game to the thread-safe list.
  128.         ArrayList<Game> list = map.get(game);
  129.         synchronized (list) {
  130.             list.add(game);
  131.         }
  132.     }
  133. }
  134.  
  135. class Game implements Comparable<Game>, Serializable {
  136.  
  137.     public static final long serialVersionUID = -2001848694019069568L;
  138.     public static final Pattern PATTERN = Pattern.compile("action=\\\"(.*?)\\\"");
  139.     public final String name;
  140.     public int id;
  141.     public String url;
  142.     public String console;
  143.     public String filePath;
  144.     public String downloadPage;
  145.     public transient String downloadLink;
  146.     public String contentType = "text/html";
  147.     public long contentLength;
  148.     public boolean hasInfo = false;
  149.     public boolean isLinked = false;
  150.     public boolean isDownloaded = false;
  151.  
  152.     public Game(String name) {
  153.         this.name = name.trim();
  154.     }
  155.  
  156.     public Game parseURL(String url) {
  157.         try {
  158.             this.url = url;
  159.             String[] strs = this.url.replace("http://coolrom.com/roms/", "").split("/");
  160.             this.console = strs[0].toUpperCase();
  161.             this.id = Integer.parseInt(strs[1]);
  162.             this.downloadPage = "http://coolrom.com/dlpop.php?id=" + id;
  163.         }
  164.         catch (Exception e) {}
  165.         return this;
  166.     }
  167.  
  168.     public void findLink() {
  169.         try {
  170.             Document doc = Jsoup.connect(downloadPage).get();
  171.             Matcher match = PATTERN.matcher(doc.data());
  172.             match.find();
  173.             downloadLink = match.group(1);
  174.             isLinked = true;
  175.         }
  176.         catch (Exception e) {}
  177.     }
  178.  
  179.     public void getInfo() {
  180.         int fails = 0;
  181.         while (contentType.equals("text/html") && fails++ < 5) {
  182.             try {
  183.                 URL url = new URL(downloadLink);
  184.                 HttpURLConnection conn = (HttpURLConnection) url.openConnection();
  185.                 conn.addRequestProperty("REFERER", downloadPage);
  186.                 contentType = conn.getContentType();
  187.                 contentLength = conn.getContentLengthLong();
  188.                 conn.disconnect();
  189.                 hasInfo = true;
  190.             }
  191.             catch (Exception e) {}
  192.         }
  193.         System.out.print(fails >= 5 ? "FAILED: " : "");
  194.         System.out.println(contentType + ": " + name + " - " + contentLength);
  195.     }
  196.  
  197.     // Need to find out what file names are safe for each file system.
  198.     public void download() {
  199.         OutputStream out = null;
  200.         HttpURLConnection conn = null;
  201.         InputStream in = null;
  202.         try {
  203.             URL url = new URL(downloadLink);
  204.             out = new BufferedOutputStream(new FileOutputStream(filePath));
  205.             conn = (HttpURLConnection) url.openConnection();
  206.             conn.addRequestProperty("REFERER", downloadPage);
  207.             in = conn.getInputStream();
  208.             byte[] buffer = new byte[1024];
  209.             int numRead;
  210.             while ((numRead = in.read(buffer)) != -1) {
  211.                 out.write(buffer, 0, numRead);
  212.             }
  213.             conn.disconnect();
  214.         }
  215.         catch (Exception e) {
  216.             e.printStackTrace();
  217.         }
  218.         finally {
  219.             try {
  220.                 if (in != null)
  221.                     in.close();
  222.                 if (out != null)
  223.                     out.close();
  224.             }
  225.             catch (Exception ioe) {}
  226.         }
  227.         isDownloaded = true;
  228.     }
  229.  
  230.     @Override
  231.     public int hashCode() {
  232.         return id;
  233.     }
  234.  
  235.     @Override
  236.     public String toString() {
  237.         return id + ": " + console + ": " + name + " - " + url;
  238.     }
  239.  
  240.     @Override
  241.     public int compareTo(Game o) {
  242.         return Integer.compare(this.id, o.id);
  243.     }
  244. }
Advertisement
Comments
  • User was banned
Add Comment
Please, Sign In to add comment