Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import java.io.BufferedOutputStream;
- import java.io.File;
- import java.io.FileOutputStream;
- import java.io.InputStream;
- import java.io.OutputStream;
- import java.io.OutputStreamWriter;
- import java.io.Serializable;
- import java.lang.reflect.Type;
- import java.net.HttpURLConnection;
- import java.net.URL;
- import java.util.ArrayList;
- import java.util.Arrays;
- import java.util.HashMap;
- import java.util.List;
- import java.util.Map;
- import java.util.regex.Matcher;
- import java.util.regex.Pattern;
- import org.jsoup.Jsoup;
- import org.jsoup.nodes.Document;
- import org.jsoup.nodes.Element;
- import org.jsoup.select.Elements;
- import com.google.gson.Gson;
- import com.google.gson.GsonBuilder;
- import com.google.gson.reflect.TypeToken;
- /**
- * This program requires Java 8 since it is using the new Lambda expressions and Streams.
- * @author Brian Wiegand
- * @since Java 8
- */
- public class CoolROMDownloader {
- public static final File currentDir = new File(".");
- /**
- * The list of the acceptable consoles.
- * You can add/remove as many as you like, but at least one is required.
- */
- // public static final List<String> CONSOLES = Arrays.asList("cps1", "cps2");
- public static final List<String> CONSOLES = Arrays.asList("atari2600", "atari5200", "atari7800", "atarijaguar", "atarilynx",
- "c64", "cps1", "cps2", "gba", "gbc", "mame", "namcosystem22", "neogeo", "neogeocd", "neogeopocket", "nes", "n64",
- "nds", "gcn", "segacd", "dc", "gamegear", "genesis", "mastersystem", "model2", "saturn", "psx", "ps2", "snes");
- /**
- * The list of the acceptable pages.
- * You can add/remove as many as you like, but at least one is required.
- */
- public static final List<String> LETTERS =
- Arrays.asList("0", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
- "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
- /**
- * The list of the acceptable regions.
- * You can add/remove as many as you like, but at least one is required.
- */
- public static final List<String> REGIONS = Arrays.asList("USA");
- /**
- * A thread-safe list we use to hold all the Game objects we create.
- */
- public static Map<String, ArrayList<Game>> map = new HashMap<String, ArrayList<Game>>();
- /**
- * Main program entry point.
- */
- public static void main(String[] args) {
- for (String string : CONSOLES) {
- map.putIfAbsent(string, new ArrayList<Game>());
- }
- // Brand new Streams and Lambda expressions with parallelism built-in.
- CONSOLES.stream().parallel().forEach(console -> {
- LETTERS.stream().parallel().forEach(letter -> {
- // This isn't needed unless you're downloading for multiple regions.
- // REGIONS.stream().parallel().forEach(region -> {
- // tryIt(console, letter, region);
- // });
- tryIt(console, letter, "USA");
- });
- });
- // Quickly sort the list, and gather info.
- for (ArrayList<Game> list : map.values()) {
- list.sort((o1, o2) -> Integer.compare(o1.id, o2.id));
- list.stream().parallel().forEach(g -> {
- g.findLink();
- g.getInfo();
- });
- }
- // Save the list as JSON.
- Type mapType = new TypeToken<Map<String, ArrayList<Game>>>() {}.getType();
- GsonBuilder builder = new GsonBuilder();
- builder.setPrettyPrinting();
- Gson gson = builder.create();
- String out = gson.toJson(map, mapType);
- File dir = new File(currentDir, "json");
- File myFile = new File(dir, "out.json");
- try {
- myFile.createNewFile();
- FileOutputStream fOut = new FileOutputStream(myFile);
- OutputStreamWriter myOutWriter = new OutputStreamWriter(fOut);
- myOutWriter.append(out);
- myOutWriter.close();
- fOut.close();
- }
- catch (Exception e) {}
- }
- /**
- * Take the console, letter, and region: Try and connect to that page, if so then proceed to find each game
- * in parallel and {@code Main::processLink} on each of the results.
- */
- public static void tryIt(String console, String letter, String region) {
- try {
- // Get the HTML of the page.
- Document doc = Jsoup.connect("http://coolrom.com/roms/" + console + "/" + letter + "/").get();
- // Region is important since it's the easiest way to quickly find each game on the page.
- doc.getElementsByClass(region).parallelStream().forEach(CoolROMDownloader::processLink);
- }
- catch (Exception e) {}
- }
- /**
- * Basically this will create and add a Game object to the thread-safe list.
- */
- public static void processLink(Element region) {
- // Find the game by the fact that it is the only link inside the Region.
- Elements gameElement = region.getElementsByAttribute("href");
- // Create the Game Object whereby text() is the game's name, and attr("href") is the page link.
- Game game = new Game(gameElement.text()).parseURL("http://coolrom.com" + gameElement.attr("href"));
- // Add the game to the thread-safe list.
- ArrayList<Game> list = map.get(game);
- synchronized (list) {
- list.add(game);
- }
- }
- }
- class Game implements Comparable<Game>, Serializable {
- public static final long serialVersionUID = -2001848694019069568L;
- public static final Pattern PATTERN = Pattern.compile("action=\\\"(.*?)\\\"");
- public final String name;
- public int id;
- public String url;
- public String console;
- public String filePath;
- public String downloadPage;
- public transient String downloadLink;
- public String contentType = "text/html";
- public long contentLength;
- public boolean hasInfo = false;
- public boolean isLinked = false;
- public boolean isDownloaded = false;
- public Game(String name) {
- this.name = name.trim();
- }
- public Game parseURL(String url) {
- try {
- this.url = url;
- String[] strs = this.url.replace("http://coolrom.com/roms/", "").split("/");
- this.console = strs[0].toUpperCase();
- this.id = Integer.parseInt(strs[1]);
- this.downloadPage = "http://coolrom.com/dlpop.php?id=" + id;
- }
- catch (Exception e) {}
- return this;
- }
- public void findLink() {
- try {
- Document doc = Jsoup.connect(downloadPage).get();
- Matcher match = PATTERN.matcher(doc.data());
- match.find();
- downloadLink = match.group(1);
- isLinked = true;
- }
- catch (Exception e) {}
- }
- public void getInfo() {
- int fails = 0;
- while (contentType.equals("text/html") && fails++ < 5) {
- try {
- URL url = new URL(downloadLink);
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- conn.addRequestProperty("REFERER", downloadPage);
- contentType = conn.getContentType();
- contentLength = conn.getContentLengthLong();
- conn.disconnect();
- hasInfo = true;
- }
- catch (Exception e) {}
- }
- System.out.print(fails >= 5 ? "FAILED: " : "");
- System.out.println(contentType + ": " + name + " - " + contentLength);
- }
- // Need to find out what file names are safe for each file system.
- public void download() {
- OutputStream out = null;
- HttpURLConnection conn = null;
- InputStream in = null;
- try {
- URL url = new URL(downloadLink);
- out = new BufferedOutputStream(new FileOutputStream(filePath));
- conn = (HttpURLConnection) url.openConnection();
- conn.addRequestProperty("REFERER", downloadPage);
- in = conn.getInputStream();
- byte[] buffer = new byte[1024];
- int numRead;
- while ((numRead = in.read(buffer)) != -1) {
- out.write(buffer, 0, numRead);
- }
- conn.disconnect();
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- finally {
- try {
- if (in != null)
- in.close();
- if (out != null)
- out.close();
- }
- catch (Exception ioe) {}
- }
- isDownloaded = true;
- }
- @Override
- public int hashCode() {
- return id;
- }
- @Override
- public String toString() {
- return id + ": " + console + ": " + name + " - " + url;
- }
- @Override
- public int compareTo(Game o) {
- return Integer.compare(this.id, o.id);
- }
- }
Advertisement