Advertisement
jonalu

mcbans scraper source

Jan 20th, 2012
241
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 2.03 KB | None | 0 0
  1.  
  2. import java.io.BufferedReader;
  3. import java.io.IOException;
  4. import java.io.InputStreamReader;
  5. import java.net.HttpURLConnection;
  6. import java.net.MalformedURLException;
  7. import java.net.URL;
  8. import java.util.ArrayList;
  9.  
  10.  
  11. public class scrapertest {
  12.     public static void main(String[] args) throws MalformedURLException, IOException {
  13.         System.out.println("Starting");
  14.        
  15.         BufferedReader stdIn = new BufferedReader(new InputStreamReader(System.in));
  16.         String command;
  17.         int count = 1;
  18.         while((command = stdIn.readLine()) != null) {
  19.             count = Integer.parseInt(command);
  20.             break;
  21.         }
  22.        
  23.         for(int i = 1; i <= count; i++) {
  24.             ArrayList<String> al = getUrlAsInBrowser(new URL("http://www.mcbans.com/list/"+i));
  25.             saveUsernamesOutOfMcBansHTML(al);
  26.             for(String s : usernames)
  27.                 System.out.println(s);
  28.         }
  29.         System.out.println("Exiting");
  30.     }
  31.     private static void saveUsernamesOutOfMcBansHTML(ArrayList<String> html) {
  32.         for(String s : html) {
  33.             if(s.startsWith("<td><a href=\"/player/")) {
  34.                 s = s.replaceFirst("<td><a href=\"/player/", "");
  35.                 s = s.replaceAll("</a></td>", "");
  36.                 int i = s.indexOf("\">");
  37.                 s = s.substring(i+2);
  38.                 if(!s.equals("console") && !s.contains("["))
  39.                     addToUsernames(s);
  40.             }
  41.         }
  42.     }
  43.     private static void addToUsernames(String username) {
  44.         if(!usernames.contains(username))
  45.             usernames.add(username);
  46.     }
  47.     private static ArrayList<String> usernames = new ArrayList<String>();
  48.     private static ArrayList<String> getUrlAsInBrowser(URL url) throws IOException {
  49.         ArrayList<String> al = new ArrayList<String>();
  50.        
  51.         System.setProperty("http.agent", "");
  52.         HttpURLConnection huc = (HttpURLConnection) url.openConnection();
  53.         huc.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2");
  54.         huc.connect();
  55.        
  56.         BufferedReader br = new BufferedReader(new InputStreamReader(huc.getInputStream()));
  57.         String s;
  58.         while((s = br.readLine())!=null)
  59.             al.add(s);
  60.        
  61.         huc.disconnect();
  62.        
  63.         return al;
  64.     }
  65. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement