WebScraperSearch.java

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
import java.io.PrintWriter;

public class SearchAutomated{

    // used to change the link for dynamic results
    public static double[] changeLink(double first, double second){

        double temp[] = new double[2];

        first = first + 0.2 ; // increment to 2
        second = second +  0.1; // increment to 1

    first = Math.round(first * 100) / 100.0;
    second = Math.round(second * 100) / 100.0;

        temp[0] = first;
        temp[1] = second;

        return temp;

    }

    public static void innerLoop(int trec){

        try{

            double k, b;
            k = 0.0;
            b = 0.0;

            for(int j=0; j <10; j++){

                double tempArray[] = changeLink(k, b);

                k = tempArray[0];
                b = tempArray[1];

                Document doc = Jsoup.connect("http://136.206.115.117:8080/IRModelGenerator/res." + trec +".BM25."+ k + "." + b).timeout(60000).maxBodySize(10*1024*1024).get();
                PrintWriter out = new PrintWriter("TREC_"+ trec +"_k_" + k + "b_" + b +"_results.csv"); //problem
                String article = doc.toString();

                int currentLine = 301;
                int innerCounter = 0;

                String stringHolder = article.substring(32, 35);
                //System.out.println(stringHolder);
                int checker = Integer.parseInt(stringHolder);
                //System.out.println(checker);


                for(int i=0; i<article.length(); i++){

                    if( innerCounter == 1000){

                        currentLine++;
                        innerCounter = 0;
                    }

                    String stringLine = Integer.toString(currentLine);

                    if(i != 0){
                        stringHolder = article.substring(0, 3);
                    }

                    checker = Integer.parseInt(stringHolder);

                    if(i == 0){
                        article = article.substring(article.indexOf(stringLine) + 0);
                    }

                    String subArticle = article.substring(0, article.indexOf("BM25." + k + "." + b) - 1); /////
                    //System.out.println(subArticle);
                    out.println(subArticle);
                    out.flush();

                    innerCounter++;

                    // this brings us onto the next line
                    article = article.substring(article.indexOf("BM25." + k + "." + b)+ 13);////

                    System.out.println("Inner Loop done:" + i);
                }

                out.close();

                System.out.println("OUTER Loop done:" + j);
            }

        }

        catch(IOException e)
        {
            System.out.println("I could not connect");
        }

        System.out.println("Program has finished running");
    }

    public static void main(String[]args){


        for(int trec=6; trec<9; trec++){

            System.out.println("I am working 1");
            innerLoop(trec);
            System.out.println("I am working 2");
        }

    }
}