rafaltonie

package rafaltonie;


import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

public class Downloader {
    private final static Logger log = Logger.getLogger(Downloader.class);

    private static String usernamePassword = "";
    private static String mainUrl = "";
    private static String mainPage = "";
    private static String desktopVideo = "C:\\Users\\...\\Desktop\\video2\\";

    public static void main(String[] args) throws IOException, InterruptedException {
        download();

    }

    private static void download() throws IOException, InterruptedException {
        List<AdressAndImage> subPages = getSubPages();

        int waitingCount = 1;
        for(int i = 14; i < subPages.size() ; i++){
            log.info(i);
            try {
                String subPage = subPages.get(i).adress;
                int folderNumber = i+1;
                String subfolderBaseName = subFolderName(subPage);

                String subFolderName = folderNumber + "_"+ subfolderBaseName + "\\";
                log.info(subPages.get(i).image);
                if(subPages.get(i).image != null){
                    downloadFile(subFolderName, subPages.get(i).image);
                }
                if(subPage.startsWith("movie")){
                    log.info(subFolderName);
                    List<String> subSubPages = getSubSubPages(subPage);
                    TimeUnit.SECONDS.sleep(0);
                    waitingCount = 1;
                    for(String subSubPage: subSubPages){
                        log.info((folderNumber) + "_____" + subSubPage);
                        TimeUnit.SECONDS.sleep(0);
                        String fileAdress = getFileAdress(subSubPage);
                        downloadFile(subFolderName, fileAdress);
                    }
                }else{
                    log.info(subFolderName);
                    downloadFile(subFolderName, subPage);
                }

            } catch (IOException e) {
                log.error("###############################################################");
                log.error("###############################################################");
                log.error("last i = " + i );
                log.error("waiting " + waitingCount++);
                TimeUnit.SECONDS.sleep(30);
                i--;
            }
        }
    }

    private static boolean isSubfolderExists(String subfolderBaseName) throws IOException {
        List<Path> files = Files.walk(Paths.get(desktopVideo)).collect(Collectors.toList());
        for(Path f:files){
            if(f.toString().contains(subfolderBaseName)){
                return true;
            }
        }
        return false;
    }

    private static List<AdressAndImage> getSubPages() throws IOException {
        Document doc = getDocumentFromPage(mainUrl + mainPage);
        Elements elements = doc.select("a[href$=.php]").select("a[href^=movie], a[href^=video]");
        List<AdressAndImage> subPages = new ArrayList<>();
        for(Element e:elements){
            AdressAndImage adressAndImage = new AdressAndImage();
            adressAndImage.adress = e.attr("href");
            if(e.children().size() > 0 ){
                adressAndImage.image = e.child(0).attr("src");
            }
            subPages.add(adressAndImage);
        }
        return subPages;
    }

    private static List<String> getSubSubPages(String subSiteName) throws IOException {
        Document doc = getDocumentFromPage(mainUrl+subSiteName);
        Elements elements = doc.select("a[href$=.php]").select("a[href^=video]");
        List<String> subPages = new ArrayList<>();
        for(Element e:elements){
            if(e.elementSiblingIndex() == 0 && e.getElementsByTag("img").size() > 0){
                subPages.add(e.attr("href"));
            }
        }
        return subPages;
    }

    /**
     * @return true if downloaded, false if already exits
     */
    private static boolean downloadFile(String subFolderName, String fileAdress) throws IOException {
        String fileUri = mainUrl + fileAdress;
        fileAdress = fileAdress.replaceFirst("video", subFolderName.split("_")[0]);
        if(fileAdress.contains("/")){//if jpg make filename shorter  and without '/'
            fileAdress = "_"+fileAdress.split("/")[2];
        }

        File targetFile = new File(desktopVideo+subFolderName+fileAdress);
        if(!targetFile.exists()){
            FileUtils.copyInputStreamToFile(getFile(fileUri), targetFile);
            return true;
        }
        return false;
    }

    private static String subFolderName(String subUrl){
        return subUrl.substring(6, subUrl.length() - 4);
    }

    private static InputStream getFile(String uri) throws IOException {
        HttpEntity entity = getPageWithAutorization(uri);
        return entity.getContent();
    }
    private static String getFileAdress(String subSubPage) throws IOException {
        Document doc = getDocumentFromPage(mainUrl+subSubPage);
        Elements elements = doc.select("a[id=player]");
        return elements.get(0).attr("href");
    }

    private static HttpEntity getPageWithAutorization(String uri) throws IOException {
        String encoding = Base64.getEncoder().encodeToString((usernamePassword).getBytes());
        HttpPost httppost = new HttpPost(uri);
        httppost.setHeader("Authorization", "Basic " + encoding);
        log.info("executing request " + httppost.getRequestLine());

        DefaultHttpClient httpclient = new DefaultHttpClient();
        HttpResponse response = httpclient.execute(httppost);
        return response.getEntity();
    }

    private static Document getDocumentFromPage(String pageAdress) throws IOException {
        HttpEntity entity = getPageWithAutorization(pageAdress);
        String result = IOUtils.toString(entity.getContent(), StandardCharsets.UTF_8);
        return Jsoup.parse(result);
    }


    static class AdressAndImage {
        String adress;
        String image;
        @Override
        public String toString() {
            return "AdressAndImage{" +
                    "adress='" + adress + '\'' +
                    ", image='" + image + '\'' +
                    '}';
        }
    }

}