downloadBooks.lua

tArgs = {...}

function startup()
bookData = {["nextChapter"] = tArgs[1]}


periphemu.create("top","speaker")
speaker = peripheral.wrap("top")

programIsRunning = true

neededInfos = {}

strangeStrings = {}

badLines = {}


structureSizeLimit = 7000
infoSizeLimit = 50000
minChapterLen = 5


addNeededInfo("chapterText","<p>","</p>",true)
addNeededInfo("chapterText2","<br>","<br>",true)
addNeededInfo("chapterName","title=\"","\"",false,"<h2><a",">","\"chr%-title\"")
addNeededInfo("nextChapter","href=\"","\"",false,"<a",">","next_chap")
addNeededInfo("bookName","title=\"","\"",false,"<h1><a",">","novel%-link")
addStrangeString("&#x27;","'")
addStrangeString("&gt;",">")
addStrangeString("&amp;","&")
addStrangeString("&nbsp;"," ")
addStrangeString("&lt;","<")
addStrangeString("<em>;","")
addStrangeString("</em>;","")
addBadLine("<img src=")
addBadLine("Error loading comments. Please try again later.")

end



function addStrangeString(strangeString,replacement)
table.insert(strangeStrings, {["string"]=strangeString,["replacement"]=replacement})
end



function getBookData()
website = http.get(bookData["nextChapter"])
    if not website then
        for i = 1,10 do
        print(i)
        sleep(0.5)
        website = http.get(bookData["nextChapter"])
            if website then
            break
            end
        end
    end

    if website then
    websiteCode = website.readAll()
    curChapter = bookData["nextChapter"]

    bookData = {}

        for k,v in pairs(neededInfos) do
            if neededInfos[k]["isTable"] then
            bookData[neededInfos[k]["name"]] = {}
            else
            bookData[neededInfos[k]["name"]] = nil
            end


            if neededInfos[k]["structureStart"] == nil then
            identifyerStart,identifyerEnd = string.find(websiteCode,neededInfos[k]["identifyer"])
                while identifyerStart ~= nil do
                infoStart = identifyerEnd+1
                infoEnd = string.find(websiteCode,neededInfos[k]["ending"],infoStart)
                    if infoEnd == nil then break end
                infoEnd = infoEnd-1

                    if infoEnd-infoStart < infoSizeLimit then
                        if neededInfos[k]["isTable"] then
                        bookData[neededInfos[k]["name"]][table.maxn(bookData[neededInfos[k]["name"]])+1] = string.sub(websiteCode,infoStart,infoEnd)
                        else
                        bookData[neededInfos[k]["name"]] = string.sub(websiteCode,infoStart,infoEnd)
                        end
                    else
                    errorMessage("Info size limit exceeded")
                    return
                    end

                    identifyerStart,identifyerEnd = string.find(websiteCode,neededInfos[k]["identifyer"],infoEnd)
                end
            end




            if neededInfos[k]["structureStart"] ~= nil then
            structureStartStart,structureStartEnd = string.find(websiteCode,neededInfos[k]["structureStart"])
                while structureStartStart ~= nil do
                structureStart = structureStartEnd+1
                structureEnd = string.find(websiteCode,neededInfos[k]["structureEnding"],structureStart)-1

                    if structureEnd-structureStart < structureSizeLimit then
                    structureText = string.sub(websiteCode,structureStart,structureEnd)
                        if neededInfos[k]["alsoInStructure"] == nil or string.find(structureText,neededInfos[k]["alsoInStructure"]) ~= nil then
                        identifyerStart,identifyerEnd = string.find(structureText,neededInfos[k]["identifyer"])
                            if identifyerStart ~= nil then
                            infoStart = identifyerEnd+1
                            infoEnd = string.find(structureText,neededInfos[k]["ending"],infoStart)-1

                                if infoEnd-infoStart < infoSizeLimit then
                                    if neededInfos[k]["isTable"] then
                                    table.insert(bookData[neededInfos[k]["name"]], string.sub(structureText,infoStart,infoEnd))
                                    else
                                    bookData[neededInfos[k]["name"]] = string.sub(structureText,infoStart,infoEnd)
                                    end
                                else
                                errorMessage("Info size limit exceeded")
                                return
                                end

                            end

                        end

                    else
                    errorMessage("Structure size limit exceeded")
                    return
                    end


                structureStartStart,structureStartEnd = string.find(websiteCode,neededInfos[k]["structureStart"],structureEnd)
                end

            end

        end

    else
    errorMessage("Website is nil.")
    end

end



function addNeededInfo(infoName,identifyer,ending,isTable,structureStart,structureEnding,alsoInStructure)
neededInfos[table.maxn(neededInfos)+1] = {}
neededInfos[table.maxn(neededInfos)]["name"] = infoName
neededInfos[table.maxn(neededInfos)]["identifyer"] = identifyer
neededInfos[table.maxn(neededInfos)]["ending"] = ending
neededInfos[table.maxn(neededInfos)]["isTable"] = isTable
neededInfos[table.maxn(neededInfos)]["structureStart"] = structureStart
neededInfos[table.maxn(neededInfos)]["structureEnding"] = structureEnding
neededInfos[table.maxn(neededInfos)]["alsoInStructure"] = alsoInStructure
end



function errorMessage(message)
speaker.playLocalMusic("alarm.MP3")
term.setBackgroundColor(colors.black)
term.setTextColor(colors.red)
print(message)

programIsRunning = false

sleep(4)
end



function saveData()

file = fs.open("books/"..bookData["bookName"]..".txt","a")

file.writeLine(bookData["chapterName"].."\n")
    for k,v in pairs(bookData["chapterText"]) do
    file.writeLine(bookData["chapterText"][k].."\n")
    end

    for k,v in pairs(bookData["chapterText2"]) do
    file.writeLine(bookData["chapterText2"][k].."\n")
    end
file.writeLine("\n\n\n\n\n\n\n\n\n\n\n")

file.flush()
file.close()


file = fs.open("lastChapters/"..bookData["bookName"],"w")
file.write(curChapter)
file.flush()
file.close()

end



function main()
    while programIsRunning do
    getBookData()
        if programIsRunning then
        correctData()
        checkIfDataIsThere()
        saveData()
        term.setTextColor(colors.green)
        print(bookData["chapterName"].."\n")
        end
    end

term.setBackgroundColor(colors.yellow)
term.setTextColor(colors.black)
print("Book done!")
term.setBackgroundColor(colors.black)
term.setTextColor(colors.white)
print("")
speaker.playLocalMusic("ding-sound-effect_1_CVUaI0C.mp3")

end



function checkIfDataIsThere()

    if table.maxn(bookData["chapterText"]) < minChapterLen and table.maxn(bookData["chapterText2"]) < minChapterLen then
    errorMessage("Chapter is too short")
    end

    if bookData["chapterName"] == nil then
    errorMessage("Chapter name is nil")
    end

    if bookData["nextChapter"] == nil then
    errorMessage("Next chapter is nil")
    end

    if bookData["bookName"] == nil then
    errorMessage("Book name is nil")
    end

end



function addBadLine(indicator)
table.insert(badLines, indicator)
end



function correctData()
spaceCount = 0
    for k,v in pairs(bookData["chapterText"]) do
        if string.sub(v,1,1) == " " then
        spaceCount = spaceCount+1
        end
    end



    if spaceCount > table.maxn(bookData["chapterText"])-6 then
        for k,v in pairs(bookData["chapterText"]) do
        bookData["chapterText"][k] = string.sub(v,2,string.len(v))
        end
    end



    for k,v in pairs(bookData["chapterText"]) do
        if string.sub(v,string.len(v),string.len(v)) == "\n" then
        bookData["chapterText"][k] = string.sub(v,1,string.len(v)-1)
        end
    end



    if string.find(bookData["chapterText"][table.maxn(bookData["chapterText"])],"nbsp") ~= nil then
    bookData["chapterText"][table.maxn(bookData["chapterText"])] = nil
    end



    for k,v in pairs(strangeStrings) do
        for k2,v2 in pairs(neededInfos) do
        curBookData = neededInfos[k2]["name"]

            if neededInfos[k2]["isTable"] then
                for k3,v3 in pairs(bookData[curBookData]) do
                strangeStart,strangeEnd = string.find(bookData[curBookData][k3],strangeStrings[k]["string"])
                    while strangeStart do
                        if strangeStart  then
                        bookData[curBookData][k3] = string.sub(bookData[curBookData][k3],1,strangeStart-1) .. strangeStrings[k]["replacement"] .. string.sub(bookData[curBookData][k3],strangeEnd+1,string.len(bookData[curBookData][k3]))
                        end
                    strangeStart,strangeEnd = string.find(bookData[curBookData][k3],strangeStrings[k]["string"])
                    end
                end
            elseif curBookData then
            strangeStart,strangeEnd = string.find(bookData[curBookData],strangeStrings[k]["string"])
                while strangeStart ~= nil do
                    if strangeStart ~= nil then
                    bookData[curBookData] = string.sub(bookData[curBookData],1,strangeStart-1) .. strangeStrings[k]["replacement"] .. string.sub(bookData[curBookData],strangeEnd+1,string.len(bookData[curBookData]))
                    end
                strangeStart,strangeEnd = string.find(bookData[curBookData],strangeStrings[k]["string"])
                end
            end
        end
    end



    for k,v in pairs(badLines) do
        for k2,v2 in pairs(bookData["chapterText"]) do
            if string.find(v2,v) ~= nil then
            table.remove(bookData["chapterText"],k2)
            end
        end

        for k2,v2 in pairs(bookData["chapterText2"]) do
            if string.find(v2,v) ~= nil then
            table.remove(bookData["chapterText2"],k2)
            end
        end
    end

end






startup()
main()