Advertisement
Guest User

Untitled

a guest
Nov 8th, 2022
121
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Lua 5.17 KB | Source Code | 0 0
  1. -- To save the output of this script to a file, run it like this: "lua mageloscrape.lua > outputfile.txt"
  2.  
  3. --local http = require("socket.http");
  4. local https = require("ssl.https");
  5. local ltn12 = require("ltn12");
  6. local DEFLATE = require 'compress.deflatelua';
  7. --[[
  8. -- if this table exists then all items are ignored except those in this table
  9. local ITEMS = {
  10.     ["Black Marble"] = true,
  11.     ["Chipped Onyx Sapphire"] = true,
  12.     ["Crushed Black Marble"] = true,
  13.     ["Crushed Coral"] = true,
  14.     ["Crushed Flame Emerald"] = true,
  15.     ["Crushed Flame Opal"] = true,
  16.     ["Crushed Jaundice Gem"] = true,
  17.     ["Crushed Lava Ruby"] = true,
  18.     ["Crushed Onyx Sapphire"] = true,
  19.     ["Crushed Opal"] = true,
  20.     ["Crushed Topaz"] = true,
  21.     ["Flawed Emerald"] = true,
  22.     ["Flawed Sea Sapphire"] = true,
  23.     ["Flawed Topaz"] = true,
  24.     ["Flawless Diamond"] = true,
  25.     ["Jaundice Gem"] = true,
  26.     ["Nephrite"] = true,
  27.     ["Pristine Emerald"] = true,
  28.     ["Chipped Black Marble"] = true,
  29.     ["Coral Crescent"] = true,
  30.     ["Flawed Chrysolite"] = true,
  31.     ["Flawed Flame Emerald"] = true,
  32.     ["Flawed Flame Opal"] = true,
  33.     ["Flawed Opal"] = true,
  34.     ["Flawed Lava Ruby"] = true,
  35.     ["Imperfect Diamond"] = true,
  36.     ["Crushed Chrysolite"] = true,
  37.     ["Crushed Diamond Dust"] = true,
  38.     ["Crushed Emerald"] = true,
  39.     ["Crushed Nephrite"] = true,
  40.     ["Crushed Sea Sapphire"] = true,
  41. };
  42. ]]
  43. local MAX_ITEM_ID = 33000; -- ignore items with an item ID above this number.  Useful if you want to filter out items from newer eras
  44. -- the script will scrape all the NPC URLs in this table
  45. local urls = {
  46.     "https://eq.magelo.com/npc/92944/a-fetid-fiend",
  47.     "https://eq.magelo.com/npc/92952",
  48.     "https://eq.magelo.com/npc/95180",
  49. };
  50.  
  51. local t = {};
  52.  
  53. if ( not https ) then
  54.     print("failed to load http lib");
  55.     os.exit();
  56. end
  57. if ( not ltn12 ) then
  58.     print("failed to load ltn12 lib");
  59.     os.exit();
  60. end
  61. if ( not DEFLATE ) then
  62.     print("failed to load deflate lib");
  63.     os.exit();
  64. end
  65.  
  66. local fname;
  67.  
  68. for urlIndex, url in ipairs(urls) do
  69.  
  70.     fname = 'mageloscrape'..urlIndex..'.txt';
  71.    
  72.     t = {};
  73.  
  74.     local success, code, headers, status = https.request {
  75.         url = url,
  76.         headers = {
  77.             --["Accept"] = "text/*, text/html",
  78.             --["Accept-Encoding"] = "gzip;q=0,deflate;q=0",
  79.             --["Accept-Encoding"] = "",
  80.         },
  81.         sink = ltn12.sink.table(t);
  82.     };
  83.     if ( not success ) then
  84.         print("failed to get web page; code == ", code);
  85.         os.exit();
  86.     end
  87.    
  88.     --[[
  89.     print("code ==", code);
  90.     print("status ==", status);
  91.     if ( headers ) then
  92.         print(headers.date);
  93.         print(headers.server);
  94.         print(headers["last-modified"]);
  95.         print(headers["content-length"]);
  96.         print(headers["connection"]);
  97.         print(headers["content-Type"]);
  98.         print("#t", #t);
  99.     end
  100.     --print(t[1]);
  101.     --print(t[2]);
  102.     ]]
  103.    
  104.     --local ofh = io.open(fname, 'w+b');
  105.     local combinedText = table.concat(t);
  106.     --ofh:write(combinedText);
  107.     --ofh:close();
  108.    
  109. --  local compressedPage = table.concat(t);
  110. --  DEFLATE.gunzip {input=compressedPage, output=ofh};
  111.  
  112.     --local page = io.open("venril.html", "r"):read("*a");
  113. --  ofh:seek("set");
  114. --  local page = ofh:read("*a");
  115. --  local page = io.open(fname, "r"):read("*a");
  116.  
  117.     local page = combinedText;
  118.     --local compressedPage = "";
  119.    
  120.     local pos = page:find("var data=(function(){return[", 20000, true);
  121.     --[[
  122.     if ( not pos ) then
  123.         --print("length == ", #compressedPage);
  124.         print("length == ", #page);
  125.         print("code == ", code);
  126.         --print(unpack(headers));
  127.         for k, v in pairs(headers) do
  128.             print(k, v);
  129.         end
  130.         print("stauts == ", status);
  131.         --print(compressedPage);
  132.     end
  133.     ]]
  134.  
  135.     local _, _, npc = page:find("<title>NPC : (.*) %- EQ %- Magelo</title>");
  136.     local _, _, level = page:find("Level : </span>(%S+)");
  137.     local items;
  138.     if ( not pos ) then
  139.         print("no items for NPC "..npc.." "..level);
  140.         print("");
  141.     else
  142.         items = page:sub(pos + 27, page:find("]]]", pos + 27, true) + 2);
  143.         items = items:gsub("%[", "{");
  144.         items = items:gsub("%]", "}");
  145.  
  146.         --print(items);
  147.  
  148.         itemsTable = assert(loadstring("return "..items))();
  149.  
  150.         local itemId, itemUrl, itemName, drops, kills, header;
  151.        
  152.         table.sort(itemsTable, function(a,b) return a[1] < b[1]; end );
  153.  
  154.         for index, item in ipairs(itemsTable) do
  155.             itemId = item[1];
  156.             itemUrl = item[2];
  157.             itemName = item[3][1];
  158.             --unknown = item[3][2];
  159.             drops = item[8][3];
  160.             kills = item[8][4];
  161.            
  162.             --print(itemId, itemUrl, itemName, drops, kills);
  163.             if ( not header ) then
  164.                 print(npc.." "..level, kills);
  165.                 header = true;
  166.             end
  167.             if ( (not ITEMS and itemId < MAX_ITEM_ID) or (ITEMS and ITEMS[itemName]) ) then
  168.                 --print(itemName, drops, itemId);
  169.                 --print(itemName, drops);
  170.                 print(itemName, itemId, drops, drops/kills);
  171.             end
  172.         end
  173.         --[[
  174.         if ( not ITEMS and itemId > 40000 ) then
  175.             for index, item in ipairs(itemsTable) do
  176.                 itemId = item[1];
  177.                 itemUrl = item[2];
  178.                 itemName = item[3][1];
  179.                 --unknown = item[3][2];
  180.                 drops = item[8][3];
  181.                 kills = item[8][4];
  182.                
  183.                 --print(itemId, itemUrl, itemName, drops, kills);
  184.                 if ( not header ) then
  185.                     print(npc, kills);
  186.                     header = true;
  187.                 end
  188.                 print(itemName, drops, itemId);
  189.             end
  190.         end
  191.         ]]
  192.         --print(unpack(i));
  193.         print("");
  194.         --ofh:close();
  195.         --os.remove('mageloscrape'..urlIndex..'.txt');
  196.     end
  197. end
  198.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement