Advertisement
anonymousvntk

Untitled

Jul 22nd, 2021
19
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 9.45 KB | None | 0 0
  1. const axios = require("axios");
  2. const cheerio = require('cheerio');
  3. const Promise = require('bluebird')
  4. const _ = require('lodash');
  5. var CryptoJS = require("crypto-js");
  6. const fs = require('fs')
  7. const download = require('../../utils/imagedownload/index');
  8. async function DownloadNgonPhong(link) {
  9. let resp = await axios.get(link);
  10. console.log("Get data manga done")
  11. resp = resp.data;
  12. const $ = cheerio.load(resp);
  13. let allChapterLink = [];
  14. let chapterTable = $(`body > section > div.info > div > div > div.m-b-0 > div.table-wrapper.chapter-table > table > tbody > tr`);
  15. for (let i = 1; i <= chapterTable.length; i++) {
  16. let currentChapter = $(`body > section > div.info > div > div > div.m-b-0 > div.table-wrapper.chapter-table > table > tbody > tr:nth-child(${i}) > td:nth-child(1) > a > span.hidden-sm.hidden-xs`);
  17. let currentChapterName = currentChapter.html();
  18. if (currentChapterName) {
  19. currentChapterName = currentChapterName.split(" ");
  20. currentChapterName = currentChapterName[currentChapterName.length -1]
  21. let currentChapterNumber = parseFloat(currentChapterName);
  22. let selectorChapterLink = $(`body > section > div.info > div > div > div.m-b-0 > div.table-wrapper.chapter-table > table > tbody > tr:nth-child(${i}) > td:nth-child(1) > a`);
  23. let currentChapterHref = selectorChapterLink.attr('href')
  24. allChapterLink.push({
  25. currentChapterName, currentChapterNumber, currentChapterHref
  26. })
  27. }
  28. }
  29. allChapterLink.reverse();
  30. console.log(`Có tổng cộng ${allChapterLink.length} chapter`)
  31. let mangaName = link.split("/");
  32. mangaName = mangaName[mangaName.length - 2];
  33. let outDir = `output/ngonphong-${mangaName}/`;
  34. console.log("Tên folder chứa ảnh: ", outDir)
  35. if (!fs.existsSync(outDir)) {
  36. fs.mkdirSync(outDir);
  37. }
  38. let chapterImageLinkFile = outDir + "images.txt";
  39. if (!fs.existsSync(chapterImageLinkFile)) {
  40. fs.writeFileSync(chapterImageLinkFile, "")
  41. }
  42. let imageLinksFile = fs.readFileSync(outDir + "images.txt");
  43. imageLinksFile = imageLinksFile.toString();
  44. let chapterToGetImages = [];
  45. if (imageLinksFile.length !== 0) {
  46. let alreadyHaveImageLinkChap = JSON.parse(imageLinksFile);
  47. //Check xem trong file có chapter nào rồi thì bỏ qua chapter đó
  48. // Chỉ get link ảnh ở những chapter chưa có link
  49. allChapterLink.map(each => {
  50. let isDownloaded = false;
  51. alreadyHaveImageLinkChap.map(ae => {
  52. if (each.currentChapterNumber === ae.chapter) {
  53. isDownloaded = true
  54. }
  55. })
  56. if (isDownloaded === false) {
  57. chapterToGetImages.push(each)
  58. }
  59. })
  60. } else {
  61. chapterToGetImages = allChapterLink;
  62. }
  63.  
  64. console.log(`Có ${chapterToGetImages.length} chapter chưa lấy ảnh`);
  65. let allChapterChunk = _.chunk(chapterToGetImages, 3);
  66. let allImageInEachChapter;
  67. if (imageLinksFile.length !== 0) {
  68. let alreadyHaveImageLinkChap = JSON.parse(imageLinksFile);
  69. allImageInEachChapter = alreadyHaveImageLinkChap;
  70. } else {
  71. allImageInEachChapter = []
  72. }
  73.  
  74. let chapterIndex = 1;
  75. await Promise.each(allChapterChunk, async (eachChapterChunk, index) => {
  76. console.log(`Current chunk: ${index}/${allChapterChunk.length} - ${eachChapterChunk.length} items`)
  77. let downloadChapterRequest = [];
  78. eachChapterChunk.map(each => {
  79. downloadChapterRequest.push(axios.get(each.currentChapterHref))
  80. })
  81. let downloadChapterResult = await Promise.all(downloadChapterRequest);
  82. downloadChapterResult = downloadChapterResult.map(each => {
  83. return each.data
  84. })
  85.  
  86. console.log(`Downloaded chunk: ${index}/${allChapterChunk.length}`)
  87. downloadChapterResult.map(async (each, cIndex) => {
  88. console.log(`Downloading Chapter ${chapterIndex}/${chapterToGetImages.length}`)
  89. let chapterDetail = cheerio.load(each);
  90. let chapterContentSelector = chapterDetail('#view-chapter')
  91. let chapterContentText = chapterContentSelector.html();
  92. let htmlContent = chapterContentText.match(/var htmlContent=(.*);/);
  93. let jsonContent = JSON.parse(JSON.parse(htmlContent[1]));
  94. const passphrase = '@9jriuQ^@'+'~4ZoLils*'+'^u$UxZ!Qzy<ytt_Z2}'
  95. var encrypted = jsonContent.ciphertext;
  96. var salt = CryptoJS.enc.Hex.parse(jsonContent.salt);
  97. var iv = CryptoJS.enc.Hex.parse(jsonContent.iv);
  98. var key = CryptoJS.PBKDF2(passphrase, salt, {
  99. hasher: CryptoJS.algo.SHA512,
  100. keySize: 64 / 8,
  101. iterations: 999
  102. });
  103. var decrypted = CryptoJS.AES.decrypt(encrypted, key, {
  104. iv: iv
  105. });
  106. let decryptedStr = decrypted.toString(CryptoJS.enc.Utf8);
  107. decryptedStr = decryptedStr.replace(/@9jriuQ\^@/g, '.');
  108. decryptedStr = decryptedStr.replace(/~4ZoLils\*/g, ':');
  109. decryptedStr = decryptedStr.replace(/\^u\$UxZ!Qzy<ytt_Z2}/g, '/');
  110. let chapterHtml = cheerio.load(decryptedStr);
  111. let chapterImgs = []
  112. chapterHtml('img').map((index, el) => {
  113. let imageLink = chapterHtml(el).attr('data-9jriuq');
  114. chapterImgs.push(imageLink)
  115. });
  116. allImageInEachChapter.push({
  117. chapter: eachChapterChunk[cIndex].currentChapterNumber,
  118. images: chapterImgs
  119. })
  120. fs.writeFileSync(chapterImageLinkFile, JSON.stringify(allImageInEachChapter))
  121. chapterIndex++;
  122. })
  123. })
  124. console.log("Đã lấy xong link ảnh")
  125. // Chia ảnh thành 2 loại tải nhanh và tải chậm
  126.  
  127. await Promise.each(allImageInEachChapter, async eachChapter => {
  128. console.log("Bắt đầu tải ảnh chapter ", eachChapter.chapter)
  129. let tempOutDir = outDir + eachChapter.chapter;
  130. if (!fs.existsSync(tempOutDir)) {
  131. fs.mkdirSync(tempOutDir);
  132. }
  133. // Check ảnh, nếu ảnh tải nhanh được thì tải 1 phát hết
  134. // Nếu ảnh ko tải nhanh được thì tải từng cái một
  135. let fastDownloadUrls = [];
  136. let slowDownloadUrls = [];
  137. for (let i = 0; i < eachChapter.images.length; i++) {
  138. let fileDest = `${tempOutDir}/${i}.jpg`;
  139. if (!fs.existsSync(fileDest)) {
  140.  
  141. let headers = {
  142. 'Connection': 'keep-alive',
  143. 'Cache-Control': 'max-age=0',
  144. 'Upgrade-Insecure-Requests': '1',
  145. 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36',
  146. 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
  147. 'Accept-Language': 'en-GB,en;q=0.9,en-US;q=0.8,vi;q=0.7'
  148. }
  149. if (eachChapter.images[i].includes("proxy/view.php?url=")) {
  150. eachChapter.images[i] = eachChapter.images[i].split("proxy/view.php?url=")[1];
  151. } else {
  152. headers["referer"] = "https://www.ocumeo.com/"
  153. }
  154. const options = {
  155. url: eachChapter.images[i],
  156. dest: `${tempOutDir}/${i}.jpg`,
  157. headers: headers
  158. }
  159. if (options.url.includes("blogspot.com")) {
  160. fastDownloadUrls.push(options)
  161. } else {
  162. slowDownloadUrls.push(options)
  163. }
  164. }
  165. }
  166. if (fastDownloadUrls.length > 0) {
  167. // Với ảnh tải nhanh được thì tải song song
  168. console.log("Đang tải ảnh có tốc độ cao")
  169. fastDownloadUrls = _.chunk(fastDownloadUrls, 10); // Ảnh tải được liên tục thì tải 1 lúc 10 cái
  170. await Promise.each(fastDownloadUrls, async (eachChunk, index) => {
  171. console.log(`Đang tải ảnh tốc độ cao chunk ${index}/${fastDownloadUrls.length}`)
  172. let downloadRequest = [];
  173. eachChunk.map(e => {
  174. downloadRequest.push(download.image(e))
  175. })
  176. await Promise.all(downloadRequest)
  177. })
  178. }
  179. if (slowDownloadUrls.length > 0) {
  180. console.log("Đang tải những ảnh có tốc độ chậm")
  181. slowDownloadUrls = _.chunk(slowDownloadUrls, 2); // Ảnh chậm thì tải 1 lúc 2 cái
  182. await Promise.each(slowDownloadUrls, async (eachChunk, index) => {
  183. console.log(`Đang tải ảnh có tốc độ chậm chunk ${index}/${slowDownloadUrls.length} - Chapter ${eachChapter.chapter}`);
  184. let downloadRequest = [];
  185. eachChunk.map(e => {
  186. downloadRequest.push(download.image(e))
  187. })
  188. await Promise.all(downloadRequest)
  189. })
  190. }
  191. })
  192. console.log("Đã tải xong toàn bộ ảnh")
  193. return allImageInEachChapter
  194. }
  195.  
  196. module.exports = {
  197. DownloadNgonPhong
  198. }
  199.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement