Guest User

Untitled

a guest
Oct 18th, 2017
90
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.43 KB | None | 0 0
  1. "use strict";
  2. /**
  3. * `FTP` client.
  4. *
  5. * @module
  6. */
  7.  
  8. var fs = require("fs");
  9. var path = require("path");
  10.  
  11. var _ = require("lodash");
  12. var Client = require("ftp");
  13. var mkdirp = require("mkdirp");
  14.  
  15. var logger = require("./logger");
  16. var Pool = require("./pool");
  17.  
  18. /**
  19. * `FTP` client.
  20. *
  21. * @class
  22. * @arg {object} opts - Client options.
  23. * @arg {string} [opts.folder] - Path to folder on FTP.
  24. * @arg {string} [opts.host] - FTP host.
  25. * @arg {number} [opts.port=21] - FTP port.
  26. * @arg {string} [opts.username] - FTP user name.
  27. * @arg {string} [opts.password] - FTP user password.
  28. * @arg {boolean} [opts.secure=true] - Flag whether FTP connection is secure.
  29. * @arg {number} [opts.numberOfThreads=1] - Threads number to download files
  30. * from FTP server.
  31. */
  32. var Ftp = module.exports = function (opts) {
  33. this.client = null;
  34. this.host = opts.host;
  35. this.port = opts.port || 21;
  36. this.username = opts.username;
  37. this.password = opts.password;
  38. this.root = opts.folder;
  39. this.numberOfThreads = opts.numberOfThreads || 1;
  40. this.secure = _.isUndefined(opts.secure) ? true : opts.secure;
  41. };
  42. /**
  43. * Connects to FTP server.
  44. *
  45. * @method
  46. * @async
  47. * @return {Promise}
  48. */
  49. Ftp.prototype.connect = function () {
  50. var options = { host: this.host,
  51. port: this.port,
  52. user: this.username,
  53. password: this.password };
  54. if (this.secure) {
  55. options.secure = true;
  56. options.secureOptions = { rejectUnathorized: false };
  57. };
  58. this.client = new Client();
  59.  
  60. return new Promise((resolve, reject) => {
  61. this.client.removeAllListeners("error");
  62. this.client.on("ready", resolve);
  63. this.client.on("error", reject);
  64. this.client.on("greeting", msg => {
  65. logger.info("FTP connection established:", msg);
  66. });
  67. this.client.connect(options);
  68. });
  69. };
  70. /**
  71. * Collects FTP files.
  72. *
  73. * @method
  74. * @async
  75. * @return {object} - Dictionary of FTP files data.
  76. */
  77. Ftp.prototype.collectFiles = async function () => {
  78. var filesData = {};
  79. var folders = [""];
  80.  
  81. logger.info("Collecting files on FTP server...");
  82. console.log("Collecting files on FTP server...");
  83.  
  84. this.client.removeAllListeners("error");
  85. this.client.on("error", msg => {
  86. console.log(msg);
  87. logger.error("FATAL", msg, () => process.exit(1));
  88. });
  89.  
  90. var collect = folder => {
  91.  
  92. return new Promise((resolve, reject) => {
  93. var remoteFolder = path.join(this.root, folder);
  94.  
  95. this.client.cwd(remoteFolder, () => {
  96. this.client.pwd((err, cwd) => {
  97. if (err) {
  98. reject(err);
  99. return;
  100. };
  101.  
  102. this.client.list(cwd, (err, filesList) => {
  103. if (err) {
  104. reject(err);
  105. return;
  106. };
  107.  
  108. for (var file of filesList) {
  109. var filePath = file.name;
  110.  
  111. if (folder) {
  112. filePath = folder + path.sep + filePath;
  113. };
  114.  
  115. if (file.type === "d") {
  116. folders.push(filePath);
  117. } else {
  118. filesData[filePath] = { path: filePath,
  119. size: file.size,
  120. date: file.date };
  121. logger.debug("Collect FTP file", filePath);
  122. process.stdout.write('.');
  123. };
  124. };
  125. resolve();
  126. });
  127. });
  128. });
  129. });
  130. };
  131.  
  132. while (folders.length) {
  133. await collect(folders.shift());
  134. };
  135.  
  136. logger.info("FTP collection is finished.");
  137. console.log("\nFTP collection is finished.");
  138. return filesData;
  139. };
  140. /**
  141. * Downloads FTP files.
  142. *
  143. * @function
  144. * @async
  145. * @arg {string[]} filePaths - List of ftp file pathes to download.
  146. * @arg {object} ftpFiles - FTP collected files data.
  147. * @arg {string} localRoot - Path to folder to download files.
  148. * @arg {number} [attemptions=10] - Number of attemptions to download file.
  149. * @return {Promise}
  150. */
  151. Ftp.prototype.downloadFiles = async function (filePaths,
  152. ftpFiles,
  153. localRoot,
  154. attemptions) => {
  155.  
  156. attemptions = attemptions || 10;
  157. var fileFails = {};
  158. var pool = new Pool(this.numberOfThreads);
  159.  
  160. logger.info("FTP files downloading...");
  161. console.log("FTP files downloading...");
  162.  
  163. this.client.removeAllListeners("error");
  164. this.client.on("error", msg => logger.error(msg));
  165.  
  166. var _download = filePath => {
  167.  
  168. return () => {
  169. logger.debug(`Downloading ${filePath} ...`);
  170.  
  171. return this.downloadFile(filePath, localRoot).then(() => {
  172.  
  173. var idx = filePaths.indexOf(filePath);
  174. filePaths.splice(idx, 1);
  175. logger.debug(`Downloaded ${filePath}`);
  176. process.stdout.write('.');
  177.  
  178. }).catch(err => {
  179.  
  180. logger.error(`Fail to download ${filePath}`, err);
  181. fileFails[filePath] = (fileFails[filePath] || 0) + 1;
  182.  
  183. if (fileFails[filePath] < attemptions) {
  184.  
  185. filePaths.push(filePath);
  186. var idx = filePaths.indexOf(filePath);
  187. filePaths.splice(idx, 1);
  188.  
  189. logger.debug(`Reschedule to download ${filePath}`);
  190. pool.add(ftpFiles[filePath].size, _download(filePath));
  191. };
  192. });
  193. };
  194. };
  195.  
  196. for (var filePath of filePaths) {
  197. pool.add(ftpFiles[filePath].size, _download(filePath));
  198. };
  199.  
  200. await new Promise(resolve => {
  201. var timerId = setInterval(() => {
  202.  
  203. if (!filePaths.length) {
  204. clearInterval(timerId);
  205. logger.info("FTP files are downloaded.");
  206. console.log("\nFTP files are downloaded.");
  207. resolve();
  208. };
  209. }, 1000);
  210. });
  211. };
  212. /**
  213. * Downloads FTP file.
  214. *
  215. * @method
  216. * @arg {string} filePath - Path of file to download from FTP.
  217. * @arg {string} localRoot - Path to folder to download file.
  218. * @return {Promise}
  219. */
  220. Ftp.prototype.downloadFile = function (filePath, localRoot) => {
  221. var remotePath = path.join(this.root, filePath)
  222. var localPath = path.join(localRoot, filePath);
  223. mkdirp.sync(path.dirname(localPath));
  224.  
  225. return new Promise((resolve, reject) => {
  226.  
  227. this.client.get(remotePath, (err, remoteStream) => {
  228.  
  229. if (err) {
  230. reject(err);
  231. return;
  232. };
  233. if (!remoteStream) {
  234. reject("No FTP stream");
  235. return;
  236. };
  237.  
  238. var localStream = fs.createWriteStream(localPath);
  239. remoteStream.pipe(localStream);
  240.  
  241. var _streamError = err => {
  242. if (fs.existsSync(localPath)) {
  243. fs.unlinkSync(localPath);
  244. };
  245. reject(err);
  246. };
  247.  
  248. localStream
  249. .on("finish", () => localStream.close(resolve))
  250. .on("error", _streamError);
  251. remoteStream.on("error", _streamError);
  252. });
  253. });
  254. };
Add Comment
Please, Sign In to add comment