Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- "use strict";
- /**
- * `FTP` client.
- *
- * @module
- */
- var fs = require("fs");
- var path = require("path");
- var _ = require("lodash");
- var Client = require("ftp");
- var mkdirp = require("mkdirp");
- var logger = require("./logger");
- var Pool = require("./pool");
- /**
- * `FTP` client.
- *
- * @class
- * @arg {object} opts - Client options.
- * @arg {string} [opts.folder] - Path to folder on FTP.
- * @arg {string} [opts.host] - FTP host.
- * @arg {number} [opts.port=21] - FTP port.
- * @arg {string} [opts.username] - FTP user name.
- * @arg {string} [opts.password] - FTP user password.
- * @arg {boolean} [opts.secure=true] - Flag whether FTP connection is secure.
- * @arg {number} [opts.numberOfThreads=1] - Threads number to download files
- * from FTP server.
- */
- var Ftp = module.exports = function (opts) {
- this.client = null;
- this.host = opts.host;
- this.port = opts.port || 21;
- this.username = opts.username;
- this.password = opts.password;
- this.root = opts.folder;
- this.numberOfThreads = opts.numberOfThreads || 1;
- this.secure = _.isUndefined(opts.secure) ? true : opts.secure;
- };
- /**
- * Connects to FTP server.
- *
- * @method
- * @async
- * @return {Promise}
- */
- Ftp.prototype.connect = function () {
- var options = { host: this.host,
- port: this.port,
- user: this.username,
- password: this.password };
- if (this.secure) {
- options.secure = true;
- options.secureOptions = { rejectUnathorized: false };
- };
- this.client = new Client();
- return new Promise((resolve, reject) => {
- this.client.removeAllListeners("error");
- this.client.on("ready", resolve);
- this.client.on("error", reject);
- this.client.on("greeting", msg => {
- logger.info("FTP connection established:", msg);
- });
- this.client.connect(options);
- });
- };
- /**
- * Collects FTP files.
- *
- * @method
- * @async
- * @return {object} - Dictionary of FTP files data.
- */
- Ftp.prototype.collectFiles = async function () => {
- var filesData = {};
- var folders = [""];
- logger.info("Collecting files on FTP server...");
- console.log("Collecting files on FTP server...");
- this.client.removeAllListeners("error");
- this.client.on("error", msg => {
- console.log(msg);
- logger.error("FATAL", msg, () => process.exit(1));
- });
- var collect = folder => {
- return new Promise((resolve, reject) => {
- var remoteFolder = path.join(this.root, folder);
- this.client.cwd(remoteFolder, () => {
- this.client.pwd((err, cwd) => {
- if (err) {
- reject(err);
- return;
- };
- this.client.list(cwd, (err, filesList) => {
- if (err) {
- reject(err);
- return;
- };
- for (var file of filesList) {
- var filePath = file.name;
- if (folder) {
- filePath = folder + path.sep + filePath;
- };
- if (file.type === "d") {
- folders.push(filePath);
- } else {
- filesData[filePath] = { path: filePath,
- size: file.size,
- date: file.date };
- logger.debug("Collect FTP file", filePath);
- process.stdout.write('.');
- };
- };
- resolve();
- });
- });
- });
- });
- };
- while (folders.length) {
- await collect(folders.shift());
- };
- logger.info("FTP collection is finished.");
- console.log("\nFTP collection is finished.");
- return filesData;
- };
- /**
- * Downloads FTP files.
- *
- * @function
- * @async
- * @arg {string[]} filePaths - List of ftp file pathes to download.
- * @arg {object} ftpFiles - FTP collected files data.
- * @arg {string} localRoot - Path to folder to download files.
- * @arg {number} [attemptions=10] - Number of attemptions to download file.
- * @return {Promise}
- */
- Ftp.prototype.downloadFiles = async function (filePaths,
- ftpFiles,
- localRoot,
- attemptions) => {
- attemptions = attemptions || 10;
- var fileFails = {};
- var pool = new Pool(this.numberOfThreads);
- logger.info("FTP files downloading...");
- console.log("FTP files downloading...");
- this.client.removeAllListeners("error");
- this.client.on("error", msg => logger.error(msg));
- var _download = filePath => {
- return () => {
- logger.debug(`Downloading ${filePath} ...`);
- return this.downloadFile(filePath, localRoot).then(() => {
- var idx = filePaths.indexOf(filePath);
- filePaths.splice(idx, 1);
- logger.debug(`Downloaded ${filePath}`);
- process.stdout.write('.');
- }).catch(err => {
- logger.error(`Fail to download ${filePath}`, err);
- fileFails[filePath] = (fileFails[filePath] || 0) + 1;
- if (fileFails[filePath] < attemptions) {
- filePaths.push(filePath);
- var idx = filePaths.indexOf(filePath);
- filePaths.splice(idx, 1);
- logger.debug(`Reschedule to download ${filePath}`);
- pool.add(ftpFiles[filePath].size, _download(filePath));
- };
- });
- };
- };
- for (var filePath of filePaths) {
- pool.add(ftpFiles[filePath].size, _download(filePath));
- };
- await new Promise(resolve => {
- var timerId = setInterval(() => {
- if (!filePaths.length) {
- clearInterval(timerId);
- logger.info("FTP files are downloaded.");
- console.log("\nFTP files are downloaded.");
- resolve();
- };
- }, 1000);
- });
- };
- /**
- * Downloads FTP file.
- *
- * @method
- * @arg {string} filePath - Path of file to download from FTP.
- * @arg {string} localRoot - Path to folder to download file.
- * @return {Promise}
- */
- Ftp.prototype.downloadFile = function (filePath, localRoot) => {
- var remotePath = path.join(this.root, filePath)
- var localPath = path.join(localRoot, filePath);
- mkdirp.sync(path.dirname(localPath));
- return new Promise((resolve, reject) => {
- this.client.get(remotePath, (err, remoteStream) => {
- if (err) {
- reject(err);
- return;
- };
- if (!remoteStream) {
- reject("No FTP stream");
- return;
- };
- var localStream = fs.createWriteStream(localPath);
- remoteStream.pipe(localStream);
- var _streamError = err => {
- if (fs.existsSync(localPath)) {
- fs.unlinkSync(localPath);
- };
- reject(err);
- };
- localStream
- .on("finish", () => localStream.close(resolve))
- .on("error", _streamError);
- remoteStream.on("error", _streamError);
- });
- });
- };
Add Comment
Please, Sign In to add comment