Guest User

Untitled

a guest
Dec 3rd, 2017
94
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 13.94 KB | None | 0 0
  1. import Collection from '../Plex/Collection';
  2. import jetpack from 'fs-jetpack';
  3. import InstagramAccount from './InstagramAccount';
  4. import {chunk, defaults, differenceBy, isString, sortBy, uniqBy} from "lodash";
  5. import configs from "../../config/app";
  6. import moment from "moment";
  7. import type {dbAccauntLog, dbAccountMeta} from "../Plex/type/db.flow";
  8. import AccountLog from "../Plex/AccountLog";
  9. import type {Collection as CollectionMongo} from "../Plex/type/mongo.flow";
  10. import {getArrayOfLinesFromFile} from "../Plex/Helpers";
  11. import Parallel from "../Plex/Parallel";
  12. import ProjectLog from "../Plex/Db/ProjectLog";
  13.  
  14. export class Process {
  15. running: boolean = true;
  16.  
  17. stop() {
  18. this.running = false
  19. }
  20. }
  21.  
  22. class AccountCollection extends Collection {
  23.  
  24. items: Array<InstagramAccount>;
  25. options: Object;
  26. projectLog: ProjectLog;
  27.  
  28. constructor(items: Array = [], opt: Object = {}) {
  29. super(items);
  30. this.options = opt;
  31. Object.assign(this, opt);
  32.  
  33. if (opt.projectName) {
  34. this.projectLog = new ProjectLog(opt.projectName);
  35. }
  36.  
  37. }
  38.  
  39. static async createFromFile(filePath, properties = {}): Promise<AccountCollection> {
  40.  
  41. const isExisted = await jetpack.existsAsync(filePath);
  42.  
  43. if (!isExisted) {
  44. throw new Error(`File is not found ${filePath} to work with AccountCollection:createFromFile`)
  45. }
  46.  
  47. let content = await jetpack.readAsync(filePath);
  48. content = content.trim();
  49.  
  50. if (!content) {
  51. throw new Error(`File content is empty ${filePath} to work with AccountCollection:createFromFile`)
  52. }
  53.  
  54. return AccountCollection.createFromArray(content.split(/\r?\n/).filter(item => !!item), properties);
  55.  
  56. }
  57.  
  58. static createFromArray(accounts, properties = {}): AccountCollection {
  59.  
  60. accounts = accounts.map((accountLine) => {
  61.  
  62. if (!isString(accountLine)) {
  63. accountLine = accountLine.toString()
  64. }
  65.  
  66. return new InstagramAccount(accountLine, properties);
  67.  
  68. });
  69.  
  70. accounts = uniqBy(accounts, 'username');
  71.  
  72. return new AccountCollection(accounts, properties);
  73.  
  74. }
  75.  
  76. static sortByStartSessionTime(list: Array<InstagramAccount>, chunkSize: number): Array<InstagramAccount> {
  77.  
  78. list = list.filter((account) => account.sessionInfo);
  79.  
  80. list = sortBy(list, (account: InstagramAccount) => account.sessionInfo.last_refreshed);
  81.  
  82. const chunkSizeForSorting = Math.ceil(list.length / chunkSize);
  83. const accountsChunks = chunk(list, chunkSizeForSorting);
  84.  
  85. const result = [];
  86. for (let i = 0; i < chunkSizeForSorting; i++) {
  87. accountsChunks.forEach(item => {
  88. if (item[i]) {
  89. result.push(item[i])
  90. }
  91. })
  92. }
  93.  
  94. return result;
  95. }
  96.  
  97. static async loadAllAccountLogFromDb(accounts: Array<InstagramAccount> = []): Promise<void> {
  98.  
  99.  
  100. const accountsCollection = await AccountLog.getDbAccountsCollection();
  101.  
  102. const accountLogs: Array<dbAccountMeta> = await accountsCollection.find({}).toArray();
  103.  
  104. let notFoundList = [];
  105. for (let i = 0; i < accounts.length; i++) {
  106. const account: InstagramAccount = accounts[i];
  107. const accountLog: dbAccauntLog = accountLogs.find(accountMeta => (accountMeta.username === account.username));
  108. if (accountLog && accountLog.createdTime) {
  109. account.dbAccountLog = accountLog;
  110. } else {
  111. notFoundList.push(account)
  112. }
  113. }
  114. if (notFoundList.length) {
  115. accounts = notFoundList
  116. } else {
  117. return
  118. }
  119.  
  120. for (let i = 0; i < accounts.length; i++) {
  121. const account: InstagramAccount = accounts[i];
  122. await account.getAccountLogFromDb(true);
  123. }
  124. }
  125.  
  126. static async loadAllAccountMetaFromDb(accounts: Array<InstagramAccount> = [], projectName): Promise<void> {
  127.  
  128. if (projectName) {
  129. const accountsCollection = await InstagramAccount.getDbAccountsCollection(projectName);
  130.  
  131. const accountMetas: Array<dbAccountMeta> = await accountsCollection.find({}).toArray();
  132.  
  133. let notFoundList = [];
  134. for (let i = 0; i < accounts.length; i++) {
  135. const account: InstagramAccount = accounts[i];
  136. const accountMeta = accountMetas.find(accountMeta => (accountMeta.username === account.username));
  137. if (accountMeta) {
  138. account.dbAccountMeta = accountMeta;
  139. } else {
  140. notFoundList.push(account)
  141. }
  142. }
  143. if (notFoundList.length) {
  144. accounts = notFoundList
  145. } else {
  146. return;
  147. }
  148. }
  149.  
  150. for (let i = 0; i < accounts.length; i++) {
  151. const account: InstagramAccount = accounts[i];
  152. account.dbAccountMeta = await account.getAccountMetaFromDb(true);
  153. }
  154. }
  155.  
  156. remove(account): null {
  157.  
  158. let index = this.items.findIndex(item => item.username == account.username);
  159.  
  160. if (index !== -1) {
  161. return this.items.splice(index, 1);
  162. }
  163.  
  164. return null;
  165.  
  166. }
  167.  
  168. clone(): AccountCollection {
  169. return new AccountCollection([].concat(this.items), this.options)
  170. }
  171.  
  172. slice(...args) {
  173. return new AccountCollection(this.items.slice(...args), this.options);
  174. }
  175.  
  176. async chunk(chunkSize: number, minChunkSize: number = 0): Promise<Array<Array<InstagramAccount>>> {
  177. const result = chunk(this.items, chunkSize).filter((item: Array) => item.length > minChunkSize);
  178. return result
  179.  
  180. }
  181.  
  182. async chunkByPromotionRules(chunkSize: number, minChunkSize?: number): Promise<Array<Array<InstagramAccount>>> {
  183.  
  184. if (!minChunkSize || minChunkSize > chunkSize) minChunkSize = chunkSize - 1;
  185.  
  186. let items: Array<InstagramAccount> = [].concat(this.items);
  187. items = items.shuffle();
  188.  
  189. let result = [];
  190. let index = 0;
  191. let uniq = [];
  192.  
  193. do {
  194.  
  195. if (uniq.length < chunkSize) {
  196.  
  197. uniq = [].concat(items);
  198.  
  199. // Удаляем одинаковые user-agents индификаторы
  200. // uniq = uniqBy(uniq, 'deviceUserAgentIdentificator');
  201.  
  202. // console.log(`!!!! AccountCollection.js:235 uniq:\n`, uniq.length, '\n!!!!! '); //TODO: удали меня
  203.  
  204. // удаляем аккаунты с маленьким промежутком созданя
  205. // uniq = await AccountCollection.clearCloseBirthday(uniq, 6);
  206.  
  207. // Сортируем по времяни создания сессии
  208. // uniq = AccountCollection.sortByStartSessionTime(uniq, chunkSize);
  209.  
  210. }
  211.  
  212. result[index] = uniq.splice(0, chunkSize);
  213. items = differenceBy(items, result[index], 'username');
  214.  
  215. index++;
  216.  
  217. } while (result[index - 1].length > minChunkSize && items.length > minChunkSize);
  218. if (result[index - 1].length < minChunkSize) {
  219. result.pop()
  220. }
  221.  
  222. return result;
  223.  
  224. }
  225.  
  226. async filter(cb: (account: InstagramAccount, index: number, data: Array<InstagramAccount>) => {}): Promise<AccountCollection> {
  227. this.items = await this.items.filterAsync(cb);
  228. return this;
  229. }
  230.  
  231. async getDbAccountsCollection(): Promise<CollectionMongo> {
  232. if ((!this.commandSettings || !this.commandSettings.db) && !this.projectName) {
  233. throw new Error('Db name not specified to work with AccountCollection:getDbAccountsCollection');
  234. }
  235. return await InstagramAccount.getDbAccountsCollection(this.commandSettings.db || this.projectName);
  236. }
  237.  
  238. async getAccountLogsDbCollection(): Promise<CollectionMongo> {
  239. return await AccountLog.getDbAccountsCollection();
  240. }
  241.  
  242. async excludeErrorAccounts(exceptionErrorList: Array<string> = [], commandDir: string = 'run_promotion'): Promise<AccountCollection> {
  243.  
  244. if (!this.projectName) {
  245. throw new Error('Project name not specified to work with AccountCollection:excludeErrorAccounts')
  246. }
  247.  
  248. let errorLines = await getArrayOfLinesFromFile(
  249. `${configs.projectsDir}/${this.projectName}/${commandDir}/errors.txt`
  250. );
  251.  
  252. let errors = errorLines.map((errorLine) => {
  253.  
  254. let errorArray = errorLine.split(':');
  255.  
  256. return {
  257. username : errorArray[0],
  258. password : errorArray[1],
  259. name : errorArray[2],
  260. message : errorArray[3]
  261. };
  262.  
  263. });
  264.  
  265. const length = this.items.length;
  266.  
  267. this.items = this.items.filter((account) => {
  268. return !errors.some(function (error) {
  269. return (error.username === account.username && !exceptionErrorList.some(exceptionError => exceptionError === error.name));
  270. });
  271. });
  272.  
  273. this.projectLog && this.projectLog.pushInfoToProject({
  274. excludeErrorAccounts : this.items.length - length || 0
  275. });
  276.  
  277. this.log(`Exclude error accounts:`, this.items.length - length || 0);
  278. return this;
  279.  
  280. }
  281.  
  282. async excludeBadSessionAccounts(): Promise<AccountCollection> {
  283.  
  284. const length = this.items.length;
  285.  
  286. this.items = await this.items.filterAsync(async (account) => {
  287.  
  288. try {
  289. let sessionId = await account.cookie.getSessionId();
  290. return true;
  291. } catch (e) {
  292. return false;
  293. }
  294.  
  295. });
  296.  
  297. this.log(`Exclude bad session accounts:`, this.items.length - length || 0);
  298.  
  299. return this;
  300.  
  301.  
  302. }
  303.  
  304.  
  305. async excludeFrozenAccounts(): Promise<AccountCollection> {
  306.  
  307. const accountsCollections = await this.getDbAccountsCollection();
  308.  
  309. let excludeAccountsFromPromotion = await accountsCollections.find({
  310. frozenTo : {$gte : moment().toDate()},
  311. username : {$in : this.items.map((a) => a.username)}
  312. }).toArray();
  313.  
  314. const length = this.items.length;
  315.  
  316. this.items = this.items.filter((account) => {
  317.  
  318. return !excludeAccountsFromPromotion.some(function (excludeAccount) {
  319. return excludeAccount.username === account.username
  320. });
  321.  
  322. });
  323.  
  324. this.projectLog && this.projectLog.pushInfoToProject({
  325. excludeFrozenAccounts : this.items.length - length || 0
  326. });
  327.  
  328. this.log(`Exclude frozen accounts:`, this.items.length - length || 0);
  329.  
  330. return this;
  331.  
  332. }
  333.  
  334. async excludeCheckpointErrorAccounts(): Promise<AccountCollection> {
  335.  
  336. const accountsCollections = await this.getDbAccountsCollection();
  337.  
  338. let excludeAccountsFromPromotion = await accountsCollections.find({
  339. checkpointError : true,
  340. username : {$in : this.items.map((a) => a.username)}
  341. }).toArray();
  342.  
  343. const length = this.items.length;
  344.  
  345. this.items = this.items.filter((account) => {
  346.  
  347. return !excludeAccountsFromPromotion.some(function (excludeAccount) {
  348. return excludeAccount.username === account.username
  349. });
  350.  
  351. });
  352.  
  353. this.projectLog && this.projectLog.pushInfoToProject({
  354. excludeCheckpointErrorAccounts : this.items.length - length || 0
  355. });
  356.  
  357. this.log(`Exclude checkpoint error accounts:`, this.items.length - length || 0);
  358.  
  359. return this;
  360.  
  361. }
  362.  
  363. async excludeYoungAccounts(): Promise<AccountCollection> {
  364.  
  365. const logsCollections = await this.getAccountLogsDbCollection();
  366.  
  367. let excludeAccountsFromPromotion = await logsCollections.find({
  368. username : {$in : this.items.map((a) => a.username)}
  369. }).toArray();
  370.  
  371. const length = this.items.length;
  372.  
  373. this.items = await this.items.filterAsync(async (account: InstagramAccount) => {
  374.  
  375. let eAccount;
  376. excludeAccountsFromPromotion.some(function (excludeAccount) {
  377. if (excludeAccount.username === account.username) {
  378. eAccount = excludeAccount;
  379. return true;
  380. }
  381. return false;
  382. });
  383.  
  384. if (!eAccount) {
  385. return true
  386. }
  387.  
  388. const createTime = await account.accountLog.getCreatedTime();
  389.  
  390. if (createTime < moment().add(-this.commandSettings.minAccountsAge, 'hours').toDate()) {
  391. return true;
  392. }
  393. return false
  394.  
  395. });
  396.  
  397. this.projectLog && this.projectLog.pushInfoToProject({
  398. excludeYoungAccounts : this.items.length - length || 0
  399. });
  400.  
  401. this.log(`Exclude young accounts:`, this.items.length - length || 0);
  402.  
  403. return this;
  404.  
  405. }
  406.  
  407. async runEach(opt: {
  408. run: Function,
  409. onEndChunk: Function,
  410. onError: Function,
  411. onStartChunk: Function,
  412. createChunks: Function,
  413. chunkSize: number,
  414. minAmountOfConcurrentAccounts: number
  415. }): Promise<Process> {
  416.  
  417. opt = defaults({}, opt, {
  418. onStartChunk : async (accountChunk: Array<InstagramAccount>, chunkNum: number, parallel: Parallel): Promise => {
  419. },
  420. run : async (account: InstagramAccount, accountNum: number, parallel: Parallel): Promise => {
  421. },
  422. onError : async (account: InstagramAccount, e: Error, parallel: Parallel): Promise => {
  423. },
  424. onEndChunk : async (accounts: Array<InstagramAccount>, chunkNum: number, parallel: Parallel): Promise => {
  425. },
  426. createChunks : async (accounts: AccountCollection, chunkSize: number): Promise<Array<Array<InstagramAccount>>> => {
  427. return await accounts.chunk(chunkSize)
  428. },
  429. chunkSize : 1,
  430. minAmountOfConcurrentAccounts : 0
  431. });
  432.  
  433. const process = new Process();
  434.  
  435. let accountChunks: Array<Array<InstagramAccount>> = await opt.createChunks(this, opt.chunkSize);
  436.  
  437. for (let chunkNum = 0; chunkNum < accountChunks.length; chunkNum++) {
  438.  
  439. if (!process.running) {
  440. break
  441. }
  442.  
  443. let accountChunk = accountChunks[chunkNum];
  444.  
  445. const parallel = new Parallel(accountChunk.map((account, accountNum) => {
  446. return async (parallel: Parallel) => {
  447. try {
  448.  
  449. await opt.run(account, accountNum, parallel);
  450. // stop execution if minAmountOfConcurrentAccounts reached and exist next accounts chunkByPromotionRules
  451. if (
  452. opt.minAmountOfConcurrentAccounts &&
  453. opt.minAmountOfConcurrentAccounts >= parallel.length
  454. ) {
  455. parallel.stop();
  456. }
  457.  
  458. if (!process.running) {
  459. parallel.stop();
  460. }
  461.  
  462. } catch (e) {
  463. await opt.onError(account, e, parallel, process);
  464. }
  465. }
  466. }));
  467. await opt.onStartChunk(accountChunk, chunkNum, parallel);
  468.  
  469. await parallel.promise;
  470.  
  471. await opt.onEndChunk(accountChunk, chunkNum, parallel);
  472.  
  473. }
  474.  
  475. console.log('!!!! runEach done !!!!'); //TODO: удали меня
  476.  
  477. }
  478.  
  479. log(...attr): void {
  480. console.log(...attr);
  481. }
  482.  
  483. }
  484.  
  485.  
  486. export default AccountCollection;
Add Comment
Please, Sign In to add comment