Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import Collection from '../Plex/Collection';
- import jetpack from 'fs-jetpack';
- import InstagramAccount from './InstagramAccount';
- import {chunk, defaults, differenceBy, isString, sortBy, uniqBy} from "lodash";
- import configs from "../../config/app";
- import moment from "moment";
- import type {dbAccauntLog, dbAccountMeta} from "../Plex/type/db.flow";
- import AccountLog from "../Plex/AccountLog";
- import type {Collection as CollectionMongo} from "../Plex/type/mongo.flow";
- import {getArrayOfLinesFromFile} from "../Plex/Helpers";
- import Parallel from "../Plex/Parallel";
- import ProjectLog from "../Plex/Db/ProjectLog";
- export class Process {
- running: boolean = true;
- stop() {
- this.running = false
- }
- }
- class AccountCollection extends Collection {
- items: Array<InstagramAccount>;
- options: Object;
- projectLog: ProjectLog;
- constructor(items: Array = [], opt: Object = {}) {
- super(items);
- this.options = opt;
- Object.assign(this, opt);
- if (opt.projectName) {
- this.projectLog = new ProjectLog(opt.projectName);
- }
- }
- static async createFromFile(filePath, properties = {}): Promise<AccountCollection> {
- const isExisted = await jetpack.existsAsync(filePath);
- if (!isExisted) {
- throw new Error(`File is not found ${filePath} to work with AccountCollection:createFromFile`)
- }
- let content = await jetpack.readAsync(filePath);
- content = content.trim();
- if (!content) {
- throw new Error(`File content is empty ${filePath} to work with AccountCollection:createFromFile`)
- }
- return AccountCollection.createFromArray(content.split(/\r?\n/).filter(item => !!item), properties);
- }
- static createFromArray(accounts, properties = {}): AccountCollection {
- accounts = accounts.map((accountLine) => {
- if (!isString(accountLine)) {
- accountLine = accountLine.toString()
- }
- return new InstagramAccount(accountLine, properties);
- });
- accounts = uniqBy(accounts, 'username');
- return new AccountCollection(accounts, properties);
- }
- static sortByStartSessionTime(list: Array<InstagramAccount>, chunkSize: number): Array<InstagramAccount> {
- list = list.filter((account) => account.sessionInfo);
- list = sortBy(list, (account: InstagramAccount) => account.sessionInfo.last_refreshed);
- const chunkSizeForSorting = Math.ceil(list.length / chunkSize);
- const accountsChunks = chunk(list, chunkSizeForSorting);
- const result = [];
- for (let i = 0; i < chunkSizeForSorting; i++) {
- accountsChunks.forEach(item => {
- if (item[i]) {
- result.push(item[i])
- }
- })
- }
- return result;
- }
- static async loadAllAccountLogFromDb(accounts: Array<InstagramAccount> = []): Promise<void> {
- const accountsCollection = await AccountLog.getDbAccountsCollection();
- const accountLogs: Array<dbAccountMeta> = await accountsCollection.find({}).toArray();
- let notFoundList = [];
- for (let i = 0; i < accounts.length; i++) {
- const account: InstagramAccount = accounts[i];
- const accountLog: dbAccauntLog = accountLogs.find(accountMeta => (accountMeta.username === account.username));
- if (accountLog && accountLog.createdTime) {
- account.dbAccountLog = accountLog;
- } else {
- notFoundList.push(account)
- }
- }
- if (notFoundList.length) {
- accounts = notFoundList
- } else {
- return
- }
- for (let i = 0; i < accounts.length; i++) {
- const account: InstagramAccount = accounts[i];
- await account.getAccountLogFromDb(true);
- }
- }
- static async loadAllAccountMetaFromDb(accounts: Array<InstagramAccount> = [], projectName): Promise<void> {
- if (projectName) {
- const accountsCollection = await InstagramAccount.getDbAccountsCollection(projectName);
- const accountMetas: Array<dbAccountMeta> = await accountsCollection.find({}).toArray();
- let notFoundList = [];
- for (let i = 0; i < accounts.length; i++) {
- const account: InstagramAccount = accounts[i];
- const accountMeta = accountMetas.find(accountMeta => (accountMeta.username === account.username));
- if (accountMeta) {
- account.dbAccountMeta = accountMeta;
- } else {
- notFoundList.push(account)
- }
- }
- if (notFoundList.length) {
- accounts = notFoundList
- } else {
- return;
- }
- }
- for (let i = 0; i < accounts.length; i++) {
- const account: InstagramAccount = accounts[i];
- account.dbAccountMeta = await account.getAccountMetaFromDb(true);
- }
- }
- remove(account): null {
- let index = this.items.findIndex(item => item.username == account.username);
- if (index !== -1) {
- return this.items.splice(index, 1);
- }
- return null;
- }
- clone(): AccountCollection {
- return new AccountCollection([].concat(this.items), this.options)
- }
- slice(...args) {
- return new AccountCollection(this.items.slice(...args), this.options);
- }
- async chunk(chunkSize: number, minChunkSize: number = 0): Promise<Array<Array<InstagramAccount>>> {
- const result = chunk(this.items, chunkSize).filter((item: Array) => item.length > minChunkSize);
- return result
- }
- async chunkByPromotionRules(chunkSize: number, minChunkSize?: number): Promise<Array<Array<InstagramAccount>>> {
- if (!minChunkSize || minChunkSize > chunkSize) minChunkSize = chunkSize - 1;
- let items: Array<InstagramAccount> = [].concat(this.items);
- items = items.shuffle();
- let result = [];
- let index = 0;
- let uniq = [];
- do {
- if (uniq.length < chunkSize) {
- uniq = [].concat(items);
- // Удаляем одинаковые user-agents индификаторы
- // uniq = uniqBy(uniq, 'deviceUserAgentIdentificator');
- // console.log(`!!!! AccountCollection.js:235 uniq:\n`, uniq.length, '\n!!!!! '); //TODO: удали меня
- // удаляем аккаунты с маленьким промежутком созданя
- // uniq = await AccountCollection.clearCloseBirthday(uniq, 6);
- // Сортируем по времяни создания сессии
- // uniq = AccountCollection.sortByStartSessionTime(uniq, chunkSize);
- }
- result[index] = uniq.splice(0, chunkSize);
- items = differenceBy(items, result[index], 'username');
- index++;
- } while (result[index - 1].length > minChunkSize && items.length > minChunkSize);
- if (result[index - 1].length < minChunkSize) {
- result.pop()
- }
- return result;
- }
- async filter(cb: (account: InstagramAccount, index: number, data: Array<InstagramAccount>) => {}): Promise<AccountCollection> {
- this.items = await this.items.filterAsync(cb);
- return this;
- }
- async getDbAccountsCollection(): Promise<CollectionMongo> {
- if ((!this.commandSettings || !this.commandSettings.db) && !this.projectName) {
- throw new Error('Db name not specified to work with AccountCollection:getDbAccountsCollection');
- }
- return await InstagramAccount.getDbAccountsCollection(this.commandSettings.db || this.projectName);
- }
- async getAccountLogsDbCollection(): Promise<CollectionMongo> {
- return await AccountLog.getDbAccountsCollection();
- }
- async excludeErrorAccounts(exceptionErrorList: Array<string> = [], commandDir: string = 'run_promotion'): Promise<AccountCollection> {
- if (!this.projectName) {
- throw new Error('Project name not specified to work with AccountCollection:excludeErrorAccounts')
- }
- let errorLines = await getArrayOfLinesFromFile(
- `${configs.projectsDir}/${this.projectName}/${commandDir}/errors.txt`
- );
- let errors = errorLines.map((errorLine) => {
- let errorArray = errorLine.split(':');
- return {
- username : errorArray[0],
- password : errorArray[1],
- name : errorArray[2],
- message : errorArray[3]
- };
- });
- const length = this.items.length;
- this.items = this.items.filter((account) => {
- return !errors.some(function (error) {
- return (error.username === account.username && !exceptionErrorList.some(exceptionError => exceptionError === error.name));
- });
- });
- this.projectLog && this.projectLog.pushInfoToProject({
- excludeErrorAccounts : this.items.length - length || 0
- });
- this.log(`Exclude error accounts:`, this.items.length - length || 0);
- return this;
- }
- async excludeBadSessionAccounts(): Promise<AccountCollection> {
- const length = this.items.length;
- this.items = await this.items.filterAsync(async (account) => {
- try {
- let sessionId = await account.cookie.getSessionId();
- return true;
- } catch (e) {
- return false;
- }
- });
- this.log(`Exclude bad session accounts:`, this.items.length - length || 0);
- return this;
- }
- async excludeFrozenAccounts(): Promise<AccountCollection> {
- const accountsCollections = await this.getDbAccountsCollection();
- let excludeAccountsFromPromotion = await accountsCollections.find({
- frozenTo : {$gte : moment().toDate()},
- username : {$in : this.items.map((a) => a.username)}
- }).toArray();
- const length = this.items.length;
- this.items = this.items.filter((account) => {
- return !excludeAccountsFromPromotion.some(function (excludeAccount) {
- return excludeAccount.username === account.username
- });
- });
- this.projectLog && this.projectLog.pushInfoToProject({
- excludeFrozenAccounts : this.items.length - length || 0
- });
- this.log(`Exclude frozen accounts:`, this.items.length - length || 0);
- return this;
- }
- async excludeCheckpointErrorAccounts(): Promise<AccountCollection> {
- const accountsCollections = await this.getDbAccountsCollection();
- let excludeAccountsFromPromotion = await accountsCollections.find({
- checkpointError : true,
- username : {$in : this.items.map((a) => a.username)}
- }).toArray();
- const length = this.items.length;
- this.items = this.items.filter((account) => {
- return !excludeAccountsFromPromotion.some(function (excludeAccount) {
- return excludeAccount.username === account.username
- });
- });
- this.projectLog && this.projectLog.pushInfoToProject({
- excludeCheckpointErrorAccounts : this.items.length - length || 0
- });
- this.log(`Exclude checkpoint error accounts:`, this.items.length - length || 0);
- return this;
- }
- async excludeYoungAccounts(): Promise<AccountCollection> {
- const logsCollections = await this.getAccountLogsDbCollection();
- let excludeAccountsFromPromotion = await logsCollections.find({
- username : {$in : this.items.map((a) => a.username)}
- }).toArray();
- const length = this.items.length;
- this.items = await this.items.filterAsync(async (account: InstagramAccount) => {
- let eAccount;
- excludeAccountsFromPromotion.some(function (excludeAccount) {
- if (excludeAccount.username === account.username) {
- eAccount = excludeAccount;
- return true;
- }
- return false;
- });
- if (!eAccount) {
- return true
- }
- const createTime = await account.accountLog.getCreatedTime();
- if (createTime < moment().add(-this.commandSettings.minAccountsAge, 'hours').toDate()) {
- return true;
- }
- return false
- });
- this.projectLog && this.projectLog.pushInfoToProject({
- excludeYoungAccounts : this.items.length - length || 0
- });
- this.log(`Exclude young accounts:`, this.items.length - length || 0);
- return this;
- }
- async runEach(opt: {
- run: Function,
- onEndChunk: Function,
- onError: Function,
- onStartChunk: Function,
- createChunks: Function,
- chunkSize: number,
- minAmountOfConcurrentAccounts: number
- }): Promise<Process> {
- opt = defaults({}, opt, {
- onStartChunk : async (accountChunk: Array<InstagramAccount>, chunkNum: number, parallel: Parallel): Promise => {
- },
- run : async (account: InstagramAccount, accountNum: number, parallel: Parallel): Promise => {
- },
- onError : async (account: InstagramAccount, e: Error, parallel: Parallel): Promise => {
- },
- onEndChunk : async (accounts: Array<InstagramAccount>, chunkNum: number, parallel: Parallel): Promise => {
- },
- createChunks : async (accounts: AccountCollection, chunkSize: number): Promise<Array<Array<InstagramAccount>>> => {
- return await accounts.chunk(chunkSize)
- },
- chunkSize : 1,
- minAmountOfConcurrentAccounts : 0
- });
- const process = new Process();
- let accountChunks: Array<Array<InstagramAccount>> = await opt.createChunks(this, opt.chunkSize);
- for (let chunkNum = 0; chunkNum < accountChunks.length; chunkNum++) {
- if (!process.running) {
- break
- }
- let accountChunk = accountChunks[chunkNum];
- const parallel = new Parallel(accountChunk.map((account, accountNum) => {
- return async (parallel: Parallel) => {
- try {
- await opt.run(account, accountNum, parallel);
- // stop execution if minAmountOfConcurrentAccounts reached and exist next accounts chunkByPromotionRules
- if (
- opt.minAmountOfConcurrentAccounts &&
- opt.minAmountOfConcurrentAccounts >= parallel.length
- ) {
- parallel.stop();
- }
- if (!process.running) {
- parallel.stop();
- }
- } catch (e) {
- await opt.onError(account, e, parallel, process);
- }
- }
- }));
- await opt.onStartChunk(accountChunk, chunkNum, parallel);
- await parallel.promise;
- await opt.onEndChunk(accountChunk, chunkNum, parallel);
- }
- console.log('!!!! runEach done !!!!'); //TODO: удали меня
- }
- log(...attr): void {
- console.log(...attr);
- }
- }
- export default AccountCollection;
Add Comment
Please, Sign In to add comment