Untitled

Swarm initialized: current node (6nuiz37za10ask1nigei1m8u2) is now a manager.

To add a worker to this swarm, run the following command:

    docker swarm join --token SWMTKN-1-4tln85kgzr7jdc9hlvemig9f82dtxlvnfr2vo6p19phcz23sbg-7h4pduq67yqcgmv8gvo0az581 209.170.197.61:2377

To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions
ID                            HOSTNAME            STATUS              AVAILABILITY        MANAGER STATUS
6nuiz37za10ask1nigei1m8u2 *   HOPCODEPROD1        Ready               Active              Leader
9evlc9rgzoa4cm4ejw2rh490t     HOPCODEPROD2        Ready               Active
e1e8a2795195oxjebn78lem04     mg8crw1             Ready               Active


=====================Push images======================

We have one docker registry on: https://docker.gogolistassist.com

Login: docker login docker.gogolistassist.com
User: magic8labs
Pass: hop7755

=====================Create services======================

# 1. Create network for all services

#!/bin/sh
docker network create \
  --driver overlay \
  --subnet 10.0.0.0/24 \
  gogolist-net


# 2. Create database service:

    2.1 Mongo DB
    mkdir /docker_ext/ggla_mongodb_prod/data -p
    mkdir /docker_ext/ggla_mongodb_prod/config -p

    #!/bin/sh
    docker service create \
        --network gogolist-net \
        --mount type=bind,src=/docker_ext/ggla_mongodb_prod/data,dst=/data/db \
        --mount type=bind,src=/docker_ext/ggla_mongodb_prod/config,dst=/data/configdb \
        --name ggla_mongodb \
        --hostname ggla_mongodb \
        --constraint 'node.id == 6nuiz37za10ask1nigei1m8u2' \
        mongo:latest mongod --replSet ggla

    2.2 Postgres
    mkdir /docker_ext/ggla_postgres_prod -p

    #!/bin/sh
    docker service create --name ggla_postgres \
        --mount type=bind,src=/docker_ext/ggla_postgres_prod,dst=/var/lib/postgresql/data \
        --env POSTGRES_PASSWORD=NugqENY6tsaT2w8t \
        --env POSTGRES_USER=user_magic8labs \
        --env POSTGRES_DB=ggla_postgresql \
        --replicas 1 \
        --network gogolist-net \
        --hostname ggla_postgres \
        --constraint "node.id == 6nuiz37za10ask1nigei1m8u2" \
        postgres:alpine

    2.3 Rabbitmq
    #!/bin/sh
    docker service create --name ggla_rabbitmq \
        --env RABBITMQ_DEFAULT_USER=magic8labs \
        --env RABBITMQ_DEFAULT_PASS=Fem5K4Cz3wZnTfpp \
        --replicas 1 \
        --network gogolist-net \
        --hostname ggla_rabbitmq \
        --constraint "node.id == 6nuiz37za10ask1nigei1m8u2" \
        --publish 15673:15672 \
        rabbitmq:management-alpine

    2.4 Rabbitmq
    #!/bin/sh
    docker service create --name ggla_redis \
        --replicas 1 \
        --network gogolist-net \
        --hostname ggla_redis \
        --constraint "node.id == 6nuiz37za10ask1nigei1m8u2" \
        redis:alpine


# 3. # Create webportal service:

#!/bin/sh
docker service create --name webportal \
    --env ROOT_URL=http://devm8.gogolistassist.com \
    --env PORT=9000 \
    --env MONGO_URL=mongodb://ggla_mongodb:27017/dropshipper \
    --env METEOR_SETTINGS='{"public":{"STRIPE_PUBLIC_TOKEN":"pk_test_G30HPbYMvwQ1uiPhd9QZsM17","CHROME_EXTENSION_ID":"igkkabmmpedmmmfdnjlbiihnhfmcblgb"},"private":{"HOST_URL":"http://devm8.gogolistassist.com","MAIL_URL_GG":"smtps://noreply.gogo.listassist%40gmail.com:bgltpbheagpmfpfp@smtp.gmail.com:587","MAIL_URL":"smtps://AKIAI6TE2WWRVWXCFP4A:AoKZPSqeURQGQr5Xs5KgcxjlCQOjU6NDIcHMaEAIMKBi@email-smtp.us-west-2.amazonaws.com:465","QUEUE_URL":"https://sqs.us-west-2.amazonaws.com/474929731385/AMZ-Subscriptions","AWS_CREDENTIALS":{"accessKeyId":"AKIAJ52ISNV5AGOPV45Q","secretAccessKey":"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","region":"us-west-2"},"OWN_MWS_CREDENTIALS":{"aWSAccessKeyId":"AKIAIMJD6TQYTB5IUEMQ","secretKey":"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"},"PRICECRAWL_HOST":"http://crawler_web:8000","PRIVATE_CRAWL_HOST":"http://crawler_web:8000","STRIPE_SECRET_TOKEN":"sk_test_atfSozXjEkCse6pfSjrt5uCm"},"AWSAccessKeyId":"AKIAJZDLDO7H46EZTXVQ","AWSSecretAccessKey":"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","AWSBucket":"gogolistassistimg","REDIS_SERVER":"ggla_redis","REDIS_PORT":6379,"MWS_CACHE_TIME":[15,"minutes"],"PHARSE":"beta","FROM_EMAIL":"GoGoListAssist <no-reply@gogolistassist.com>","AGENDA":{"GET_ORDER_LIST":[15,"minutes"],"SEND_FEEDBACK_MSG":[3,"minutes"],"CLEAR_REPORTS":[1,"weeks"],"REPRICING":[3,"hours"],"SUBSCRIPTION_REPRICING":[5,"minutes"],"AUTO_UPDATE_BOOKMARK":"10:00pm"},"BETA_END":"2017-03-28T09:20:41Z"}' \
    --replicas 2 \
    --network gogolist-net \
    --hostname webportal \
    --name webportal \
    --with-registry-auth \
    --constraint "node.id == 6nuiz37za10ask1nigei1m8u2" \
    --publish 8081:9000 \
    docker.gogolistassist.com/webportal:latest


# 5. # Create webadmin service:

#!/bin/sh
docker service create --name webadmin \
    --env ROOT_URL=http://devm8admin.gogolistassist.com \
    --env PORT=8080 \
    --env MONGO_URL=mongodb://ggla_mongodb:27017/dropshipper \
    --env METEOR_SETTINGS='{ "public":{"STRIPE_PUBLIC_TOKEN":"pk_test_G30HPbYMvwQ1uiPhd9QZsM17"},"private":{"HOST_URL":"http://devm8admin.gogolistassist.com", "WEBPORTAL_URL":"http://devm8.gogolistassist.com", "MAIL_URL_GG":"smtps://noreply.gogo.listassist%40gmail.com:bgltpbheagpmfpfp@smtp.gmail.com:587","MAIL_URL":"smtps://AKIAI6TE2WWRVWXCFP4A:AoKZPSqeURQGQr5Xs5KgcxjlCQOjU6NDIcHMaEAIMKBi@email-smtp.us-west-2.amazonaws.com:465","STRIPE_SECRET_TOKEN":"sk_test_atfSozXjEkCse6pfSjrt5uCm"},"BETA_END":"2017-04-30T00:00:00Z"}' \
    --replicas 2 \
    --network gogolist-net \
    --hostname webadmin \
    --name webadmin \
    --with-registry-auth \
    --constraint "node.id == 6nuiz37za10ask1nigei1m8u2" \
    --publish 8080:8080 \
    docker.gogolistassist.com/webadmin:latest


# 6. Create web service:

#!/bin/sh
docker service create --name crawler_web \
    --env DATABASE_URL=postgres://user_magic8labs:NugqENY6tsaT2w8t@ggla_postgres:5432/ggla_postgresql \
    --env ALLOWED_HOSTS=209.170.197.60,devm8.gogolistassist.com,devm8crawler.gogolistassist.com,localhost,127.0.0.1 \
    --env SECRET_KEY=6s3XbcUwANWSYAAHHMdkxyYcK6Rrs6EfuddMZaLxY3jk5Jt37t \
    --env CELERY_BROKER_URL=amqp://magic8labs:Fem5K4Cz3wZnTfpp@ggla_rabbitmq// \
    --env CELERY_RESULT_BACKEND=redis://ggla_redis:6379/0 \
    --env REDIS_CACHE=redis://ggla_redis:6379/1 \
    --env RESULT_MONGODB=mongodb://ggla_mongodb:27017/ \
    --env PROXY_HOST=http://usloft1630.startdedicated.com/api/proxy.php \
    --env PROXY_SECRET=75c814b517635205ee3c8753d0dddad6 \
    --env BONANZA_API_KEY=XsCGvhtdQV8jWT4C6E2XmRt2qzSt64RALVXKLM2QqxwxzkfJM4!43734 \
    --env MWS_ACCESS_KEY=AKIAIMJD6TQYTB5IUEMQ \
    --env MWS_SECRET_KEY=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 \
    --env TARANTOOL_CACHE=tarantool \
    --env KEEPA_API=1vdm5ldrq6puh54fo56o2tfl8uqfr7lekql78j050llko1u9o2g6gq2q06gh3jdl \
    --env CELERYD_POOL_RESTARTS=True \
    --env PROXY_MANAGEMENT=proxy_m8l \
    --mode global \
    --network gogolist-net \
    --hostname crawler_web \
    --publish 8000:8000 \
    --with-registry-auth \
    --constraint "node.id == 6nuiz37za10ask1nigei1m8u2" \
    docker.gogolistassist.com/crawlerengine_web:latest

    * Load users fixture:
        docker exec -it crawler_web.9evlc9rgzoa4cm4ejw2rh490t.$(docker service ps -f 'name=crawler_web' crawler_web -q --no-trunc) bash -c "./manage.py loaddata users"

    * Load crawlers fixture:
        docker exec -it crawler_web.9evlc9rgzoa4cm4ejw2rh490t.$(docker service ps -f 'name=crawler_web' crawler_web -q --no-trunc) bash -c "./manage.py loaddata crawlers"


# 7. Create tarantool service:
#!/bin/sh
docker service create --name tarantool_caching \
    --network gogolist-net \
    --hostname tarantool_caching \
    --publish 3301:3301 \
    --with-registry-auth \
    --constraint "node.id == vdobubscvjby82b6rz8l1ukji" \
    --mount type=bind,src=/docker_ext/tarantool_prod/data,dst=/opt/tarantool \
    docker.gogolistassist.com/tarantool-ggla:latest sh -c "tarantool /code/magic_8_labs.lua"


# 8123. Create proxy management service:
docker service create --name flower_management \
    --env DATABASE_URL=postgres://user_magic8labs:NugqENY6tsaT2w8t@ggla_postgres:5432/ggla_postgresql \
    --env ALLOWED_HOSTS=209.170.197.60,devm8.gogolistassist.com,devm8crawler.gogolistassist.com,localhost,127.0.0.1 \
    --env SECRET_KEY=6s3XbcUwANWSYAAHHMdkxyYcK6Rrs6EfuddMZaLxY3jk5Jt37t \
    --env CELERY_BROKER_URL=amqp://magic8labs:Fem5K4Cz3wZnTfpp@ggla_rabbitmq// \
    --env CELERY_RESULT_BACKEND=redis://ggla_redis:6379/0 \
    --env REDIS_CACHE=redis://ggla_redis:6379/1 \
    --env RESULT_MONGODB=mongodb://ggla_mongodb:27017/ \
    --env PROXY_HOST=http://usloft1630.startdedicated.com/api/proxy.php \
    --env PROXY_SECRET=75c814b517635205ee3c8753d0dddad6 \
    --env BONANZA_API_KEY=XsCGvhtdQV8jWT4C6E2XmRt2qzSt64RALVXKLM2QqxwxzkfJM4!43734 \
    --env MWS_ACCESS_KEY=AKIAIMJD6TQYTB5IUEMQ \
    --env MWS_SECRET_KEY=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 \
    --env TARANTOOL_CACHE=tarantool \
    --env KEEPA_API=1vdm5ldrq6puh54fo56o2tfl8uqfr7lekql78j050llko1u9o2g6gq2q06gh3jdl \
    --env CELERYD_POOL_RESTARTS=True \
    --env PROXY_MANAGEMENT=proxy_m8l \
    --network gogolist-net \
    --hostname flower_management \
    --publish 5555:5555 \
    --with-registry-auth \
    --constraint "node.id == 6nuiz37za10ask1nigei1m8u2" \
    docker.gogolistassist.com/crawlerengine_worker:latest celery -A crawler_engine flower


# 8. Create proxy management service:
docker service create --name proxy_m8l \
    --env DATABASE_URL=postgres://user_magic8labs:NugqENY6tsaT2w8t@ggla_postgres:5432/ggla_postgresql \
    --env ALLOWED_HOSTS=209.170.197.60,devm8.gogolistassist.com,devm8crawler.gogolistassist.com,localhost,127.0.0.1 \
    --env SECRET_KEY=6s3XbcUwANWSYAAHHMdkxyYcK6Rrs6EfuddMZaLxY3jk5Jt37t \
    --env CELERY_BROKER_URL=amqp://magic8labs:Fem5K4Cz3wZnTfpp@ggla_rabbitmq// \
    --env CELERY_RESULT_BACKEND=redis://ggla_redis:6379/0 \
    --env REDIS_CACHE=redis://ggla_redis:6379/1 \
    --env RESULT_MONGODB=mongodb://ggla_mongodb:27017/ \
    --env PROXY_HOST=http://usloft1630.startdedicated.com/api/proxy.php \
    --env PROXY_SECRET=75c814b517635205ee3c8753d0dddad6 \
    --env BONANZA_API_KEY=XsCGvhtdQV8jWT4C6E2XmRt2qzSt64RALVXKLM2QqxwxzkfJM4!43734 \
    --env MWS_ACCESS_KEY=AKIAIMJD6TQYTB5IUEMQ \
    --env MWS_SECRET_KEY=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 \
    --env TARANTOOL_CACHE=tarantool \
    --env KEEPA_API=1vdm5ldrq6puh54fo56o2tfl8uqfr7lekql78j050llko1u9o2g6gq2q06gh3jdl \
    --env CELERYD_POOL_RESTARTS=True \
    --env PROXY_MANAGEMENT=proxy_m8l \
    --network gogolist-net \
    --hostname proxy_m8l \
    --publish 5000:5000 \
    --with-registry-auth \
    --constraint "node.id == 6nuiz37za10ask1nigei1m8u2" \
    docker.gogolistassist.com/crawlerengine_worker:latest bash -c "python3 ./proxy_management/run.py"


# 9. Create worker service:

#!/bin/sh
docker service create --name crawler_worker \
    --env DATABASE_URL=postgres://user_magic8labs:NugqENY6tsaT2w8t@ggla_postgres:5432/ggla_postgresql \
    --env ALLOWED_HOSTS=209.170.197.60,devm8.gogolistassist.com,devm8crawler.gogolistassist.com,localhost,127.0.0.1 \
    --env SECRET_KEY=6s3XbcUwANWSYAAHHMdkxyYcK6Rrs6EfuddMZaLxY3jk5Jt37t \
    --env CELERY_BROKER_URL=amqp://magic8labs:Fem5K4Cz3wZnTfpp@ggla_rabbitmq// \
    --env CELERY_RESULT_BACKEND=redis://ggla_redis:6379/0 \
    --env REDIS_CACHE=redis://ggla_redis:6379/1 \
    --env RESULT_MONGODB=mongodb://ggla_mongodb:27017/ \
    --env PROXY_HOST=http://usloft1630.startdedicated.com/api/proxy.php \
    --env PROXY_SECRET=75c814b517635205ee3c8753d0dddad6 \
    --env BONANZA_API_KEY=XsCGvhtdQV8jWT4C6E2XmRt2qzSt64RALVXKLM2QqxwxzkfJM4!43734 \
    --env MWS_ACCESS_KEY=AKIAIMJD6TQYTB5IUEMQ \
    --env MWS_SECRET_KEY=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 \
    --env TARANTOOL_CACHE=tarantool \
    --env KEEPA_API=1vdm5ldrq6puh54fo56o2tfl8uqfr7lekql78j050llko1u9o2g6gq2q06gh3jdl \
    --env CELERYD_POOL_RESTARTS=True \
    --env PROXY_MANAGEMENT=proxy_m8l \
    --replicas 3 \
    --network gogolist-net \
    --hostname crawler_worker \
    --with-registry-auth \
    --constraint "node.id == e1e8a2795195oxjebn78lem04" \
    docker.gogolistassist.com/crawlerengine_worker:latest celery -A crawler_engine worker -l info -P eventlet -X update_supplier,amazon_advertising_call -c 100


# 10. Create autoupdate worker service:

#!/bin/sh
docker service create --name crawler_worker_autoupdate \
    --env DATABASE_URL=postgres://user_magic8labs:NugqENY6tsaT2w8t@ggla_postgres:5432/ggla_postgresql \
    --env ALLOWED_HOSTS=209.170.197.60,devm8.gogolistassist.com,devm8crawler.gogolistassist.com,localhost,127.0.0.1 \
    --env SECRET_KEY=6s3XbcUwANWSYAAHHMdkxyYcK6Rrs6EfuddMZaLxY3jk5Jt37t \
    --env CELERY_BROKER_URL=amqp://magic8labs:Fem5K4Cz3wZnTfpp@ggla_rabbitmq// \
    --env CELERY_RESULT_BACKEND=redis://ggla_redis:6379/0 \
    --env REDIS_CACHE=redis://ggla_redis:6379/1 \
    --env RESULT_MONGODB=mongodb://ggla_mongodb:27017/ \
    --env PROXY_HOST=http://usloft1630.startdedicated.com/api/proxy.php \
    --env PROXY_SECRET=75c814b517635205ee3c8753d0dddad6 \
    --env BONANZA_API_KEY=XsCGvhtdQV8jWT4C6E2XmRt2qzSt64RALVXKLM2QqxwxzkfJM4!43734 \
    --env MWS_ACCESS_KEY=AKIAIMJD6TQYTB5IUEMQ \
    --env MWS_SECRET_KEY=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 \
    --env TARANTOOL_CACHE=tarantool \
    --env KEEPA_API=1vdm5ldrq6puh54fo56o2tfl8uqfr7lekql78j050llko1u9o2g6gq2q06gh3jdl \
    --env CELERYD_POOL_RESTARTS=True \
    --env PROXY_MANAGEMENT=proxy_m8l \
    --replicas 1 \
    --network gogolist-net \
    --hostname crawler_worker_autoupdate \
    --with-registry-auth \
    --constraint "node.id == e1e8a2795195oxjebn78lem04" \
    docker.gogolistassist.com/crawlerengine_worker:latest celery -A crawler_engine worker -l info -n auto_update_random_string -P eventlet -X update_supplier,amazon_advertising_call -c 100


# 11. Create advertising worker service:

#!/bin/sh
docker service create --name crawler_worker_advertising_api_call \
    --env DATABASE_URL=postgres://user_magic8labs:NugqENY6tsaT2w8t@ggla_postgres:5432/ggla_postgresql \
    --env ALLOWED_HOSTS=209.170.197.60,devm8.gogolistassist.com,devm8crawler.gogolistassist.com,localhost,127.0.0.1 \
    --env SECRET_KEY=6s3XbcUwANWSYAAHHMdkxyYcK6Rrs6EfuddMZaLxY3jk5Jt37t \
    --env CELERY_BROKER_URL=amqp://magic8labs:Fem5K4Cz3wZnTfpp@ggla_rabbitmq// \
    --env CELERY_RESULT_BACKEND=redis://ggla_redis:6379/0 \
    --env REDIS_CACHE=redis://ggla_redis:6379/1 \
    --env RESULT_MONGODB=mongodb://ggla_mongodb:27017/ \
    --env PROXY_HOST=http://usloft1630.startdedicated.com/api/proxy.php \
    --env PROXY_SECRET=75c814b517635205ee3c8753d0dddad6 \
    --env BONANZA_API_KEY=XsCGvhtdQV8jWT4C6E2XmRt2qzSt64RALVXKLM2QqxwxzkfJM4!43734 \
    --env MWS_ACCESS_KEY=AKIAIMJD6TQYTB5IUEMQ \
    --env MWS_SECRET_KEY=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 \
    --env TARANTOOL_CACHE=tarantool \
    --env KEEPA_API=1vdm5ldrq6puh54fo56o2tfl8uqfr7lekql78j050llko1u9o2g6gq2q06gh3jdl \
    --env CELERYD_POOL_RESTARTS=True \
    --env PROXY_MANAGEMENT=proxy_m8l \
    --replicas 1 \
    --network gogolist-net \
    --hostname crawler_worker_advertising_api_call \
    --with-registry-auth \
    --constraint "node.id == e1e8a2795195oxjebn78lem04" \
    docker.gogolistassist.com/crawlerengine_worker:latest celery -A crawler_engine worker -l info -Q update_supplier,amazon_advertising_call -n update_supplier -P eventlet -c 100