pool-health-check.py

#!/usr/bin/python
import json
import pycurl
import pprint
import socket
from StringIO import StringIO
import paramiko
import time
response_buffer = StringIO()

ethos_distro_handle = pycurl.Curl()
ethos_distro_handle.setopt(ethos_distro_handle.URL,'http://<Change>.ethosdistro.com/?json=yes')
ethos_distro_handle.setopt(ethos_distro_handle.WRITEFUNCTION, response_buffer.write)
ethos_distro_handle.perform()
ethos_distro_handle.close()
ethos_distro_json = json.loads(response_buffer.getvalue())
for rig in ethos_distro_json['rigs']:
    if ethos_distro_json['rigs'][rig]['condition'] == 'unreachable':
        print('Rig %s (%s) is Unreachable\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
        try:
            rig_ssh_connection = paramiko.SSHClient()
            rig_ssh_connection.load_system_host_keys()
            rig_ssh_connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            rig_ssh_connection.connect(ethos_distro_json['rigs'][rig]['ip'],22,'ethos','live',timeout=5)
            rig_ssh_connection.exec_command('r')
            rig_ssh_connection.close()
        except:
            print('Could not SSH to rig %s (%s)\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
    elif ethos_distro_json['rigs'][rig]['condition'] == 'stuck_miners':
        print('Rig %s (%s) has stuck miners. Will reboot between 22:00 and 06:00 ET.\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
        if int(time.strftime("%H")) > 20 or int(time.strftime("%H")) < 7:
            try:
                rig_ssh_connection = paramiko.SSHClient()
                rig_ssh_connection.load_system_host_keys()
                rig_ssh_connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
                rig_ssh_connection.connect(ethos_distro_json['rigs'][rig]['ip'],22,'ethos','live',timeout=5)
                rig_ssh_connection.exec_command('r')
                rig_ssh_connection.close()
            except:
                print('Could not SSH to rig %s (%s)\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
    elif ethos_distro_json['rigs'][rig]['condition'] == 'overheat' or ethos_distro_json['rigs'][rig]['condition'] == 'throttle':
        print('Rig %s (%s) overheated or throttled. Will clear thermals between 22:00 and 06:00 ET.' % (rig, ethos_distro_json['rigs'][rig]['ip']))
        if int(time.strftime("%H")) > 20 or int(time.strftime("%H")) < 7:
            try:
                rig_ssh_connection = paramiko.SSHClient()
                rig_ssh_connection.load_system_host_keys()
                rig_ssh_connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
                rig_ssh_connection.connect(ethos_distro_json['rigs'][rig]['ip'],22,'ethos','live',timeout=5)
                rig_ssh_connection.exec_command('clear-thermals && update')
                rig_ssh_connection.close()
            except:
                print('Could not SSH to rig %s (%s)\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
    elif ethos_distro_json['rigs'][rig]['condition'] == 'adl_error':
       print('Rig %s (%s) is reporting ADL error. Clearing flag.\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
        try:
            rig_ssh_connection = paramiko.SSHClient()
            rig_ssh_connection.load_system_host_keys()
            rig_ssh_connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            rig_ssh_connection.connect(ethos_distro_json['rigs'][rig]['ip'],22,'ethos','live',timeout=5)
            rig_ssh_connection.exec_command('echo "" > /var/run/adl_error.file')
            rig_ssh_connection.close()
        except:
            print('Could not SSH to rig %s (%s)\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
    elif ethos_distro_json['rigs'][rig]['condition'] == 'no_hash':
        print('Rig %s (%s) is Not Mining, Restarting Proxy\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
        try:
            rig_ssh_connection = paramiko.SSHClient()
            rig_ssh_connection.load_system_host_keys()
            rig_ssh_connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            rig_ssh_connection.connect(ethos_distro_json['rigs'][rig]['ip'],22,'ethos','live',timeout=5)
            rig_ssh_connection.exec_command('restart-proxy')
            rig_ssh_connection.close()
        except:
            print('Could not SSH to rig %s (%s)\n' % (rig, ethos_distro_json['rigs'][rig]['ip']))
    else:
        print('Rig %s is OK' % (rig))