BBPyProject: collect slog2 output and post somewhere

In app.py:

    def onGetDiagnostics(self, mode='recent'):
        try:
            from . import reporting
            diag = reporting.Diagnostics(mode)
            path, length = diag.package()
            tart.send('diagnosticsWritten', path=path, length=length)

            status = diag.send()
            tart.send('diagnosticsSent', status=status)

        except Exception as ex:
            text = traceback.format_exc()
            tart.send('traceback', text=text)

In reporting.py:

'''Collection of log files and other info for problem reporting.'''

import os
import re
import base64
import datetime as dt
import logging
import pprint
import shutil
import socket
import subprocess
import time
import traceback
import zipfile

import tart

SLOGGER2_FOLDER = '/tmp/slogger2'
MAX_SANDBOX_LEN = 63

try:
    # figure out what our Push target id is
    MANIFEST_PATH = 'app/META-INF/MANIFEST.MF'
    with open(MANIFEST_PATH) as f:
        for line in f:
            if line.startswith('Package-Name:'):
                PACKAGE_NAME = line.split(None, 1)[1].strip()
                break
except IOError:
    PACKAGE_NAME = 'TestApp'

ZIPPATH = 'tmp/{}.diag.zip'.format(PACKAGE_NAME)


class Diagnostics:
    def __init__(self, mode='recent'):
        self.mode = mode

        self.fzip = zipfile.ZipFile(ZIPPATH, 'w', zipfile.ZIP_DEFLATED)

        from http import client
        client.HTTPConnection.debuglevel = 1

        # You must initialize logging, otherwise you'll not see debug output.
        logging.basicConfig()
        logging.getLogger().setLevel(logging.DEBUG)
        requests_log = logging.getLogger('requests.packages.urllib3')
        requests_log.setLevel(logging.DEBUG)
        requests_log.propagate = True

        from bb.bps import BPS_FAILURE
        from . import netstatus as ns

        self.proxies = {}

        details = ns.netstatus_proxy_details_t()
        if ns.netstatus_get_proxy_details(details) != BPS_FAILURE:
            if details.http_proxy_host:
                authority = '{}:{}'.format(
                    details.http_proxy_host.decode('ascii'),
                    details.http_proxy_port)

                if details.http_proxy_login_user:
                    authority = '{}:{}@{}'.format(
                        details.http_proxy_login_user.decode('ascii'),
                        details.http_proxy_login_password.decode('ascii'),
                        authority)

                self.proxies['http'] = authority
                print('will use proxies', self.proxies)

        ns.netstatus_free_proxy_details(details)

        self.perimeter = os.environ['PERIMETER']
        self.slog2name = os.path.basename(os.getenv('SANDBOX') or os.getcwd())[:MAX_SANDBOX_LEN]
        print('perimeter', self.perimeter)
        print('slog2name', self.slog2name)


    def list_logs(self):
        '''Retrieve list of slog2 buffer sets written by the same group id
        as this app is running under, ordered by timestamp.'''
        mygid = os.getgid()

        logs = []
        for name in os.listdir(SLOGGER2_FOLDER):
            bpath = os.path.join(SLOGGER2_FOLDER, name)
            stats = os.stat(bpath)
            if stats.st_gid == mygid:
                logs.append((stats.st_mtime, bpath))

        return sorted(logs)


    def list_session_logs(self):
        '''Filter list of logs by our own process id so we include only the
        current run: assumes pid in filename.'''
        logs = []
        pid = str(os.getpid())
        pattern = '(' + re.escape(self.slog2name) + r'|TartStart\.so)\.(\d+)'
        for mtime, path in list_logs():
            match = re.search(pattern, path)
            if match:
                if match.group(2) == pid:
                    logs.append((mtime, path))
            else:
                print('warning: no pid in', path)

        return logs


    def add_logs(self, logs):
        CMD = ['/bin/slog2info', '-l']
        if self.mode == 'recent':
            logs = logs[-6:]
        for mtime, path in logs:
            try:
                text = subprocess.check_output(CMD + [path])
            except Exception as ex:
                text = traceback.format_exc()

            if isinstance(text, str):
                text = text.encode('utf-8')
            filename = os.path.basename(path)
            if filename.startswith(self.slog2name):
                filename = 'session' + filename[len(self.slog2name):]
            zinfo = zipfile.ZipInfo(filename + '.txt',
                time.localtime(mtime)[:6])
            self.fzip.writestr(zinfo, text)


    def add_logslog(self):
        '''Add logs/log file'''
        try:
            with open('logs/log', 'rb') as f:
                data = f.read()

            self.fzip.writestr('logs_log.txt', data)
        except IOError:
            pass
            # text = traceback.format_exc()
            # tart.send('traceback', text=text)


    def add_environ(self):
        text = '\n'.join('{:20} = {}'.format(k, v) for k, v in sorted(dict(os.environ).items()))
        self.fzip.writestr('environ.txt', text)
        tart.send('gotEnviron', text=text)


    def add_cwd_listing(self):
        CMD = ['ls', '-la']
        try:
            text = subprocess.check_output(CMD)
        except Exception as ex:
            text = traceback.format_exc()

        self.fzip.writestr('sandbox-listing.txt', text)


    def add_pps(self):
        lines = []
        for path in [
            '/pps/system/.all',
            '/pps/services/.all',
            '/pps/services/networking/.all',
            '/pps/services/networking/enterprise/.all',
            '/pps/services/networking/interfaces/.all',
            '/pps/services/networking/all/.all',
            '/pps/services/networking/all/interfaces/.all',
            '/pps/services/wifi/.all',
            '/pps/system/filesystem/local/.all',
            '/pps/system/filesystem/remote/.all',
            '/pps/system/filesystem/remote-enterprise/.all',
            '/pps/system/filesystem/removable/.all',
            ]:
            lines.append('{}:'.format(path))
            try:
                with open(path) as f:
                    text = f.read()
            except Exception as ex:
                text = traceback.format_exc()

            lines.append(text)

        self.fzip.writestr('pps.txt', '\n'.join(lines))


    def package(self):
        try:
            self.add_environ()
            self.add_cwd_listing()
            self.add_logslog()
            self.add_pps()

            logs = self.list_logs()
            # print('logs', logs)
            self.add_logs(logs)

        finally:
            self.fzip.close()

        return self.fzip.filename, os.stat(self.fzip.filename).st_size


    def send(self):
        import requests
        from requests.models import PreparedRequest
        from requests.structures import CaseInsensitiveDict

        # hot-patch requests to avoid problem with
        def prepare_headers(self, headers):
            if headers:
                headers = dict((name, value) for name, value in headers.items())
                self.headers = CaseInsensitiveDict(headers)
            else:
                self.headers = CaseInsensitiveDict()

        PreparedRequest.prepare_headers = prepare_headers


        from urllib import request
        print('urllib proxies', request.getproxies())

        path = self.fzip.filename

        headers = {'Content-Type': 'application/octet-stream',
             # 'content-transfer-encoding': 'binary',
             }

        proxyDict = self.proxies

        saved_to = 'failed!'
        data = open(path, 'rb').read()

        ts = dt.datetime.now().strftime('%Y%m%dT%H%M%S')
        name = ts + '-' + os.path.basename(path)

        # path = os.path.join('shared/Box/FooBar', name)
        # with open(path, 'wb') as f:
        #     f.write(data)
        #     print('wrote to', path)
        #     saved_to = path

        # NOTE: cannot write to /accounts/1000/shared or pshared/ although we can read

        # try:
        #     path = os.path.join('shared/misc', name)
        #     with open(path, 'wb') as f:
        #         f.write(data)
        #         print('wrote to', path)
        #         saved_to = path
        # except IOError:
        #     text = traceback.format_exc()
        #     # tart.send('traceback', text=text)

        #     tart.send('traceback', text=text)

        params = {'filename': name}

        try:
            print('trying server')
            url = 'http://example.com:8080/cgi-bin/savediag.py'
            r = requests.post(url,
                params=params,
                headers=headers,
                proxies=proxyDict,
                data=data, # base64.b64encode(data),
                timeout=32,
                )
            print('example.com', r.status_code)
            print('headers:', pprint.pformat(dict(r.headers.items())))
            print('body:', r.text)
            saved_to = 'example.com'

        except Exception as ex:
            text = traceback.format_exc()
            print('error', text)

        print('returning')
        return saved_to


# EOF

If you need netstatus, which isn't in tart/python/bb as of this moment:

'''Wrappers for netstatus routines'''

import ctypes
from ctypes import (c_bool, c_float, c_double, c_int, c_char, c_char_p, c_void_p, c_uint,
    POINTER, Structure, CFUNCTYPE)

from bb._wrap import _func, _register_funcs

NETSTATUS_INFO = 0x01

class netstatus_proxy_details_t(Structure):
    _fields_ = [
        ('http_proxy_host', c_char_p),
        ('http_proxy_port', c_int),
        ('https_proxy_host', c_char_p),
        ('https_proxy_port', c_int),
        ('ftp_proxy_host', c_char_p),
        ('ftp_proxy_port', c_int),
        ('http_proxy_login_user', c_char_p),
        ('http_proxy_login_password', c_char_p),
        ]

class netstatus_interface_list_t(Structure):
    _fields_ = [
        ('num_interfaces', c_int),
        ('interfaces', POINTER(c_char_p)),
        ]

NETSTATUS_INTERFACE_TYPE_UNKNOWN       = 0
NETSTATUS_INTERFACE_TYPE_WIRED         = 1
NETSTATUS_INTERFACE_TYPE_WIFI          = 2
NETSTATUS_INTERFACE_TYPE_BLUETOOTH_DUN = 3
NETSTATUS_INTERFACE_TYPE_USB           = 4
NETSTATUS_INTERFACE_TYPE_VPN           = 5
NETSTATUS_INTERFACE_TYPE_BB            = 6
NETSTATUS_INTERFACE_TYPE_CELLULAR      = 7
netstatus_interface_type_t = c_int

NETSTATUS_IP_STATUS_ERROR_UNKNOWN        = 0
NETSTATUS_IP_STATUS_OK                   = 1
NETSTATUS_IP_STATUS_ERROR_NOT_CONNECTED  = 2
NETSTATUS_IP_STATUS_ERROR_NOT_UP         = 3
NETSTATUS_IP_STATUS_ERROR_NOT_CONFIGURED = 4
NETSTATUS_IP_STATUS_ERROR_IP6_OFF        = 5
NETSTATUS_IP_STATUS_ERROR_NO_IP_ADDRESS  = 6
NETSTATUS_IP_STATUS_ERROR_NO_IP6_ADDRESS = 7
NETSTATUS_IP_STATUS_ERROR_NO_IP_GATEWAY  = 8
NETSTATUS_IP_STATUS_ERROR_NO_IP6_GATEWAY = 9
NETSTATUS_IP_STATUS_ERROR_NO_NAME_SERVER = 10
netstatus_ip_status_t = c_int

# BPS_API int netstatus_request_events(int flags);
# BPS_API int netstatus_stop_events(int flags);
# BPS_API int netstatus_get_domain();
# BPS_API int netstatus_get_availability(bool *is_available);
# BPS_API int netstatus_get_default_interface(char **interface);
netstatus_get_proxy_details = _func(c_int, POINTER(netstatus_proxy_details_t))
# BPS_API int netstatus_get_proxy_details_for_url(const char *url, const char *interface, netstatus_proxy_details_t *details);
netstatus_free_proxy_details = _func(None, POINTER(netstatus_proxy_details_t))
# BPS_API bool netstatus_event_get_availability(bps_event_t *event);
# BPS_API const char * netstatus_event_get_default_interface(bps_event_t *event);
# BPS_API bool netstatus_event_get_http_proxy_login_required(bps_event_t *event);
# BPS_API const char * netstatus_event_get_http_proxy_host(bps_event_t *event);
# BPS_API int netstatus_event_get_http_proxy_port(bps_event_t *event);
# BPS_API const char * netstatus_event_get_https_proxy_host(bps_event_t *event);
# BPS_API int netstatus_event_get_https_proxy_port(bps_event_t *event);
# BPS_API const char * netstatus_event_get_ftp_proxy_host(bps_event_t *event);
# BPS_API int netstatus_event_get_ftp_proxy_port(bps_event_t *event);
# BPS_API int netstatus_get_interfaces(netstatus_interface_list_t *interface_list);
# BPS_API void netstatus_free_interfaces(netstatus_interface_list_t *interface_list);
# BPS_API int netstatus_get_interface_details(const char *interface, netstatus_interface_details_t **details);
# BPS_API void netstatus_free_interface_details(netstatus_interface_details_t **details);
# BPS_API const char * netstatus_interface_get_name(netstatus_interface_details_t *details);
# BPS_API netstatus_interface_type_t netstatus_interface_get_type(netstatus_interface_details_t *details);
# BPS_API bool netstatus_interface_is_connected(netstatus_interface_details_t *details);
# BPS_API bool netstatus_interface_is_up(netstatus_interface_details_t *details);
# BPS_API netstatus_ip_status_t netstatus_interface_get_ip_status(netstatus_interface_details_t *details);
# BPS_API netstatus_ip_status_t netstatus_interface_get_ip6_status(netstatus_interface_details_t *details);
# BPS_API int netstatus_interface_get_num_ip_addresses(netstatus_interface_details_t *details);
# BPS_API const char * netstatus_interface_get_ip_address(netstatus_interface_details_t *details, int index);
# BPS_API const char * netstatus_interface_get_ip_address_netmask(netstatus_interface_details_t *details, int index);
# BPS_API const char * netstatus_interface_get_ip_broadcast_address(netstatus_interface_details_t *details);
# BPS_API int netstatus_interface_get_num_ip_gateways(netstatus_interface_details_t *details);
# BPS_API const char * netstatus_interface_get_ip_gateway(netstatus_interface_details_t *details, int index);
# BPS_API const char * netstatus_interface_get_link_address(netstatus_interface_details_t *details);
# BPS_API int netstatus_interface_get_num_name_servers(netstatus_interface_details_t *details);
# BPS_API const char * netstatus_interface_get_name_server(netstatus_interface_details_t *details, int index);
# BPS_API const char * netstatus_interface_get_search_domains(netstatus_interface_details_t *details);

from bb.bps import _dll
_register_funcs(_dll, globals(), True)