deep_get_size_of.py

def deep_get_size_of(o, ids=None, progress_callback=None):
    """Find the memory footprint of a Python object

    This is a recursive function that drills down a Python object graph
    like a dictionary holding nested dictionaries with lists of lists
    and tuples and sets.

    The sys.getsizeof function does a shallow size of only. It counts each
    object inside a container as pointer only regardless of how big it
    really is.
    https://code.tutsplus.com/tutorials/understand-how-much-memory-your-python-objects-use--cms-25609

    :param o: the object
    :param ids:
    :param progress_callback
    :return:
    """
    try:
        import collections.abc as collections_abc  # only works on python 3.3+
    except ImportError:
        import collections as collections_abc
    import sys
    if ids is None:
        ids = set()
    d = deep_get_size_of
    if id(o) in ids:
        return 0
    r = sys.getsizeof(o)
    ids.add(id(o))
    if isinstance(o, str):
        return r
    if isinstance(o, collections_abc.Mapping):
        total = 0
        count = len(o.iteritems())
        iterator = 0
        for k, v in o.iteritems():
            total += d(k, ids) + d(v, ids)
            if progress_callback is not None:
                iterator += 1.0
                progress_callback(iterator/count)
        return r + total
        # return r + sum(d(k, ids) + d(v, ids) for k, v in o.iteritems())
        # this single line does what the above lines do, but without calling progress_callback
    if isinstance(o, collections_abc.Container):
        total = 0
        count = len(o)
        iterator = 0
        for x in o:
            total += d(x, ids)
            if progress_callback is not None:
                iterator += 1.0
                progress_callback(iterator/count)
        return r + total
        # return r + sum(d(x, ids) for x in o)
    return r