pep382.py module sketch 2

# Performance note: this slows import of all builtin/frozen modules, and adds
# an isdir() check for each directory on sys.path that precedes the target.  If
# you have a lot of (directory) path entries and import a lot of stuff from
# far down the path, this may be very slow if (e.g.) those paths are over a
# network.  So, we operate in an "explicit only" mode by default, where only
# registered namespace packages get the extra processing this way.

__all__ = [
    'is_namespace', 'register_namespace', 'namespace_subpath',
    'iter_namespaces', 'extend_namespaces', 'meta_importer',
]

import sys, new, zipimport, os

try:
    _make_set = set
except NameError:
    _make_set = dict.fromkeys   # Python 2.3, use a dict instead of a set

if not hasattr(sys, 'namespace_packages'):
    sys.namespace_packages = _make_set([])

def is_namespace(pkgname):
    return pkgname in sys.namespace_packages

def register_namespace(pkgname):
    if not is_namespace(pkgname):
        sys.namespace_packages.update(_make_set([pkgname]))
    # Make sure we can process at least the registered namespace(s)
    meta_importer.install(True)

def get_importer(entry):
    global ImpImporter, get_importer
    try:
        from pkgutil import get_importer, ImpImporter
    except ImportError:
        from pkg_resources import get_importer, ImpWrapper as ImpImporter
    ImpImporter.namespace_subpath = _fs_namespace_subpath
    return get_importer(entry)


def namespace_subpath(finder, fullname):
    if hasattr(finder, 'namespace_subpath'):
        return finder.namespace_subpath(fullname)
    elif isinstance(finder, zipimport.zipimporter):
        return _zip_namespace_subpath(finder, fullname)

def iter_namespaces(parent=''):
    """Iterate over registered namespaces that are a child of `parent`"""
    parent = parent.split('.')
    for name in sys.namespace_packages:
        path = name.split('.')
        path.pop()
        if path==parent:
            yield name

def extend_namespaces(path_entry, parent=''):
    """Extend imported namespaces' paths with subpaths of `path_entry`"""
    import imp
    imp.acquire_lock()
    try:
        finder = get_importer(path_entry)

        for name in iter_namespaces(parent):
            if name not in sys.modules:
                # only extend already-imported namespaces
                continue

            # Is there a portion for this namespace under path_entry?
            subpath = namespace_subpath(finder, name)
            if subpath is None:
                continue

            # Yes, so extend its __path__, and recurse to child namespaces
            path = getattr(sys.modules[name], '__path__', [])
            if subpath not in path:
                path.append(subpath)
                extend_namespaces(subpath, name)

    finally:
        imp.release_lock()

class NamespaceMetaImporter:
    """Implement PEP 382 namespace protocol for Python 2.x"""
    registration_required = True

    def find_module(self, name, path):
        is_ns = is_namespace(name)
        if self.registration_required and not is_ns:
            # We're only running for explicitly-registered packages, fall back
            # to normal import
            return None
        loader = None
        nspth = []
        for entry in path:
            finder = get_importer(entry)
            subpath = namespace_subpath(finder, name)
            if subpath is not None:
                parents.append(finder)
                nspth.append(subpath)
                is_ns = True
                if loader is None:
                    loader = finder.find_module(name)
                    loader = self._wrap_loader(loader, subpath, name)
            elif not is_ns:
                # only search normal modules until first ns found!
                loader = finder.find_module(name)
                if not nspth and loader is not None:
                    # Found regular module or package first, return it now
                    return loader
        if not nspth:
            # Nothing found, fall through to normal import process
            return None

        register_namespace(name)    # make sure we're in sys.namespace_packages
        nspkg = sys.modules.setdefault(name, new.module(name))
        nspkg.__path__ = nspth
        if loader is not None:
            return loader

        # Namespace package with no __init__ - return dummy loader
        return self

    def load_module(self, name):
        # We already initialized an empty namespace package, so just return it
        return sys.modules[name]

    def wrap_loader(self, loader, subpath, name):
        if loader is None:
            return None
        elif hasattr(loader, 'is_package') and not loader.is_package(name):
            # It's a module, not a package: ignore it
            return None
        finder = get_importer(subpath)
        subloader = finder.find_module(name+'.__init__')
        if subloader is None:
            # Guess it must be a module after all; skip it
            return None

        # Ok, now we have a parent loader (which we want to set as the
        # __loader__, if we set one), and a subloader, which we'll use to fetch
        # the __init__ code from, assuming it has a get_code() method:

        if isinstance(finder, ImpImporter) or not hasattr(subloader, 'get_code'):
            # Use as-is.  This should work for pkgutil and pkg_resources'
            # ImpLoader classes, or anything else that wraps imp in similar
            # fashion, but could have weird consequences for specialized
            # importers.  (OTOH, they probably don't support checking namespace
            # existence, so we probably won't get here for them anyway!)
            return subloader

        # This should works for zipimport and most other things with a get_code
        return CodeLoader(loader, subloader, name+'.__init__')

    def install(self, require_registration=False):
        """Put this meta-importer on sys.meta_path"""
        if self not in sys.meta_path:
            # We install at the end, so other meta-hooks take precedence
            sys.meta_path.append(self)
        if not require_registration:
            self.require_registration = False



    def iter_modules(self, prefix=''):
        # This is just here to make sure that an appropriate iter_modules()
        # hook is registered for zipimporter objects
        try:
            from pkgutil import iter_importer_modules, ImpImporter
        except ImportError:
            pass
        else:
            iter_importer_modules.register(ImpImporter, _fs_iter_modules)
            iter_importer_modules.register(
                zipimport.zipimporter, _zip_iter_modules
            )
        return []


meta_importer = NamespaceMetaImporter()


class CodeLoader(object):
    """Run __init__ code in package, without erasing __path__"""

    def __init__(self, loader, subloader, subname):
        self.loader = loader
        self.subloader = subloader
        self.subname = subname

    def load_module(self, fullname):
        pkg = sys.modules[fullname]
        if self.loader is not None:
            pkg.__loader__ = self.loader
        code = self.subloader.get_code(self.subname)
        fn = getattr(code, 'co_filename', None)
        if fn:
            pkg.__file__ = fn
        exec code in pkg.__dict__
        return pkg





def _fs_namespace_subpath(imp_importer, fullname):
    subname = fullname.split('.')[-1]
    prefix = os.path.join(imp_importer.path, subname)
    if os.path.isdir(prefix):
        for filename in listdir(prefix):
            if filename.endswith('.ns'):
                return prefix
    return None # not a namespace

def _zip_namespace_subpath(zipimporter, fullname):
    subname = fullname.split('.')[-1]
    prefix = zipimporter.prefix + subname
    for filename in zipimport._zip_directory_cache[zipimporter.archive]:
        if filename.endswith('.ns') and os.path.dirname(filename)==prefix:
            return zipimporter.archive + os.sep + prefix
    return None # not a namespace

def _fs_iter_modules(self, prefix=''):
    # Namespace-enhanced iter_modules for filesystem imports

    if self.path is not None and os.path.isdir(self.path):
        from re import compile
        # XXX should support unicode identifiers on Python 3
        isident = re.compile(r'^[A-Za-z_][A-Za-z_0-9]*$').match

        for base, dirs, files in os.walk(self.path):
            if base==self.path:
                # Walk possible packages
                dirs[:] = (d for d in dirs if isident(d))
            else:
                # Don't traverse subdirs of those packages
                del dirs[:]

                for f in files:
                    if f.endswith('.ns'):
                        yield prefix + os.path.basename(base), True

        # Now add the standard modules and packages
        for t in self.iter_modules(prefix):
            yield t

def _zip_iter_modules(importer, prefix=''):
    # Namespace-enhanced iter_modules for zip imports

    dirlist = zipimport._zip_directory_cache[importer.archive].keys()
    dirlist.sort()
    _prefix = importer.prefix
    plen = len(_prefix)
    yielded = {}
    import inspect
    for fn in dirlist:
        if not fn.startswith(_prefix):
            continue
        fn = fn[plen:].split(os.sep)
        if len(fn)==2 and (
            fn[1].startsswith('__init__.py') or fn[1].endswith('.ns')
        ):
            if fn[0] not in yielded:
                yielded[fn[0]] = 1
                yield prefix + fn[0], True
        if len(fn)!=1:
            continue
        modname = inspect.getmodulename(fn[0])
        if modname=='__init__':
            continue
        if modname and '.' not in modname and modname not in yielded:
            yielded[modname] = 1
            yield prefix + modname, False