Auto Detect Language.py

#!python.exe
import pickle, re
from os import path

__author__ = 'Charles Grunwald (Juntalis) <[email protected]>'

def __setup_configobj__(deps, info):
    """ Setup configobj either in the lib folder or the local folder """
    from os import unlink, mkdir
    import shutil

    # Download and unzip
    tmpfile = deps.download(info['url'], info['filename'])
    tmpfolder = path.splitext(tmpfile)[0]
    deps.unzip(tmpfile, tmpfolder)
    unlink(tmpfile)

    modfolder = path.join(tmpfolder, 'configobj-4.7.2')
    modpath = [path.join(modfolder, 'configobj.py'), path.join(modfolder, 'validate.py')]

    (modpath, modconfigobj) = deps.install(modpath, 'configobj')
    shutil.rmtree(tmpfolder, True)
    return (modconfigobj, modpath)

""" Our dependencies """
__dependencies__ = {
    'configobj':
        (__setup_configobj__, {
            'filename' : 'configobj.zip',
            'url' : 'http://www.voidspace.org.uk/downloads/configobj-4.7.2.zip'
        })
}

class ScriptDeps:
    """ Simple class to install any script dependencies we don't have at startup. """
    __modules__ = {}

    def __init__(self, dependencies):
        deps = dependencies
        for dep in deps.keys():
            (setup, info) = deps[dep]
            self.__modules__[dep] = setup(self, info)

    def get(self, name):
        if self.__modules__.has_key(name):
            return self.__modules__[name]
        return None

    def unzip(self, filename, dir):
        """ Extract zip file: filename to folder: dir. """
        import zipfile, os
        from cStringIO import StringIO
        zf = zipfile.ZipFile( filename )
        namelist = zf.namelist()
        dirlist = filter( lambda x: x.endswith( '/' ), namelist )
        filelist = filter( lambda x: not x.endswith( '/' ), namelist )
        # make base
        pushd = os.getcwd()
        if not path.isdir( dir ):
            os.mkdir( dir )
        os.chdir( dir )
        # create directory structure
        dirlist.sort()
        for dirs in dirlist:
            dirs = dirs.split( '/' )
            prefix = ''
            for dir in dirs:
                dirname = path.join( prefix, dir )
                if dir and not path.isdir( dirname ):
                    os.mkdir( dirname )
                prefix = dirname
        # extract files
        for fn in filelist:
            try:
                out = open( fn, 'wb' )
                buffer = StringIO( zf.read( fn ))
                buflen = 2 ** 20
                datum = buffer.read( buflen )
                while datum:
                    out.write( datum )
                    datum = buffer.read( buflen )
                out.close()
            except:
                import sys
                sys.stderr.write('Error while unzipping %s..\n' % filename)
        os.chdir( pushd )

    def download(self, url, filename):
        """ Download dependencies specified by dictionary __dependencies__"""
        from urllib import urlretrieve as download
        from tempfile import gettempdir as tempdir

        # Iterate through dependencies, downloading and moving.
        result = path.join(tempdir(), filename)
        download(url, result)
        return result

    def _install_global(self, modpath, modname):
        # install to the lib folder under Notepad++'s plugin directory.
        import shutil
        libdir = path.join(notepad.getNppDir(), 'plugins', 'PythonScript', 'lib')
        modconfigobj = None
        try:
            # install to pythonscript lib folder.
            for modfile in modpath:
                shutil.move(modfile, libdir)
        except:
            (modpath, modconfigobj) = self._install_user(modpath, modname)
        else:
            modconfigobj = __import__(modname)
            modpath = path.join(libdir, '%s.py' % modname)
        return (modpath, modconfigobj)

    def _install_user(self, modpath, modname):
        # install to User's AppData PythonScript/lib
        import shutil
        libdir = path.join(path.abspath(path.dirname(path.dirname(__file__))), 'lib')
        if not path.exists(libdir) or not path.isdir(libdir):
            mkdir(libdir)
        for modfile in modpath:
            shutil.move(modfile, libdir)
        modconfigobj = __import__(modname)
        modpath = path.join(libdir, '%s.py' % modname)
        return (modpath, modconfigobj)

    def _install_stub(self, libdir):
        initfile = path.join(libdir, '__init__.py')
        if not path.exists(initfile):
            f = open(initfile,'w')
            f.write('# Stub')
            f.close()
        return initfile

    def install(self, modpath, modname, install_global=False):
        if install_global:
            return self._install_global(modpath, modname)
        return self._install_user(modpath, modname)

class SimpleCache(dict):
    """
    Simple local cache.
    It saves local data in singleton dictionary with convenient interface

    Downloaded from http://code.activestate.com/recipes/577492-simple-local-cache-and-cache-decorator/

    Author: Andrey Nikishaev
    License: GPL
    Copyright 2010, http://creotiv.in.ua
    """

    def __new__(cls,*args):
        if not hasattr(cls,'_instance'):
            cls._instance = dict.__new__(cls)
        else:
            raise Exception('SimpleCache already initialized')
        return cls._instance

    @classmethod
    def getInstance(cls):
        if not hasattr(cls,'_instance'):
            cls._instance = dict.__new__(cls)
        return cls._instance

    def get(self,name,default=None):
        """Multilevel get function.
        Code:
        Config().get('opt.opt_level2.key','default_value')
        """
        if not name:
            return default
        levels = name.split('.')
        data = self
        for level in levels:
            try:
                data = data[level]
            except:
                return default

        return data

    def set(self,name,value):
        """Multilevel set function
        Code:
        Config().set('opt.opt_level2.key','default_value')
        """
        levels = name.split('.')
        arr = self
        for name in levels[:-1]:
            if not arr.has_key(name):
                arr[name] = {}
            arr = arr[name]
        arr[levels[-1]] = value

    def getset(self,name,value):
        """Get cache, if not exists set it and return set value
        Code:
        Config().getset('opt.opt_level2.key','default_value')
        """
        g = self.get(name)
        if not g:
            g = value
            self.set(name,g)
        return g

def scache(func):
    def wrapper(*args, **kwargs):
        cache = SimpleCache.getInstance()
        fn = "scache." + func.__module__ + func.__class__.__name__ + \
             func.__name__ + str(args) + str(kwargs)
        val = cache.get(fn)
        if not val:
            res = func(*args, **kwargs)
            cache.set(fn,res)
            return res
        return val
    return wrapper


# Try to import configobj module. If we cant, download it and set it up.
try:
    import configobj
    has_configobj = True
except ImportError:
    notepad.messageBox('Could not find module "configobj". Downloading and setting it up now..', 'Dependencies')
    deps = ScriptDeps(__dependencies__)
    (configobj, configobj_path) = deps.get('configobj')
    has_configobj = configobj is not None
    if has_configobj:
        notepad.messageBox('Module "configobj" setup successfully. You can find it at:\n\n%s' % configobj_path, 'Setup Successful')
    else:
        notepad.messageBox('Error: Could not import configobj.\nDownload at: http://www.voidspace.org.uk/python/configobj.html', 'Import Error')
        exit()


class LanguageAutoDetector:
    """ Main class """
    __log = None
    __config = None
    __config_path = None
    __cache = None
    __detections = [
        'filename',
        'partial_filename',
        'xml',
        'shebang',
        'try_shebang',
        'contains_string'
        #'regex'
    ]
    __cache_ignore = {
        'contains_string' : None,
        'try_shebang' : 'shebang',
        'partial_filename' : 'filename',
        'regex' : None
    }

    def __init__(self, config_file=None):
        config = self.__load_config(config_file)
        if config['cache']['enabled']:
            self.__load_cache()
        #self.__log = logger.FileLogger()


    def __new__(cls,*args):
        if not hasattr(cls,'_instance'):
            cls._instance = dict.__init__(cls, args)
        else:
            raise Exception('LanguageAutoDetector already initialized')
        return cls._instance

    @classmethod
    def getInstance(cls):
        if not hasattr(cls,'_instance'):
            cls._instance = dict.__new__(cls)
        return cls._instance

    @scache
    def config(self, key=None, default=None):
        if self.__config is None:
            config = self.__load_config()
        else:
            config = self.__config

        if key is None:
            return config

        return self.__getdict(config, key, default)

    @scache
    def cache(self, key=None, default=None):
        if self.__cache is None:
            cache = self.__load_cache()
        else:
            cache = self.__cache

        if key is None:
            return cache

        return self.__getdict(cache, key, default)

    def set_lang(self, result, bufferID):
        ret = False
        lang = self.__test_language(result)
        if lang is None:
            if self.config('errors.invalid_lang'):
                notepad.messageBox('Error: Specified language %s invalid.' % result, 'Config Error')
        else:
            notepad.setLangType(lang, bufferID)
            ret = True
        return ret

    def detect(self, args):
        detections = self.config('detections.order')
        bufferID = args['bufferID']
        args['filename'] = path.basename(notepad.getBufferFilename(bufferID))
        notepad.activateBufferID(bufferID)
        for detection in detections:
            func = getattr(self, 'detection_%s' % detection.lower())
            result = func(args)
            console.write('Detection %s: %s\n' % (detection.lower(),result))
            if result is not None:
                if self.set_lang(result, bufferID):
                    break

    # filename
    # partial_filename
    # xml
    # shebang
    # try_shebang
    # contains_string

    def detection_filename(self, args):
        filename = args['filename']
        config = self.config('detections.filename')
        for lang in config.keys():
            if filename in config[lang]:
                return lang
        return None

    @scache
    def detection_partial_filename(self, args):
        filename = args['filename']
        config = self.config('detections.partial_filename')
        for lang in config.keys():
            for pattern in config[lang]:
                rgx = re.compile("^%s$" % pattern, re.IGNORECASE)
                if rgx.match(filename):
                    # TODO: Cache here filename -> lang
                    return lang
        return None

    def detection_xml(self, args):
        # Check for xml stuff.
        xml_config = self.config('detections.xml')
        xml_filename = args['filename']
        self.xml_lang = None
        def check(contents, lineNumber, totalLines):
            val = contents.strip().lower()
            if len(val) == 0:
                return 1
            else:
                for pattern in xml_config:
                    pattern = pattern.lower()
                    if val.startswith(pattern):
                        self.xml_lang = 'xml'
                        ext = path.splitext(xml_filename)[1]
                        if len(ext) > 0:
                            xml_cache = { ext : 'xml' } # TODO: Cache here
            return totalLines - lineNumber
        editor.forEachLine(check)
        return self.xml_lang

    def detection_shebang(self, args):
        shebang = self.__getshebang()
        if shebang is None:
            return None
        config = self.config('detections.shebang')
        for lang in config.keys():
            for pattern in config[lang]:
                rgx = re.compile(r"^#!((?:/[^\s]+/env(?:\.[a-z]+)? |/[^\s]+/|[A-Z]:[^\s]*\\|[A-Z]:[^\s]*\\env\.[a-z]+ ?)?%s[\d.-_]*(?:\.[0-9a-z-_.])*)\b" % pattern, re.IGNORECASE)
                if rgx.search(shebang):
                    # TODO: Cache here shebang -> lang
                    return lang
        return None

    def detection_try_shebang(self, args):
        shebang = self.__getshebang()
        if shebang is None:
            return None
        rgx = re.compile(r"^#!(?:/[^\s]+/env(?:\.[a-z]+)? |/[^\s]+/|[A-Z]:[^\s]*\\|[A-Z]:[^\s]*\\env\.[a-z]+ ?)?([^\s]+)[\d.-_]*(?:\.[0-9a-z-_.])*\b", re.IGNORECASE)
        match = rgx.search(shebang)
        if match:
            result = match.group(1)
            lang = self.__test_language(result)
            if lang is None:
                return None
            return result
        return None

    def detection_contains_string(self, args):
        text = editor.getText()
        config = self.config('detections.contains_string')
        for lang in config.keys():
            for pattern in config[lang]:
                rgx = re.compile("%s" % pattern, re.IGNORECASE | re.MULTILINE)
                if rgx.search(text):
                    # TODO: Cache here shebang -> lang
                    return lang
        return None

    def detection_regex(self, args):
        pass

    def __getshebang(self):
        line = editor.getLine(0)
        if len(line) <= 2:
            return None
        if line[0:2] == '#!':
            return line
        return None

    def __getdict(self, dict, name, default=None):
        """Multilevel get function.
        Code:
        Config().get('opt.opt_level2.key','default_value')
        """
        if not name:
            return default
        levels = name.split('.')
        data = dict
        for level in levels:
            try:
                data = data[level]
            except:
                return default
        return data

    def __load_config(self, cfg=None):
        """ Load configuration for script. If it doesn't exist, write the default
            configuration to file. """

        # Figure out config path.
        if cfg is None:
            cfg = path.join(notepad.getPluginConfigDir(), 'py_autolang.cfg')
        self.__config_path = cfg
        if path.exists(cfg):
            self.__config = configobj.ConfigObj(cfg)
        else:
            self.__config = self.__default_config()
            self.__save_config()
        return self.__config

    def __default_config(self, cfg=None):
        """ Default configuration for script. """

        # Figure out config path.
        if cfg is None and self.__config_path is None:
            cfg = path.join(notepad.getPluginConfigDir(), 'py_autolang.cfg')
        elif cfg is None and self.__config_path is not None:
            cfg = self.__config_path

        config = configobj.ConfigObj()
        config.filename = cfg

        # Main config
        config['script'] = {}
        config['script']['enabled'] = True
        config['script']['autoload'] = True

        # Errors
        config['errors'] = {}
        config['errors']['invalid_lang'] = True # Message box on invalid lexer specified.
        config['errors']['invalid_lexer'] = True # Message box on invalid lang specified.

        # Script cache
        config['cache'] = {}
        config['cache']['enabled'] = True
        cache_folder = path.abspath(path.join(path.dirname(cfg), 'cache'))
        valid_folder = False
        while not valid_folder:
            if path.exists(cache_folder):
                if not path.isdir(cache_folder):
                    import string, random
                    cache_folder += '-' + ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(3))
                else:
                    valid_folder = True
            else:
                from os import mkdir
                mkdir(cache_folder)
        config['cache']['folder'] = cache_folder

        # Logging
        config['logging'] = {}
        config['logging']['console'] = False
        config['logging']['console_auto_open'] = False
        config['logging']['file'] = False

        # Loading event
        config['file_load_event'] = {}
        config['file_load_event']['no_extension'] = True
        config['file_load_event']['default_lexer'] = True
        config['file_load_event']['always'] = False


        """
            Detection methods order - This tells what order to use detection methods.
            So if filename is executed before shebang, and filename matches a detection,
            the shebang line wont be run. To disable a detection method, set to 0. If
            any two methods have the same number, an error will be thrown.

            Possible detections:
                shebang
                try_shebang
                xml
                filename
                partial_filename
                contains_string
                regex - Not yet implemented
        """
        order = ['filename', 'partial_filename', 'xml', 'shebang', 'try_shebang', 'contains_string']

        # Default Detections
        ## Shebangs
        ## These all match the follow regex pattern:
        ### r"^#!(?:/[^\s]+/env(?:\.[a-z]+) |/[^\s]+/|[A-Z]:[^\s]*\\|[A-Z]:[^\s]*\\env\.[a-z]+ ?)?%s[\d.-_]*(?:\.[0-9a-z-_.])*\b" % key
        shebang = {

            # Shell
            ## Bash/Korn Shell/C Shell/Z Shell/etc
            'bash' : ['(?:[czk]|ba)?sh'],

            # Python
            ## CPython - http://python.org/
            ## Cross Twine Linker (xtpython) - http://crosstwine.com/linker/python.html
            ## Unpython Python to C compiler (unpython) - http://code.google.com/p/unpython/
            ## IPython (ipython) - http://ipython.org/
            ## PyPy - http://pypy.org/
            ## Iron Python - http://ironpython.net/
            ## Mozilla Embedded Python Console - http://www.thomas-schilz.de/MozPython/
            ## TinyPy - http://www.tinypy.org/
            ## Snipy - Personal project, you can remove if you want.
            ## Enthought SciPy distribution - http://www.enthought.com/
            ## Jython (jython) - http://www.jython.org/
            ## Cython (Optimizing Python to C Compiler) - http://cython.org/
            ## Typhon (typhon) - https://github.com/vic/typhon
            ## Mython (mython) - http://mython.org/

            # Languages Close Enough to Python """
            ## Nimrod  - http://force7.de/nimrod/download.html
            ## Serpent - http://sourceforge.net/projects/serpent/
            ## Boo - http://boo.codehaus.org/

            'python' : [
                '(?:xt|un|i)?pythonw?',
                '(?:py|i|moz|tiny|sni)pyw?(?:-c)?',
                'epdw?',
                '[jctm]ythonw?',
                '(?:nimrod|ser?pent)',
                'boo(?:c|i|ish)?'
            ],

            # Perl
            ## Perl - http://www.perl.org/
            ## Parrot - http://parrot.org/
            ### Hm, this one might be hard. Leaving parrot as perl for now
            'perl' : [ 'w?perl', 'parrot' ],

            # Ruby
            ## Ruby - http://www.ruby-lang.org/en/
            ## Iron Ruby (ir, etc) - http://ironruby.net/
            ## JRuby - http://jruby.org/
            ## Ruby on Rails - http://rubyonrails.org/
            'ruby' : [
                '[ji]?[ie]r[wbi]{0,2}?(?:_swing)?',
                '[ej]?ruby[wc]?',
                'rake'
            ],

            # Javascript
            ## Node.Js - http://nodejs.org/
            ## Narwhal - https://github.com/tlrobinson/narwhal
            ## JSDB - http://www.jsdb.org/
            ## Ringo Javascript - http://ringojs.org/
            ## GlueScript - http://gluescript.sourceforge.net/
            ## Rhino - http://www.mozilla.org/rhino/
            'javascript' : [
                '(?:node|npm)',
                '(?:narwhal|tusk)',
                '(?:jsdb|ringo|gluew?)',
                '(?:rhino|js)'
            ],

            # PHP
            'php' : [
                '(?:i?php(?:-cgi|-cli|-win)?|pharc?)'
            ]

        }

        filename = {
            'bash' : ['configure'],
            'makefile' : ['Makefile']
        }

        partial_filename = {
            'makefile' : ['Makefile\..+']
        }

        contains_string = {
            'bash' : ['^mk_add_options', '^ac_add_options']
        }

        config['detections'] = {
            'order' : order,
            'filename' : filename,
            'partial_filename' : partial_filename,
            'xml' : ['<?xml ', '<!DOCTYPE'],
            'shebang' : shebang,
            'contains_string' : contains_string
        }

        return config

    def __save_config(self):
        self.__config.write()

    def __load_cache(self):
        config = self.config()
        folder = config['cache']['folder']

        cache = {}
        for detection in config['detections']['order']:
            if detection in self.__cache_ignore:
                if self.__cache_ignore[detection] is not None:
                    detection = self.__cache_ignore[detection]
                    if cache.has_key(detection):
                        continue
                else:
                    continue
            f = path.join(folder, detection)
            if path.exists(f) and path.isfile(f):
                input = open(f, 'rb')
                cache[detection] = {
                    'changed' : False,
                    'value' : pickle.load(input)
                }
                input.close()
            else:
                cache[detection] = {
                    'changed' : False,
                    'value' : None
                }
        self.__cache = cache
        return self.__cache

    def __save_cache(self):
        config = self.config()
        folder = config['cache']['folder']
        saved = []
        for detection in config['detections']['order']:
            if detection in self.__cache_ignore:
                if self.__cache_ignore[detection] is not None:
                    detection = self.__cache_ignore[detection]
                    if detection in saved:
                        continue
                else:
                    continue
            f = path.join(folder, detection)
            cache = self.cache(detection)
            if cache['changed'] and cache['value'] is not None:
                output = open(f, 'wb')
                pickle.dump(cache['value'], output)
                output.close()
                self.__cache[detection]['changed'] = False
            saved.append(detection)

    def __test_lexer(self, lexer):
        result = True
        old_lexer = editor.getLexerLanguage()
        editor.setLexerLanguage(lexer)
        if editor.getLexerLanguage() == 'null':
            result = False
        editor.setLexerLanguage(old_lexer)
        return result

    def __test_language(self, lang):
        import Npp
        lang = lang.upper()
        result = None
        try:
            result = getattr(Npp.LANGTYPE, lang)
        except AttributeError:
            result = None
        return result

detector = LanguageAutoDetector()
notepad.clearCallbacks([NOTIFICATION.FILEOPENED])
notepad.callback(detector.detect, [NOTIFICATION.FILEOPENED])