Depfinder

#!/usr/bin/python3

# Copyright (c) 2013-2016 Lena Svensson
# (reversed, insert dot and at): com gmail tehnegal
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.


########################################################################
########################################################################
##                                                                    ##
##  This script searches for dependencies on systems using pacman     ##
##  package manager.                                                  ##
##                                                                    ##
##  Warning! First time this script is run it creates a cache, which  ##
##  might take a terribly long time. If you have an old machine with  ##
##  only one cpu core, expect 30 minutes... If you have 2 or more     ##
##  cpu's, it starts several subprocecss, which doesn't stop by       ##
##  pressing ctrl + C.                                                ##
##                                                                    ##
########################################################################
########################################################################


import os
import re
import sys
import getopt
import subprocess
from multiprocessing import Pool, cpu_count

opts,args = getopt.getopt(sys.argv[1:], 'dhlrs')

LIST = 1
DETAILED = 0
SUPER_DETAILED = 0
REVERSED = 0

for opt in opts:
    for o in opt:
        if o == '-h':
            usage()
            sys.exit()
        elif o == '-d':
            DETAILED = 1
            LIST = 0
        elif o == '-l':
            LIST = 1
        elif o == '-r':
            REVERSED = 1
            LIST = 0
        elif o == '-s':
            SUPER_DETAILED = 1
            LIST = 0


def usage():
    print("Usage: depfinder [option] [pkg]")
    print("Options: ")
    print("\t -d see a detailed list of dependencies")
    print("\t -l see a list of dependencies")
    print("\t -s see a super detailed list of dependencies")
    print("\t -r reversed dependency search, find pkgs that depend on a pkg")
    print("")
    print("Without option: defaults to '-l'")


try:
    qpkg = sys.argv[-1]
except IndexError:
    print('This script needs at least one agrument, the package to be examined.')
    usage()
    sys.exit(1)

if len(sys.argv) <= 1:
    usage()
    sys.exit(1)


def create_cache(name_ver_rel):
    """
    Create a cache for the binaries of all installed packages.
    Without it, this script would take terribly long time.
    """
    print("Creating cache for " + name_ver_rel)

    cachefile_ = os.path.join(cachedir, name_ver_rel)
    with  open(cachefile_, 'w+') as cachefile:
        pacman_file = os.path.join(pacmandb, name_ver_rel, 'files')
        with open(pacman_file) as f:
            files = f.read().splitlines()
            files_to_check = []
            for line in files:
                if line == '%FILES%':
                    continue
                elif line == '%BACKUP%':
                    break
                elif line[-1:] == '/':
                    continue

                # No need to check for binaries in these directories
                elif line[:11] == 'usr/include':
                    continue
                elif    line[:13] == 'usr/share/doc':
                    continue
                elif    line[:13] == 'usr/share/man':
                    continue
                elif    line[:14] == 'usr/share/i18n':
                    continue
                elif    line[:15] == 'usr/share/fonts':
                    continue
                elif    line[:16] == 'usr/share/locale':
                    continue
                elif    line[:15] == 'usr/share/icons':
                    continue
                elif    line[:17] == 'usr/share/pixmaps':
                    continue
                elif    line[:7] == 'usr/src':
                    continue
                elif    line[:3] == 'etc':
                    continue
                elif    line[:3] == 'var':
                    continue
                else:
                    files_to_check.append(line)

        for file_ in files_to_check:
            if not re.search("lib/ld-[0-9]\.[0-9].*\.so", file_):
                file_ = '/' + file_
                if os.path.isfile(file_):
                    output = subprocess.check_output(['file', file_])
                    output = str(output)
                    if re.search('ELF', output):
                        if re.search('dynamically', output):
                            if not re.search('Windows', output):
                                if not re.search('statically', output):
                                    if not re.search('kernel', output):
                                        cachefile.write(file_ + '\n')


def find_new_pkgs():
    '''
    Returns a list of new packages installed in the system,
    that should be added to the cache.
    '''
    if not os.path.isdir(cachedir):
        os.makedirs(cachedir)

    # Find out what's already in the cache
    cached = subprocess.check_output(['ls', cachedir]).splitlines()

    # Remove cachefiles from removed packages
    for cached_pkg in cached:
        cached_pkg = cached_pkg.decode('ascii')
        cached_pkg_file = os.path.join(cachedir, cached_pkg)
        db_dir = os.path.join(pacmandb, cached_pkg)
        if not os.path.isdir(db_dir):
            os.remove(cached_pkg_file)

    # Find installed packages that's not in cache
    new_pkgs = []
    for inst_pkg in installed:
        cachefile = os.path.join(cachedir, inst_pkg)
        if not os.path.isfile(cachefile):
            new_pkgs.append(inst_pkg)

    return new_pkgs


def create_dbs():
    '''
    Create a database of owners of all files in filesystem.
    Create another database of packages names and versions.
    '''
    global pkgowners_db, installed_db

    pkgowners_db = {}
    installed_db = {}

    for name_ver_rel in installed:
        remove = re.search('.*-', name_ver_rel)
        rel = name_ver_rel[remove.end():]
        remove = re.search('-' + rel + '$', name_ver_rel)
        name_ver = name = name_ver_rel[:remove.start()] + name_ver_rel[remove.end():]

        remove = re.search('.*-', name_ver)
        ver = name_ver[remove.end():]

        remove = re.search('-' + ver, name_ver)
        name = name_ver[:remove.start()] + name_ver[remove.end():]
        ver_rel = ver + '-' + rel

        # Add it to the db of names and versions of all packages
        installed_db[name] = ver_rel

        pacman_file = os.path.join(pacmandb, name_ver_rel, 'files')
        with open(pacman_file) as f:
            files = f.read().splitlines()
            for line in files:
                if line == '%FILES%':
                    continue
                elif len(line) == 0:
                    break
                elif line[-1] == '/':
                    continue
                else:
                    pkgowners_db[line] = name


def installed_check():
    '''
    Check if the queried package is installed
    '''
    if not qpkg in installed_db:
        print("Package {} is not installed and can't be examined.".format(qpkg))
        sys.exit(1)


def bin_check():
    '''
    Check if the queried package has dynamically linked binaries
    '''
    if len(qbinaries) == 0:
        print("This package doesn't have any dynamically linked binaries,")
        print("so no dependencies can be found by this script.")
        exit()

def lib_check():
    '''
    Check if the queried package has shared libraries
    '''
    for line in qbinaries:
        if re.search('\.so', line):
            return True
        else:
            continue

def find_libs():
    '''
    Find all libraries the package links to
    '''
    for binary in qbinaries:
        if SUPER_DETAILED:
            print("")
            print(binary)

        (dirname, filename) = os.path.split(binary)
        try:
            output = subprocess.check_output(['scanelf', '-BF', "%n", binary]).decode('ascii').split(',')
        except subprocess.CalledProcessError as e:
            problems.append(binary)
            continue

        for lib in output:
            # Remove the ' /path/to/binary' after last lib
            if re.search(' /*', lib):
                remove = re.search(' /*', lib)
                lib = lib[:remove.start()]

            # In case nothing was found and lib is now empty
            if not lib:
                continue

            if lib not in libraries:
                libraries.append(lib)

            if SUPER_DETAILED:
                lib1 = 'usr/lib/' + lib
                lib2 = 'lib/' + lib
                found = 0
                for lib_test in lib1, lib2:
                    if lib_test in pkgowners_db:
                        if not pkgowners_db[lib_test] == qpkg:
                            owner = pkgowners_db[lib_test]
                            # Freetype-infinality should be shown as freetype
                            if owner == 'freetype-infinality':
                                owner = 'freetype'
                            print('    /{} \t{}'.format(lib_test, owner))
                            found = 1

                if not found:
                    # Check if it's qpkg's own library
                    file_test = '/var/lib/pacman/local/' +  qname_ver_rel + '/files'
                    try:
                        output = subprocess.check_output(['grep', lib, file_test]).decode('ascii').splitlines()
                        for i in output:
                            owner = pkgowners_db[i]
                            # Freetype-infinality should be shown as freetype
                            if owner == 'freetype-infinality':
                                owner == 'freetype'
                            print('    /{} \t{}'.format(i, owner))
                            found = 1
                    except subprocess.CalledProcessError as e:
                            problems.append(e)

                # If still not found, add it to missing
                if not found:
                        missing.append(lib)


def output():
    owners = []
    # Test if the library is found in /lib or /usr/lib
    for lib in libraries:
        lib1 = 'usr/lib/' + lib
        lib2 = 'lib/' + lib
        found = 0
        for lib_test in lib1, lib2:
            if lib_test in pkgowners_db:
                pkg = pkgowners_db[lib_test]
                # Freetype-infinality should be shown as freetype
                if pkg == 'freetype-infinality':
                    pkg = 'freetype'

                if DETAILED:
                    owners.append('(' + pkg + ') /' + lib_test)
                else:
                    if not pkgowners_db[lib_test] == qpkg:
                        owners.append(pkg)
                found = pkg
        if not found:
            # Check if the package itself has that library, otherwise add it to missing
            file_test = '/var/lib/pacman/local/' +  qname_ver_rel + '/files'
            try:
                output = subprocess.check_output(['grep', lib, file_test]).decode('ascii').splitlines()
            except subprocess.CalledProcessError as e:
                missing.append(lib)
                continue

            for i in output:
                pkg = pkgowners_db[i]
                if DETAILED:
                    # Freetype-infinality should be shown as freetype
                    if pkg == 'freetype-infinality':
                        pkg == 'freetype'

                    owners.append('(' + pkg + ') /' + i)
                found = pkg

        if not found:
            missing.append(lib)

    if DETAILED:
        owners.sort()
        for i in owners:
            print(i)
    elif LIST:
        unique = []
        for i in owners:
            if i not in unique:
                unique.append(i)
        unique.sort()
        for i in unique:
            print(i, end=' ')
        print("")


def rev_depsearch(name_ver_rel):
    '''
    Find out which packages depend on the queried package
    '''
    if name_ver_rel == qname_ver_rel:
        # Don't check if qpkg depends on itself...
        return

    # Extract the name of the package
    rel = name_ver_rel.split('-')[-1]
    ver = name_ver_rel.split('-')[-2]
    ver_rel = '-' + ver + '-' + rel
    remove = re.search(ver_rel, name_ver_rel)
    name = name_ver_rel[:remove.start()]

    # Find all binaries belonging to the package to be examined
    cachefile = os.path.join(cachedir, name_ver_rel)
    with open(cachefile) as f:
        binaries = f.read().splitlines()

    # Find out what the binaries link to
    for b in binaries:
        (dirname, filename) = os.path.split(b)
        try:
            output = subprocess.check_output(['scanelf', '-BF', "%n", b]).decode('ascii').split(',')
        except subprocess.CalledProcessError as e:
            # It's a later problem...
            problems.append(e)
            continue

        for lib in output:
            # Remove the ' /path/to/binary' after last lib
            if re.search(' /*', lib):
                remove = re.search(' /*', lib)
                lib = lib[:remove.start()]

            lib1 = 'usr/lib/' + lib
            lib2 = 'lib/' + lib
            found = 0

            # Test if the lib is found in /lib or /usr/lib
            for lib_test in lib1, lib2:
                if lib_test in pkgowners_db:
                    # See if qpkg owns it
                    if pkgowners_db[lib_test] == qpkg:
                        # Freetype-infinality should be shown as freetype
                        if name == 'freetype-infinality':
                            name = 'freetype'
                        print(name)
                        found = 1

                        # If a package anyway depends on qpkg, don't show it
                        # in 'some dependencies not found'-list.
                        if name in missing:
                            missing.remove(name)

                        # Go on to next package
                        return


HOME = os.getenv("HOME")
cachedir = HOME + '/.cache/depfinder'
pacmandb = '/var/lib/pacman/local'
installed_ = subprocess.check_output(['ls', pacmandb]).splitlines()
installed = []

for name_ver_rel in installed_:
    name_ver_rel = name_ver_rel.decode('ascii')
    if name_ver_rel != 'ALPM_DB_VERSION':
        installed.append(name_ver_rel)

# Update cache  (Uses all cpu cores.)
new_pkgs = find_new_pkgs()
cpus = cpu_count() + 1
pool = Pool(cpus)
pool.map(create_cache, new_pkgs)

create_dbs()
installed_check()

qname_ver_rel = qpkg + '-' + installed_db[qpkg]
cachefile = os.path.join(cachedir, qname_ver_rel)
with open(cachefile, 'r') as cachefile:
    qbinaries = cachefile.read().splitlines()

bin_check()

if REVERSED:
    if not lib_check():
        print("This package doesn't have any shared libraries")
        sys.exit(0)

libraries = []
problems = []
missing = []

if REVERSED:
    print("Searching for packages that depend on {}... ".format(qpkg))
    poolb = Pool(cpus)
    poolb.map(rev_depsearch, installed)
else:
    # SUPER_DETAILED prints directly from find_libs
    find_libs()

if LIST or DETAILED:
    output()

if missing:
    print("")
    print("Some dependencies not found.")
    for m in missing:
        print(m)

if problems:
    print("")
    for p in problems:
        print(p)