Mike Lively

<?php
/**
 * PHP TAP Test Harness 1_0_0_BETA
 * Aggragates any stand alone testing library capable of outputting TAP
 * compatible results.
 *
 * To use, place in the same directory as the tests you would like to
 * run (any subdirectories will also be run.)
 *
 * There are two constants that can be used to control the testing
 * environment:
 *
 * TAP_PHP_CLI - The path to your php-cli executable
 * TAP_PHP_CLI_ARGS - Any command line arguments to pass to php-cli.
 *     (-dinclude_path will be of particular use.)
 *
 * If there are files in the directory structure that you would like to
 * ignore, you can add them to the $_EXCLUDE_FILES array.
 *
 * This software is licensed under a variant of the BSD license.
 * For more information see: http://www.digitalsandwich.com/license.bsd
 *
 * Copyright (c) 2006, Mike Lively <http://digitalsandwich.com>
 * All Rights Reserved
 *//**
 * Configuration Options
 */
define('TAP_PHP_CLI', '/usr/bin/php');
define('TAP_PHP_CLI_ARGS', '-dinclude_path=.');

$_EXCLUDE_FILES = array('test-more.php', 'test-harness.php');

/// END OF CONFIGURATION \\\

define('TAP_TEST_DIRECTIVE_SKIP', 1);
define('TAP_TEST_DIRECTIVE_TODO', 2);

define('TAP_TEST_STATUS_OK', 1);
define('TAP_TEST_STATUS_NOTOK', 2);

define('TAP_TEST_FILE_STATUS_PASS', 1);
define('TAP_TEST_FILE_STATUS_FAIL', 2);
define('TAP_TEST_FILE_STATUS_SKIP', 3);

//verbosity bitmasks
define('TAP_VERBOSITY_SILENT', 0);
define('TAP_VERBOSITY_DEFAULT', 0x01);
define('TAP_VERBOSITY_DETAIL', 0x02);
define('TAP_VERBOSITY_DEBUG', 0x04);

$_BASE_DIR = dirname(__FILE__);

parse_command_arguments();

$session = init_test_session_stats();

run_test_dir($_BASE_DIR, $session);

//Print out Skip summary.
if (count($session['skip_test_files'])) {
    print_in_mode("Skipped Tests                  Total Skip Skipped List of Skipped\n");
    print_in_mode("----------------------------------------------------------------------\n");
    foreach ($session['skip_test_files'] as $test_file) {
        $file = relativize_filename($test_file['file']);
        $total = $test_file['plan'];
        $skip = $test_file['skip'];
        $skipped = $total?round($skipped / $total * 100, 2):0;
        print_in_mode(str_pad($file, 30, ' ') . ' ');
        print_in_mode(str_pad($total, 5, ' ', STR_PAD_LEFT) . ' ');
        print_in_mode(str_pad($skip, 4, ' ', STR_PAD_LEFT) . ' ');
        print_in_mode(str_pad('%' . $skipped, 7, ' ', STR_PAD_LEFT));

        if ($test_file['status'] == TAP_TEST_FILE_STATUS_SKIP) {
            print_in_mode(' ' . $test_file['reason'] . "\n");
        } else {
            $first = true;
            foreach ($test_file['skip_tests'] as $test) {
                if ($first) {
                    $first = false;
                    print_in_mode(' ' . "{$test['name']}: {$test['reason']}\n");
                } else {
                    print_in_mode(str_repeat(' ', 49));
                    print_in_mode(' ' . "{$test['name']}: {$test['reason']}\n");
                }
            }
        }
        print_in_mode("\n");
    }
}

//Print out Bonus summary.
if (count($session['bonus_test_files'])) {
    print_in_mode("Bonus Tests                    Total Bonus Bonuses List of Bonuses\n");
    print_in_mode("----------------------------------------------------------------------\n");
    foreach ($session['bonus_test_files'] as $test_file) {
        $file = relativize_filename($test_file['file']);
        $total = $test_file['plan'];
        $bonus = $test_file['bonus'];
        $bonuses = $total?round($bonus / $total * 100, 2):0;
        print_in_mode(str_pad($file, 30, ' '));
        print_in_mode(str_pad($total, 5, ' ', STR_PAD_LEFT));
        print_in_mode(str_pad($bonus, 5, ' ', STR_PAD_LEFT));
        print_in_mode(str_pad('%' . $bonuses, 7, ' ', STR_PAD_LEFT));

        $first = true;
        foreach ($test_file['bonus_tests'] as $test) {
            if ($first) {
                $first = false;
                print_in_mode(' ' . "{$test['name']}: {$test['reason']}\n");
            } else {
                print_in_mode(str_repeat(' ', 50));
                print_in_mode(' ' . "{$test['name']}: {$test['reason']}\n");
            }
        }

        print_in_mode("\n");
    }
}

//Print out Fail summary.
if (count($session['fail_test_files'])) {
    print_in_mode("Failed Tests                   Total Fail Failed List of Failed\n");
    print_in_mode("----------------------------------------------------------------------\n");
    foreach ($session['fail_test_files'] as $test_file) {
        $file = relativize_filename($test_file['file']);
        $total = $test_file['plan'];
        $fail = $test_file['plan'] - $test_file['ok'];
        $failed = $total?round($fail / $total * 100, 2):0;
        print_in_mode(str_pad($file, 30, ' '));
        print_in_mode(str_pad($total, 5, ' ', STR_PAD_LEFT));
        print_in_mode(str_pad($fail, 4, ' ', STR_PAD_LEFT));
        print_in_mode(str_pad('%' . $failed, 6, ' ', STR_PAD_LEFT));

        if ($test_file['status'] == TAP_TEST_FILE_STATUS_FAIL && $test_file['reason'] != '') {
            print_in_mode(' ' . $test_file['reason'] . "\n");
        } else {
            $first = true;
            foreach ($test_file['fail_tests'] as $test) {
                if ($first) {
                    $first = false;
                    print_in_mode(' ' . "{$test['name']}\n");
                } else {
                    print_in_mode(str_repeat(' ', 48));
                    print_in_mode(' ' . "{$test['name']}\n");
                }
            }
        }
        print_in_mode("\n");
    }
}

//Print out total Summary
$all_files = $session['t_count'];
$failed_files = $session['t_fail'];
$skipped_files = $session['t_skip'];
$passed_files = $session['t_pass'];

$all_tests = $session['count'];
$passed_tests = $session['ok'];
$failed_tests = $session['count'] - $session['ok'];
$skipped_tests = $session['skip'];
$bonus_tests = $session['bonus'];

if ($bonus_tests) {
    print_in_mode("$bonus_tests TODO tests passed. You may want to consider removing their TODO status.\n");
}

if ($skipped_files || $skipped_tests) {
    if ($skipped_files) print_in_mode("$skipped_files test scripts skipped.  ");
    if ($skipped_tests) print_in_mode("$skipped_tests tests skipped.  ");
    print_in_mode("\n");
}

if ($failed_files) {
    print_in_mode("Failed $failed_files/$all_files test scripts, %" . round(($all_files?$passed_files/$all_files:0) * 100, 2) . " okay. ");
} else {
    print_in_mode("All test scripts passed!  ");
}

if ($failed_tests) {
    print_in_mode("Failed $failed_tests/$all_tests subtests, %" . round(($all_files?$passed_tests/$all_tests:0) * 100, 2) . " okay. ");
} else {
    print_in_mode("All subtests passed!  ");
}

print_in_mode("\n");
if (!$failed_files && !$failed_tests) {
    return 1;
} else {
    return 0;
}

/// LIBRARY FUNCTIONS \\\

function parse_command_arguments() {
    $argv = $_SERVER['argv'];
    $GLOBALS['current_verbosity'] = TAP_VERBOSITY_DEFAULT;

    while (count($argv)) {
        $argument = array_shift($argv);

        switch ($argument) {
            case '--silent':
            case '-s':
                $GLOBALS['current_verbosity'] = TAP_VERBOSITY_SILENT;
            break;
            case '--detail':
            case '-v':
                $GLOBALS['current_verbosity'] |= TAP_VERBOSITY_DETAIL;
            break;
            case '--debug':
                $GLOBALS['current_verbosity'] |= TAP_VERBOSITY_DEBUG;
            break;
        }
    }
}

function print_in_mode($line, $mode = TAP_VERBOSITY_DEFAULT) {
    if ($GLOBALS['current_verbosity'] & $mode) {
        echo "$line";
    }
}

function run_test_dir($dirname, &$session) {
    $dirname = rtrim($dirname, '/') . '/';
    if (!is_dir($dirname)) {
        print_in_mode("$dirname: Is not a valid directory\n", TAP_VERBOSITY_DEFAULT);
        return;
    }

    if (!$dh = opendir($dirname)) {
        print_in_mode("$dirname: Could not open directory\n", TAP_VERBOSITY_DEFAULT);
        return;
    }

    $fileArray = array();
    $dirArray = array();
    while (($file = readdir($dh)) !== false) {
        if (!in_array($file, array_merge($GLOBALS['_EXCLUDE_FILES'], array('.', '..')))) {
            if (is_file($dirname . $file)) {
                $fileArray[] = $file;
            } elseif (is_dir($file)) {
                $dirArray[] = $file;
            }
        }
    }
    closedir($dh);

    sort($dirArray);
    sort($fileArray);

    foreach ($dirArray as $dir) {
        run_test_dir($dirname . $dir, $session);
    }

    foreach ($fileArray as $file) {
        ob_start();
        $test_file = run_test_file($dirname . $file);
        $test_buffer = ob_get_clean();
        switch ($test_file['status']) {
            case TAP_TEST_FILE_STATUS_PASS:
                print_in_mode(str_pad(relativize_filename($test_file['file']), 50, '.') . "ok\n");
            break;
            case TAP_TEST_FILE_STATUS_FAIL:
                print_in_mode(str_pad(relativize_filename($test_file['file']), 50, '.') . "not ok\n");
            break;
            case TAP_TEST_FILE_STATUS_SKIP:
                print_in_mode(str_pad(relativize_filename($test_file['file']), 50, '.') . "skipped\n");
            break;
        }
        echo $test_buffer;
        add_test_file_2_session($test_file, $session);
    }
}

function run_test_file($filename) {
    $test_file = init_test_file_stats();
    $test_file['file'] = $filename;

    $time_start = microtime(true);
    // Set up proc
    if (!is_executable(TAP_PHP_CLI)) {
        die("Couldn't find PHP exectuble. Please set TAP_PHP_CLI\n");
    }
    $command = TAP_PHP_CLI . ' ' . TAP_PHP_CLI_ARGS . ' ' . escapeshellarg($test_file['file']) . ' 2>&1';
    $descriptors = array(
        1 => array('pipe', 'w'),
    );
    $pipes = array();
    $cwd = dirname($test_file['file']);

    $process = proc_open($command, $descriptors, $pipes, $cwd);

    if (is_resource($process)) {
        process_test_results($pipes[1], $test_file);
        fclose($pipes[1]);
        proc_close($process);
    } else {
        $test_file['status'] = TAP_TEST_FILE_STATUS_FAIL;
        $test_file['reason'] = "Unable to execute the test file.";
    }
    $test_file['time'] = microtime(true) - $time_start;

    return $test_file;
}

function process_test_results($pipes, &$test_file) {
    //set up regexs
    $regex_plan = '/^1\.\.([0-9]+)(\s+#\s*(.*?)\s*)?$/';
    $regex_testline = '/^(ok|not ok)(\s+([0-9]+))?(\s+([^#]+))?(\s+#\s*(SKIP|TODO)\s*(.*?))?\s*$/i';
    $regex_diagnostics = '/^#\s*(.*?)\s*$/';
    $regex_bail = '/^bail out!\s*(.*?)\s*$/i';

    $is_first_line = true;
    $is_plan_in_middle = false;
    $is_plan_set = false;
    $current_test = 0;

    //expect pass
    $test_file['status'] = TAP_TEST_FILE_STATUS_PASS;
    while (!feof($pipes)) {
        $line = trim(fgets($pipes));

        print_in_mode("$line\n", TAP_VERBOSITY_DETAIL);
        if ($line == '') {
            //do nothing on empty lines
            continue;
        }

        if ($is_plan_in_middle) {
            print_in_mode("# Error: Plan found in middle of file. Please move to the begining or end.\n", TAP_VERBOSITY_DEFAULT);
            $is_plan_in_middle = false;
        }

        $matches = array();
        if (preg_match($regex_plan, $line, $matches)) {
            $plan = $matches[1];
            $skip = $matches[3];

            print_in_mode("Plan Line ($plan|$skip)\n", TAP_VERBOSITY_DEBUG);

            if ($is_plan_set) {
                print_in_mode("# Error: Multiple Plans Specified Using First Plan\n", TAP_VERBOSITY_DEFAULT);
            } else {
                $is_plan_set = true;
                $test_file['plan'] = $plan;
                if (!$is_first_line) {
                    $is_plan_in_middle = true;
                }

                if ($skip) {
                    $test_file['plan'] = 0;
                    $test_file['status'] = TAP_TEST_FILE_STATUS_SKIP;
                    $test_file['reason'] = $skip;
                    break;
                }
            }

        } elseif (preg_match($regex_testline, $line, $matches)) {
            $status = $matches[1];
            $number = $matches[3];
            $name = $matches[5];
            $directive = $matches[7];
            $reason = $matches[8];

            print_in_mode("Test Line ($status|$number|$name|$directive|$reason)\n", TAP_VERBOSITY_DEBUG);

            $current_test++;
            $test = init_test_stats();

            switch ($status) {
                case 'ok':
                    $test['status'] = TAP_TEST_STATUS_OK;
                break;
                case 'not ok':
                    $test['status'] = TAP_TEST_STATUS_NOTOK;
                break;
            }

            switch($directive) {
                case 'TODO':
                    $test['directive'] = TAP_TEST_DIRECTIVE_TODO;
                break;
                case 'SKIP':
                    $test['directive'] = TAP_TEST_DIRECTIVE_SKIP;
                break;
            }

            $test['number'] = $number?$number:$current_test;
            $test['name'] = $name;
            $test['reason'] = $reason;

            add_test_2_test_file($test, $test_file);
        } elseif (preg_match($regex_diagnostics, $line, $matches)) {
            print_in_mode("Diagnostic Line\n", TAP_VERBOSITY_DEBUG);
            //do nothing
        } elseif (preg_match($regex_bail, $line, $matches)) {
            $reason = $matches[1];
            print_in_mode("Bail Out Line ($reason)\n", TAP_VERBOSITY_DEBUG);

            $test_file['status'] = TAP_TEST_FILE_STATUS_FAIL;
            $test_file['reason'] = $reason;
            break;
        }
        else
        {
            print_in_mode("# Error: Unrecognized output ($line)\n", TAP_VERBOSITY_DEFAULT);
            $test_file['status'] = TAP_TEST_FILE_STATUS_FAIL;
        }

    }

    //set plan if one does not exist
    if ($test_file['plan'] == 0) {
        $test_file['plan'] = $test_file['count'];
    } elseif ($test_file['count'] > $test_file['plan']) { //TODO: not sure if this is right.
        $test_file['plan'] = $test_file['count'];
    }
}

function init_test_stats() {
    return array(
        'status' => 0,
        'number' => 0,
        'name' => '',
        'directive' => 0,
        'reason' => '',
    );
}

function init_test_file_stats() {
    return array(
        'file' => '',
        'count' => 0,
        'ok' => 0,
        'skip' => 0,
        'bonus' => 0,
        'plan' => 0,
        'status' => 0,
        'reason' => '',
        'time' => 0,
        'all_tests' => array(),
        'fail_tests' => array(),
        'skip_tests' => array(),
        'bonus_tests' => array(),
    );
}

function init_test_session_stats() {
    return array(
        'count' => 0,
        'ok' => 0,
        'skip' => 0,
        'bonus' => 0,
        't_pass' => 0,
        't_skip' => 0,
        't_fail' => 0,
        't_count' => 0,
        'time' => 0,
        'all_test_files' => array(),
        'fail_test_files' => array(),
        'skip_test_files' => array(),
        'bonus_test_files' => array(),
    );
}

function add_test_2_test_file($test, &$test_file) {
    $test_file['count']++;
    $test_file['all_tests'][] = $test;

    if ($test['status'] == TAP_TEST_STATUS_OK ||
      $test['directive'] == TAP_TEST_DIRECTIVE_SKIP ||
      $test['directive'] == TAP_TEST_DIRECTIVE_TODO)    {
          $test_file['ok']++;
    } else {
        $test_file['fail_tests'][] = $test;
        $test_file['status'] = TAP_TEST_FILE_STATUS_FAIL;
    }

    if ($test['directive'] == TAP_TEST_DIRECTIVE_SKIP) {
        $test_file['skip']++;
        $test_file['skip_tests'][] = $test;
    }

    if ($test['directive'] == TAP_TEST_DIRECTIVE_TODO &&
      $test['status'] == TAP_TEST_STATUS_OK) {
          $test_file['bonus']++;
        $test_file['bonus_tests'][] = $test;
    }
}

function add_test_file_2_session($test_file, &$session) {
    $session['count'] += $test_file['plan'];
    $session['ok'] += $test_file['ok'];
    $session['skip'] += $test_file['skip'];
    $session['bonus'] += $test_file['bonus'];
    $session['time'] += $test_file['time'];
    $session['all_test_files'][] = $test_file;
    $session['t_count']++;

    switch ($test_file['status']) {
        case TAP_TEST_FILE_STATUS_PASS:
            $session['t_pass']++;
        break;
        case TAP_TEST_FILE_STATUS_FAIL:
            $session['t_fail']++;
        break;
        case TAP_TEST_FILE_STATUS_SKIP:
            $session['t_pass']++;
            $session['t_skip']++;
        break;
    }

    if (($test_file['plan'] > $test_file['ok']) || ($test_file['status'] == TAP_TEST_FILE_STATUS_FAIL)) {
        $session['fail_test_files'][] = $test_file;
    }

    if (($test_file['skip'] > 0) || ($test_file['status'] == TAP_TEST_FILE_STATUS_SKIP)) {
        $session['skip_test_files'][] = $test_file;
    }

    if ($test_file['bonus'] > 0) {
        $session['bonus_test_files'][] = $test_file;
    }
}

function relativize_filename($filename) {
    if (strpos($filename, $GLOBALS['_BASE_DIR']) === 0) {
        return trim(substr($filename, strlen($GLOBALS['_BASE_DIR'])), '/');
    }
    return $filename;
}
?>