qcount

'''Quality Counter - 5/ago/2020

Counts how many "Good", "Bad" and "Whatever" images are in each subfolder.
Run this from the folder containing the subfolders with the labeled images (as generated by quality-labeler.py)

Sample output:
    Jersey_1 | Good: 337  | Bad: 181  | +-: 149  | 51% 27% 22% (337/667)
    Jersey_2 | Good: 116  | Bad: 141  | +-: 37   | 39% 48% 13% (116/294)
    Jersey_3 | Good: 309  | Bad: 116  | +-: 73   | 62% 23% 15% (309/498)
       Total | Good: 762  | Bad: 438  | +-: 259  | 52% 30% 18% (762/1459)

    Puruna_1 | Good: 62   | Bad: 776  | +-: 37   | 7.1% 89% 4.2% (62/875)
    Puruna_2 | Good: 89   | Bad: 1132 | +-: 75   | 6.9% 87% 5.8% (89/1296)
    Puruna_3 | Good: 70   | Bad: 1656 | +-: 67   | 3.9% 92% 3.7% (70/1793)
       Total | Good: 221  | Bad: 3564 | +-: 179  | 5.6% 90% 4.5% (221/3964)

      USP-P1 | Good: 161  | Bad: 50   | +-: 124  | 48% 15% 37% (161/335)
      USP-P2 | Good: 211  | Bad: 56   | +-: 43   | 68% 18% 14% (211/310)
      USP-P3 | Good: 230  | Bad: 67   | +-: 45   | 67% 20% 13% (230/342)
       Total | Good: 602  | Bad: 173  | +-: 212  | 61% 18% 21% (602/987)

       Total | Good: 1585 | Bad: 4175 | +-: 650  | 25% 65% 10% (1585/6410)
'''

import os
import fnmatch
import shutil
import re
from itertools import groupby


MODE = 1  # 0 = All subfolders, 1 = Predetermined split
COUNT_AUG = False  # Counts Horizontally Flipped folders created for Data Augmentation
AUG_SUFFIX = "_aug"  # Which folders are used is defined later
COUNT_MIDS = True  # Counts labels over 1
GROUP_MIDS = True  # Groups labels over 1 as being 0
NUM_CLASSES = 4
SPLIT_KEY = 'Z'



local_count = part_count = total_count = [0] * NUM_CLASSES
DF = './'  # Data folder


# split lists. train, valid, test
splits_dict = {}

# full sample
splits_dict['X'] = [[DF+'Puruna_1', DF+'Jersey_3', DF+'USP-P2'],
                    [DF+'Puruna_3', DF+'Jersey_2', DF+'USP-P3'],
                    [DF+'Puruna_2', DF+'Jersey_1', DF+'USP-P1']]

splits_dict['Z'] = [[DF+'Puruna_3', DF+'Jersey_3', DF+'Nelore~USP-P3'],
                    [DF+'Puruna_2', DF+'Jersey_1', DF+'Nelore~USP-P2'],
                    [DF+'Puruna_1', DF+'Jersey_2', DF+'Nelore~USP-P1']]

splits_dict['Z_part'] = [[DF+'Puruna_1', DF+'Jersey_2', DF+'Nelore~USP-P1']]

splits_dict['G'] = [[DF+'Puruna_1', DF+'Jersey_2', DF+'Nelore~USP-P2'],
                    [DF+'Puruna_3', DF+'Jersey_1', DF+'Nelore~USP-P3'],
                    [DF+'Puruna_2', DF+'Jersey_3', DF+'Nelore~USP-P1']]


# intra-sensor
splits_dict['IS1'] = [[DF+'Jersey_1-P1', DF+'Puruna_1'],
                      [DF+'Jersey_1-P2', DF+'Puruna_2'],
                      [DF+'Jersey_1-P3', DF+'Puruna_3']]

# intra-classe
splits_dict['IC1'] = [[DF+'Jersey_1'],
                      [DF+'Jersey_2'],
                      [DF+'Jersey_3']]

splits_dict['IC2'] = [[DF+'Puruna_1'],
                      [DF+'Puruna_2'],
                      [DF+'Puruna_3']]


# test. small sample para carregar rapido e testar mudancas no codigo
splits_dict['test'] = [[DF+'Nelore~USP-P1'],
                       [DF+'Nelore~USP-P2'],
                       [DF+'Nelore~USP-P3']]


if MODE == 0:
    subfolders_init = [f.path for f in os.scandir(DF) if f.is_dir()]
    subfolders_init.sort()
    if not COUNT_AUG:
        subfolders_init = [i for i in subfolders_init if not i.endswith(AUG_SUFFIX)]
    subfolders = [list(i) for j, i in groupby(subfolders_init, lambda a: re.split('|'.join(map(re.escape, ('_', '-'))), a.split('/')[-1], 0)[0])]   # Splits on _ and -

if MODE == 1:
    subfolders = splits_dict[SPLIT_KEY]
    if COUNT_AUG:  # Looks for hflip folders and adds them
        for split in subfolders:
            for s in split:
                if os.path.isdir(s + AUG_SUFFIX):  # Check if hflip folder exists
                    split.append(s + AUG_SUFFIX)


def print_results(folder_name, counter):
    good = counter[1]
    bad = counter[0]
    whatever = counter[2]
    artifact = counter[3]

    total = good+bad
    if COUNT_MIDS:
        total += whatever
        total += artifact

    good_ratio = (float(good)/total)*100
    if COUNT_MIDS:
        if GROUP_MIDS:
            bad += whatever
            bad += artifact
            bad_ratio = (float(bad)/total)*100
            print("%20s | Good: %-4d | Bad: %-4d | %.4g%% %.4g%% (%d/%d)" % (folder_name, good, bad, good_ratio, bad_ratio, good, total))
        else:
            bad_ratio = (float(bad)/total)*100
            whatever_ratio = (float(whatever)/total)*100
            artifact_ratio = (float(artifact)/total)*100
            print("%20s | Good: %-5d | Bad: %-5d | +-: %-5d | A: %-5d | %.2g%% %.2g%% %.2g%% %.2g%% (%d/%d)" % (folder_name, good, bad, whatever, artifact, good_ratio, bad_ratio, whatever_ratio, artifact_ratio, good, total))
    else:
        bad_ratio = (float(bad)/total)*100
        print("%20s | Good: %-4d | Bad: %-4d | %.4g%% %.4g%% (%d/%d)" % (folder_name, good, bad, good_ratio, bad_ratio, good, total))

for split in subfolders:
    part_count = [0] * NUM_CLASSES

    for s in split:
        list_files = []
        for root, dirnames, filenames in os.walk(s):
            for filename in fnmatch.filter(filenames, '*.[Jj][Pp][Gg]'):  # Case insensitive with regards to the file extension
                list_files.append(os.path.join(root, filename))
        list_files.sort()

        local_count = [0] * NUM_CLASSES

        for f in list_files:
            local_count[int(f.split('/')[-2])] += 1

        for i, pos_count in enumerate(local_count):
            part_count[i] += pos_count

        if sum(local_count) > 0:  # Ignores empty subfolders
            print_results(s.split('/')[-1], local_count)

    if sum(part_count) > 0:
        print_results("Total", part_count)
        print()
    for i, pos_count in enumerate(part_count):
        total_count[i] += pos_count

if sum(total_count) > 0:
    print_results("Total", total_count)