CurrencyDetection.py

import numpy as np
import cv2
import os
import pytesseract
from PIL import ImageGrab
from screeninfo import get_monitors


# strg + k + c / k + u

dirname = os.path.dirname(__file__)
sift = cv2.xfeatures2d.SIFT_create()
bf = cv2.BFMatcher(cv2.NORM_L1, crossCheck=True)
pytesseract.pytesseract.tesseract_cmd = 'C:\\Program Files\\Tesseract-OCR\\tesseract.exe'

# matching threshold for detecting templates
threshold = 0.50

currency_path = os.path.join(dirname, 'TrainingImages/')
currency_images = []
currency_names = []
folder_contains = os.listdir(currency_path)
for cl in folder_contains:
    imgCur = cv2.imread(os.path.join(currency_path, cl))
    # imgCurGrey = cv2.cvtColor(imgCur, cv2.COLOR_BGR2GRAY)
    currency_images.append(imgCur)
    currency_names.append(os.path.splitext(cl)[0])

screens = get_monitors()
res_main_monitor = screens[0]
screen_w = res_main_monitor.width
screen_h = res_main_monitor.height

if screen_w & screen_h == 2560 & 1400:
    tw_loction_x = 200
    tw_location_y = 300
    tw_w = 1300
    tw_h = 1200
elif screen_w & screen_h == 1920 & 1080:
    tw_loction_x = 200
    tw_location_y = 200
    tw_w = 1420
    tw_h = 800
else:
    print("Wrong Resolution")

scrn_rec_img = []
match_list = []
match_results = []
scrn_rec_img = ImageGrab.grab(bbox=(tw_loction_x, tw_location_y, tw_w, tw_h))
scrn_rec_img_np = np.array(scrn_rec_img)
test_image = cv2.imread(os.path.join(
    dirname, 'TestingImages/TestImageAllCurrencys.png'))
# override screen recording with testing images
scrn_rec_img = test_image
# img 16 in chaos orb
curr_img = currency_images[15]
matches = cv2.matchTemplate(scrn_rec_img, curr_img, cv2.TM_CCOEFF_NORMED)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(matches)
match_y_loc, match_x_loc = np.where(matches >= threshold)
# draw rectangle
curr_width = curr_img.shape[1]
curr_height = curr_img.shape[0]
# max_loc[0]=width [1] = height bboxes are drawn from location top left to bottom right corner
rect_bbxs = []
for (x, y) in zip(match_x_loc, match_y_loc):
    rect_bbxs.append([x, y, curr_width, curr_height])
    rect_bbxs.append([x, y, curr_width, curr_height])
rect_bbxs, weights = cv2.groupRectangles(rect_bbxs, 1, 0.2)
# condenses the list of rectangles to signle detection per area
for (x, y, w, h) in rect_bbxs:
    cv2.rectangle(scrn_rec_img, (x - 4, y - 4), (x + w + 4, y + h + 4), (0, 255, 255), 2)

# print(rect_bbxs[0])

# currently only takes a sample image later will take a screenshot of active monitor
scrn_rec_img_gray = cv2.cvtColor(scrn_rec_img, cv2.COLOR_BGR2GRAY)
_, thresh1 = cv2.threshold(np.array(scrn_rec_img_gray), 220, 255, cv2.THRESH_BINARY_INV)
# _, thresh1 = cv2.threshold(np.array(scrn_rec_img_gray), 240, 255, cv2.THRESH_OTSU)
# thresh1 = cv2.adaptiveThreshold(scrn_rec_img_gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 11, 2)
# unpacks coordinates of bounding boxes previously detected with template matching. total of 2 boxes are stored in rect_bbx
for (x, y, w, h) in rect_bbxs:
    cropped = thresh1[y:y+h-32, x:x+w-23]
    scaled_up = cv2.pyrUp(cropped)
    # tried scaling up etc to maybe detect easier
    blurred = cv2.GaussianBlur(scaled_up, (5, 5), 0)
    edged = cv2.Canny(blurred, 50, 250, 255)
    # edgefinding with canny
    print(pytesseract.image_to_string(edged, config='digits'))
    # print found digits
    cv2.imshow('test', scaled_up)
    cv2.waitKey(0)


# blurred = cv2.GaussianBlur(scrn_rec_img_gray, (5, 5), 0)
# edged = cv2.Canny(blurred, 30, 40, 255)

# ret, thresh1 = cv2.threshold(scrn_rec_img_gray, 0, 255, cv2.THRESH_OTSU | cv2.THRESH_BINARY_INV)
# rect_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))
# dilation = cv2.dilate(thresh1, rect_kernel, iterations=1)
# contours, hierarchy = cv2.findContours(dilation, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
# img_cpy = scrn_rec_img_gray.copy()
# found_numbers = []
# for cnt in contours:
#     x, y, w, h = cv2.boundingRect(cnt)
#     # Drawing a rectangle on copied image
#     rect = cv2.rectangle(img_cpy, (x, y), (x + w, y + h), (0, 255, 0), 2)
#     # Cropping the text block for giving input to OCR
#     cropped = img_cpy[y:y + h, x:x + w]
#     text = pytesseract.image_to_string(cropped)
#     found_numbers.append(text)
# print(text)


added_cur = 0
# for bbox in rect_bbxs:
# cv2.imshow('record', cropped)
# cv2.waitKey()
cv2.destroyAllWindows


# def findDesc(currency_images):
#     descList = []
#     for img in currency_images:
#         _, des = sift.detectAndCompute(img, None)
#         descList.append(des)
#     return descList


# def idCurrency(frame, descList):
#     _, des_frame = sift.detectAndCompute(frame, None)
#     bf = cv2.BFMatcher()
#     for des_cur in descList:
#          matches = bf.match(des_cur, des_frame, k=2)
#          for m, n in matches:
#              if m.distance < 0.75 * n.distance:


# currency_descriptors = findDesc(currency_images)


# while(True):
#     scrn_rec_img = ImageGrab.grab(bbox=(tw_loction_x, tw_location_y, tw_w, tw_h))
#     scrn_rec_img_np = np.array(scrn_rec_img)
#     frame = cv2.cvtColor(scrn_rec_img_np, cv2.COLOR_BGR2GRAY)
#     cv2.imshow('test', frame)
#     cv2.waitKey(1)
# cv2.destroyAllWindows()


# test_image = cv2.cvtColor(test_image, cv2.COLOR_BGR2GRAY)

# kp_test, descrpt_test = sift.detectAndCompute(test_image, None)

# chaos_orb_image = cv2.imread(os.path.join(
#     dirname, 'TrainingImages/ChaosOrb.png'))
# chaos_orb_image = cv2.cvtColor(chaos_orb_image, cv2.COLOR_BGR2GRAY)

# kp_chaos, descrpt_chaos = sift.detectAndCompute(chaos_orb_image, None)
# matches = bf.match(descrpt_chaos, descrpt_test)
# matches = sorted(matches, key=lambda x: x.distance)

# img_1 = cv2.drawMatches(chaos_orb_image, kp_chaos, test_image, kp_test, matches[:10], chaos_orb_image, flags=2)
# cv2.imshow('test', img_1)
# cv2.waitKey(0)