FMHY Dupe Checker Updated 1.9

"""
Written in Python. Run this command for dependencies:
pip install requests PyQt5 pyperclip fake-headers
"""

import requests
from fake_headers import Headers
import re
from threading import Thread, Event
from contextlib import suppress
from pyperclip import copy
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import pyqtSignal
from requests.exceptions import ReadTimeout, ConnectionError
import time
import csv

# fake headers
headers = Headers(headers=True)

# wiki scrape
elapsed = time.perf_counter()
url_regex = re.compile(r'(?:https?|ftp|file):\/\/(?:ww(?:w|\d+)\.)?((?:[\w_-]+(?:\.[\w_-]+)+)[\w.,@?^=%&:\/~+#-]*[\w@?^=%&~+-])')
wiki = set(re.findall(url_regex, requests.get("https://raw.githubusercontent.com/nbats/FMHYedit/main/single-page").text))
print(f'Wiki scraped in {time.perf_counter() - elapsed:0.4f} sec. Found {len(wiki)} links.')


def handle_req(url, item, callback):
    try:
        resp = requests.get(url, headers=headers.generate(), timeout=10)
        if resp is None: resp = 'Failed'
    except ReadTimeout:
        callback(url, 'Timeout', item)
    except ConnectionError:
        callback(url, 'Error', item)
    except Exception as e:
        callback(url, str(e), item)
    else:
        callback(url, resp, item)

def async_request(*args):
    thread = Thread(target=handle_req, args=args, daemon=True)
    thread.start()


class httpTestSignalWrapper(QtCore.QObject):
    signal = pyqtSignal(str, object, object)

class checkLinksSignalWrapper(QtCore.QObject):
    signal = pyqtSignal()


class Ui_MainWindow(object):
    group_url_regex = re.compile(r'((?:https?|ftp|file):\/\/(?:ww(?:w|\d+)\.)?)((?:[\w_-]+(?:\.[\w_-]+)+)[\w.,@?^=%&:\/~+#-]*[\w@?^=%&~+-])')

    def setupUi(self, MainWindow):
        # blob of code up ahead from pyuic5. needed to keep this in one file
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(750, 450)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.gridLayout = QtWidgets.QGridLayout(self.centralwidget)
        self.gridLayout.setContentsMargins(9, 3, 9, 0)
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.label = QtWidgets.QLabel(self.centralwidget)
        font = QtGui.QFont()
        font.setFamily("Segoe UI Variable Display")
        font.setPointSize(14)
        self.label.setFont(font)
        self.horizontalLayout.addWidget(self.label)
        spacerItem = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.horizontalLayout.addItem(spacerItem)
        self.label_2 = QtWidgets.QLabel(self.centralwidget)
        font = QtGui.QFont()
        font.setFamily("Segoe UI Variable Display")
        font.setPointSize(14)
        self.label_2.setFont(font)
        self.horizontalLayout.addWidget(self.label_2)
        self.gridLayout.addLayout(self.horizontalLayout, 0, 0, 1, 2)
        self.inputBox = QtWidgets.QPlainTextEdit(self.centralwidget)
        font = QtGui.QFont()
        font.setFamily("Calibri")
        font.setPointSize(11)
        self.inputBox.setFont(font)
        self.inputBox.setFrameShape(QtWidgets.QFrame.NoFrame)
        self.gridLayout.addWidget(self.inputBox, 1, 0, 1, 1)
        self.gridFrame = QtWidgets.QFrame(self.centralwidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.gridFrame.sizePolicy().hasHeightForWidth())
        self.gridFrame.setSizePolicy(sizePolicy)
        self.gridLayout_3 = QtWidgets.QGridLayout(self.gridFrame)
        self.gridLayout_3.setContentsMargins(1, 1, 1, 4)
        self.gridLayout_3.setVerticalSpacing(4)
        self.horizontalLayout_2 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_2.setContentsMargins(0, 0, 0, 0)
        self.copyDupes = QtWidgets.QPushButton(self.gridFrame)
        self.copyDupes.setEnabled(False)
        self.copyDupes.setFont(font)
        self.horizontalLayout_2.addWidget(self.copyDupes)
        self.copyValid = QtWidgets.QPushButton(self.gridFrame)
        self.copyValid.setEnabled(False)
        self.copyValid.setFont(font)
        self.horizontalLayout_2.addWidget(self.copyValid)
        self.copyTested = QtWidgets.QPushButton(self.gridFrame)
        self.copyTested.setEnabled(False)
        self.copyTested.setFont(font)
        self.horizontalLayout_2.addWidget(self.copyTested)
        self.checkSelected = QtWidgets.QPushButton(self.gridFrame)
        self.checkSelected.setVisible(False)
        self.checkSelected.setFont(font)
        self.horizontalLayout_2.addWidget(self.checkSelected)
        self.exportCsv = QtWidgets.QPushButton(self.gridFrame)
        self.exportCsv.setFlat(True)
        font = QtGui.QFont()
        font.setFamily("Segoe MDL2 Assets")
        font.setPointSize(13)
        self.exportCsv.setFont(font)
        self.exportCsv.setText("\uE896")
        self.exportCsv.setFixedSize(30, 30)
        self.horizontalLayout_2.addWidget(self.exportCsv)
        self.gridLayout_3.addLayout(self.horizontalLayout_2, 1, 0, 1, 1)
        self.outputTree = QtWidgets.QTreeWidget(self.gridFrame)
        self.outputTree.setFrameShape(QtWidgets.QFrame.NoFrame)
        self.outputTree.header().setSectionsMovable(False)
        self.outputTree.setColumnCount(3)
        self.outputTree.setColumnWidth(0, 50)
        self.outputTree.setColumnWidth(1, 180)
        # MainWindow.resizeEvent = lambda event: \
        #     self.outputTree.setColumnWidth(1, 100 + (event.size().width() - 750) / 2)
        # multi select tree
        self.outputTree.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
        self.outputTree.setRootIsDecorated(False)
        self.gridLayout_3.addWidget(self.outputTree, 0, 0, 1, 1)
        self.gridLayout.addWidget(self.gridFrame, 1, 1, 1, 1)
        MainWindow.setCentralWidget(self.centralwidget)

        # status code font
        self.status_font = QtGui.QFont()
        self.status_font.setFamily("Calibri")
        self.status_font.setPointSize(10)

        # status code colors
        self.status_colors = {
            range(200, 300): '#31cd64',
            range(300, 400): '#33a7ff',
            range(400, 500): '#fda92a',
            range(500, 600): '#fc4f52',
        }

        # connections
        self.copyDupes.clicked.connect(lambda: copy('\n'.join(self.duped_links)))
        self.copyValid.clicked.connect(lambda: copy('\n'.join(self.valid_links)))
        self.copyTested.clicked.connect(lambda: copy('\n'.join(self.getTestedLinks())))
        self.exportCsv.clicked.connect(self.exportCsvDialog)
        self.checkSelected.clicked.connect(self._testSelectedLinks)
        self.inputBox.textChanged.connect(self.checkLinks)
        self.outputTree.itemSelectionChanged.connect(self.onSelection)
        self.http_test_sig = httpTestSignalWrapper()
        self.http_test_sig.signal.connect(self.finishTest)
        self.call_back_checkLinks = checkLinksSignalWrapper()
        self.call_back_checkLinks.signal.connect(self.checkLinks)

        self.testing_items = set()
        self.tested_items = {}
        self._is_free = Event()
        self._is_free.set()
        self.line_thread = None
        self._new_event = False

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def exportCsvDialog(self):
        file_dialog = QtWidgets.QFileDialog()
        file_dialog.setAcceptMode(QtWidgets.QFileDialog.AcceptSave)
        file_dialog.setNameFilter("CSV (*.csv)")
        file_dialog.setDefaultSuffix("csv")
        # show dialog
        if file_dialog.exec_():
            file_path = file_dialog.selectedFiles()[0]
            links = re.findall(self.group_url_regex, self.inputBox.toPlainText())
            try:
                with open(file_path, 'w', newline='') as csvfile:
                    writer = csv.writer(csvfile, dialect='excel', quoting=csv.QUOTE_MINIMAL)
                    writer.writerow(['Request URL', 'Unique?', '# Redirects', 'Status', 'Reason'])
                    for link in links:
                        full_link = ''.join(link)
                        if full_link in self.tested_items:
                            if type(self.tested_items[full_link]) is str:
                                _redirects = _status_code = ''
                                _reason = self.tested_items[full_link]
                            else:
                                _redirects = str(len(self.tested_items[full_link].history))
                                _reason = self.tested_items[full_link].reason
                                _status_code = '=CONCAT('+', " > ", '.join(
                                f'HYPERLINK("{r.url}", "{r.status_code}")'
                                for r in (
                                    *self.tested_items[full_link].history,
                                    self.tested_items[full_link])
                                )+')'
                        else:
                            _reason = _redirects = _status_code = ''
                        writer.writerow([
                            full_link,
                            'FALSE' if link[1] in wiki else 'TRUE',
                            _redirects,
                            _status_code,
                            _reason
                        ])
            except PermissionError:
                QtWidgets.QMessageBox.critical(self.centralwidget, "Error", "Permission denied")

    def getTestedLinks(self):
        return [
            l for l in self.tested_items
                if l in self.valid_links and type(self.tested_items[l]) is not str
                and self.tested_items[l].status_code in range(200, 300)
            ]

    def finishTest(self, url, resp, item):
        if url in self.testing_items:
            self.testing_items.remove(url)
            self.tested_items[url] = resp
            self.copyTested.setEnabled(True)
        try:
            item.text(2)
        except RuntimeError:
            return  # item was deleted
        widget = QtWidgets.QWidget()
        widget.setLayout(layout := QtWidgets.QHBoxLayout())
        layout.setContentsMargins(0, 0, 0, 0)
        layout.setSpacing(2)
        layout.setAlignment(QtCore.Qt.AlignLeft)
        self.outputTree.setItemWidget(item, 2, widget)
        item.setText(2, "")
        if type(resp) is str:
            label = QtWidgets.QLabel(f' {resp} ')
            label.setFont(self.status_font)
            label.setStyleSheet('background-color: #A12729; color: white; border-radius: 6px;')
            layout.addWidget(label)
            return
        for r in (*resp.history, resp):
            label = QtWidgets.QLabel(f" {r.status_code} ")
            color = next((self.status_colors[k] for k in self.status_colors if r.status_code in k), '#000000')
            label.setStyleSheet(f'background-color: {color}; color: white; border-radius: 6px;')
            label.setToolTip(r.url)
            label.setToolTipDuration(-1)
            label.setFont(self.status_font)
            layout.addWidget(label)

    def _testSelectedLinks(self):
        selected = self.getRanItems()
        self.testing_items.update([i.text(1) for i in selected])  # remember tested items
        for item in selected:
            item.setText(2, "Testing...")
        self.outputTree.clearSelection()
        self.checkSelected.setVisible(False)
        for item in selected:
            async_request(item.text(1), item, self.http_test_sig.signal.emit)

    def getRanItems(self):
        return [i for i in self.outputTree.selectedItems()
                if i.text(1) not in {*self.tested_items, *self.testing_items}]

    def onSelection(self):
        if selected := self.getRanItems():
            self.checkSelected.setText(QtCore.QCoreApplication.translate("MainWindow", f"Test ({len(selected)}) \U0001F50D"))
            self.checkSelected.setVisible(True)
        else:
            self.checkSelected.setVisible(False)

    def _waitForEvent(self):
        self._new_event = True
        self._is_free.wait()
        self.call_back_checkLinks.signal.emit()

    def checkLinks(self):
        if not self._is_free.is_set():
            if self.line_thread and self.line_thread.is_alive():
                return
            self.line_thread = Thread(target=self._waitForEvent, daemon=True)
            self.line_thread.start()
            return
        self._is_free.clear()
        text = self.inputBox.toPlainText()
        if text:
            self.inputBox.setPlaceholderText('')
        else:
            self.inputBox.setPlaceholderText(self._placeholderText)
        self.outputTree.clear()
        self.copyValid.setEnabled(False)
        self.copyDupes.setEnabled(False)
        self.copyTested.setEnabled(False)
        self.exportCsv.setEnabled(False)
        self.checkSelected.setVisible(False)

        links = re.findall(self.group_url_regex, text)
        self.valid_links, self.duped_links, self.tested_links = [], [], []
        # populate tree
        for n, link in enumerate(links):
            if self._new_event:
                self._new_event = False
                self._is_free.set()
                return
            item = QtWidgets.QTreeWidgetItem(self.outputTree)
            full_link = ''.join(link)
            item.setText(1, full_link)
            if len(links) > 100 and not n % 10:
                # process in chunks to allow for UI updates
                QtWidgets.QApplication.processEvents()
            with suppress(RuntimeError):
                if full_link in self.tested_items:
                    self.finishTest(full_link, self.tested_items[full_link], item)
                elif full_link in self.testing_items:
                    item.setText(2, "Testing...")
                if link[1] in wiki:
                    item.setText(0, "\u274C")
                    for _ in range(3):
                        item.setBackground(_, QtGui.QColor(255, 128, 0))
                    self.duped_links.append(full_link)
                else:
                    item.setText(0, "\u2705")
                    self.valid_links.append(full_link)
        # toggle buttons
        self.copyValid.setEnabled(bool(self.valid_links))
        self.copyDupes.setEnabled(bool(self.duped_links))
        self.copyTested.setEnabled(bool(self.getTestedLinks()))
        self.exportCsv.setEnabled(True)
        # handle copy buttons
        self._is_free.set()

    def retranslateUi(self, MainWindow):
        # Set text (with translations)
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "Dupe Checker v1.9"))
        self.label.setText(_translate("MainWindow", "FMHY Dupe Tester"))
        self.label_2.setText(_translate("MainWindow", "by cevoj35548"))
        self._placeholderText = _translate("MainWindow", "Paste a list of links here")
        self.inputBox.setPlaceholderText(self._placeholderText)
        self.copyValid.setText(_translate("MainWindow", "Copy \u2705"))
        self.copyDupes.setText(_translate("MainWindow", "Copy \u274C"))
        self.copyTested.setText(_translate("MainWindow", "Copy \U0001F50D"))
        self.checkSelected.setText(_translate("MainWindow", "Test \U0001F50D"))
        self.outputTree.headerItem().setText(0, _translate("MainWindow", "Check"))
        self.outputTree.headerItem().setText(1, _translate("MainWindow", "Link"))
        self.outputTree.headerItem().setText(2, _translate("MainWindow", "Status"))


if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    MainWindow.show()
    sys.exit(app.exec_())