URL parser for https://github.com/gehaxelt/python-rss2irc

From e28ac7ffe5d7fe80d7dbf2ed8432c84d9d3f5892 Mon Sep 17 00:00:00 2001
From: Joel Maxuel <joel@shell.skynet>
Date: Wed, 5 May 2021 19:25:39 -0300
Subject: [PATCH] Include the URL parser from EEVbot

---
 bot.py           | 76 ++++++++++++++++++++++++++++++++++++++++++++++++
 requirements.txt |  3 ++
 2 files changed, 79 insertions(+)

diff --git a/bot.py b/bot.py
index 86956a3..143018a 100644
--- a/bot.py
+++ b/bot.py
@@ -3,18 +3,32 @@ import threading
 import irc.bot
 import irc.client
 import irc.connection
+import json
 import time
 import re
+import string
 import sys
 import feedparser
 import datetime
 import dateutil.parser
 import requests
+import urllib
 from colour import Colours
 from db import FeedDB
 from config import Config
 from feedupdater import FeedUpdater
+from bs4 import BeautifulSoup
+from urlextract import URLExtract

+fcount = 1.0
+pubfltime = 0
+tformat = "%Y/%m/%d %H:%M:%S"
+ccstrip = re.compile('[\x02\x0F\x16\x1D\x1F]|\x03(\d{,2}(,\d{,2})?)?')
+
+IRC_BOLD = "\x02"
+IRC_ITALIC = "\x1D"
+IRC_UNDERLINE = "\x1F"
+IRC_REGULAR = "\x0F"

 class IRCBot(irc.bot.SingleServerIRCBot):
     def __init__(self, config, db, on_connect_cb):
@@ -129,6 +143,49 @@ class IRCBot(irc.bot.SingleServerIRCBot):
             answer = "Something was wrong."
         return answer

+    ## EEV Bot hacks ... section inserted from:
+    ## eevBot - Austnet IRC bot
+
+    ## A twitter IRC bot, modified for AustNet #eevblog.
+    ## Original creator - https://mike.verdone.ca/twitter
+    ## Original source code - https://github.com/sixohsix/twitter/blob/master/twitter/ircbot.py
+    ## Modified / maintained by - electrohead / corp[at]hush[dot]ai
+
+    ## A big thanks to "the internet" for the ideas and help with bugs, and what not.
+
+    def ytlinkparse(self, yturl):
+        q = urllib.parse.urlparse(yturl)
+        if q.hostname == 'youtu.be': return q.path[1:]
+        if q.hostname in {'www.youtube.com', 'youtube.com'}:
+            if q.path == '/watch': return urllib.parse.parse_qs(q.query)['v'][0]
+            if q.path[:7] == '/embed/': return q.path.split('/')[2]
+            if q.path[:3] == '/v/': return q.path.split('/')[2]
+        return None
+
+    def url_parse(self, stext):
+        ext1 = URLExtract()
+        urls = ext1.find_urls(stext)
+        if urls:
+            for url in urls:
+                if url.find("youtube") != -1 or url.find("youtu.be") != -1:
+                    xparams = {"format": "json", "url": "https://www.youtube.com/watch?v=%s" % self.ytlinkparse(url)}
+                    xurl = "https://www.youtube.com/oembed"
+                    qstring = urllib.parse.urlencode(xparams)
+                    xurl = xurl + "?" + qstring
+                    with urllib.request.urlopen(xurl) as response:
+                        response_text = response.read()
+                        data = json.loads(response_text.decode())
+                        self.send_msg(self.__config.CHANNEL, IRC_BOLD + "[Title] - Youtube: " + data['title'] + IRC_REGULAR)
+                else:
+                    r = requests.get(url, allow_redirects=False)
+                    tparse = BeautifulSoup(r.text, features="html5lib")
+                    tfind = tparse.find_all('title')
+                    self.send_msg(self.__config.CHANNEL, IRC_BOLD + "[Title] - " + tfind[0].get_text() + IRC_REGULAR)
+                print("** URL_PARSE - " + url)
+                break
+
+    ## /EEV
+
     def on_privmsg(self, connection, event):
         """Handles the bot's private messages"""
         if len(event.arguments) < 1:
@@ -153,9 +210,28 @@ class IRCBot(irc.bot.SingleServerIRCBot):
         # Send the answer as a private message
         if msg == "!help":
             self.send_msg(event.source.nick, self.__help_msg())
+        # Suggest a paste platform
+        if msg == "!paste":
+            self.send_msg(self.__config.CHANNEL, "Paste services: http://ix.io/  -  http://sprunge.us/  -  https://www.pastebin.com/")
         # Send the answer as a public message
         if botnick.lower() in msg:
             self.send_msg(self.__config.CHANNEL, self.welcome_msg())
+        global pubfltime
+        if Config.lastpubmsg - pubfltime < fcount:
+            return
+        try:
+            """Handles the bot's public (channel) messages"""
+            tsrc = event.source.split('!')[0]
+            sc1 = re.sub(r'[^\x00-\x7f]', r'', event.arguments[0])
+            sc2 = ccstrip.sub('', sc1)
+            args = [i for i in sc2.split(' ') if i]
+            if (not args):
+                return
+            tfirst = args[0].lower().strip()
+            tfirst = re.sub(":|-|,", "", tfirst)
+            self.url_parse(event.arguments[0])
+        except Exception as e:
+            print(datetime.datetime.now(), e)

     def on_nicknameinuse(self, connection, event):
         """Changes the nickname if necessary"""
diff --git a/requirements.txt b/requirements.txt
index 17a5bfe..b17a57f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,6 @@ irc==19.0.1
 python_dateutil==2.8.1
 requests==2.25.1
 sqlite3worker==1.1.7
+bs4==0.0.1
+urlextract>=1.1.0
+html5lib>=1.1
--
2.20.1