Import Videos to MythTV

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import imaplib
import email
import datetime
import re
import subprocess
import os
import shutil
import socket
import glob
import json
from pprint import pprint
from stat import *
import urllib2
import xml.etree.ElementTree as ET

import MythTV
from MythTV import MythDB, Recorded
from MythTV.database import DBDataWrite

reURLs = re.compile(r"""((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.‌​][a-z]{2,4}/)(?:[^\s()<>]+|(([^\s()<>]+|(([^\s()<>]+)))*))+(?:(([^\s()<>]+|(‌​([^\s()<>]+)))*)|[^\s`!()[]{};:'".,<>?«»“”‘’]))""", re.DOTALL)
reHTTP = re.compile('^http.*(youtu.?be|vimeo)')
reSPAM = re.compile('.*email_spam.*')
rePlus = re.compile('[+]([^@]+)')

defaultTitle = "Whatever"
gmailUserID = "Someone@gmail.com"
gmailPassword = "very secret"


def GetURLs(M):

  videoURLs = { }
  msgIDs = [ ]

  rv, data = M.uid("search", None, "ALL")
  if rv != 'OK':
      print "No messages found!"
      return videoURLs

  for num in data[0].split():

      print "Message", num

      msgIDs.append(num)
      rv, data = M.uid("fetch", num, '(RFC822)')
      if rv != 'OK':
          print "ERROR getting message", num
          return videoURLs

      msg = email.message_from_string(data[0][1])
      msgTo = email.utils.parseaddr(msg["To"])[1]
      msgPluss = rePlus.findall(msgTo)
      msgPlus = msgPluss[0] if len(msgPluss) > 0 else defaultTitle
      if not  msgPlus in videoURLs:
          videoURLs[msgPlus] = set([])
      M.store(num, "+X-GM-LABELS", msgPlus)
      date_tuple = email.utils.parsedate_tz(msg['Date'])
      print msgPlus, msg["Subject"]
      for section in msg.walk():
          if section.get_content_maintype() == "text":
             text = section.get_payload(decode=True)
             urls = [ url[0].split('"')[0] for url in reURLs.findall(text) if reHTTP.match(url[0]) and not reSPAM.match(url[0]) ]
             if len(urls) > 0:
                 print "   ", urls
                 videoURLs[msgPlus].update(set(urls))

  return msgIDs, videoURLs


def dbRowFromJason(json):
    startTime = datetime.datetime.fromtimestamp(vid["modTime"])
    endTime = datetime.datetime.fromtimestamp(vid["modTime"] + vid["duration"])

    row = {
            "chanid" : 1000,
            "starttime" : startTime.isoformat(),
            "endtime" : endTime.isoformat(),
            "progstart" : startTime.isoformat(),
            "progend" : endTime.isoformat(),
            "originalairdate" : startTime.date(),
            "lastmodified" : endTime.isoformat(),

            "category" : 'Podcast',
            "seriesid" : json["extractor"],
            "programid" : json["display_id"],
            "season" : 0,
            "episode" : 0,
            "inetref" : 0,
            "hostname" : socket.gethostname().split(".")[0],
            "recgroup" : "Default",
            "playgroup" : "Default",
            "previouslyshown" : False,
            "stars" : 5,
            "preserve" : True,
            "watched" : False,
            "storagegroup" : "Default",
            "inputname" : "Internet",

            "title" : json["EmailTitle"],
            "subtitle" : json["fulltitle"],
            "description" : json["description"],
            "basename" : json["FileName"],
            "filesize" : json["FileSize"],

            "autoexpire" : False
    }
    return row


def TargetDisk():
    def bumpOne():
        TargetDisk.curDisk = (TargetDisk.curDisk % 10) + 1
    def target():
        return "/disks/d%d/tv" % TargetDisk.curDisk
    bumpOne()
    while not os.path.exists(target()):
      bumpOne()
    return target()
TargetDisk.curDisk = 0


M = imaplib.IMAP4_SSL('imap.gmail.com')

try:
    M.login(gmailUserID, gmailPassword)
except imaplib.IMAP4.error:
    print "LOGIN FAILED!!!"
    exit

rv, data = M.select("inbox")
if rv == 'OK':
    msgIDs, urls = GetURLs(M)

    if len(urls) > 0:

        for pref in urls:
            dir = "DL/" + pref
            if not os.path.exists(dir):
                os.makedirs(dir)
            cmd = ["youtube-dl", "--write-description", "--write-info-json", "--write-annotations", "--download-archive", "mcDownload.history", "--ignore-errors"]
            cmd.extend(["--output", dir + "/%(id)s.%(ext)s"])
            cmd.extend(urls[pref])
            print cmd
            subprocess.call(cmd)

    resp = urllib2.urlopen("http://localhost:6544/Dvr/GetRecordedList")
    tree = ET.parse(resp)
    xml = tree.getroot()
    existingPrograms = set([ prog.text for prog in list(xml.iter("ProgramId")) if prog.text ])
    #existingPrograms = set([ ])

    if not os.path.exists("Processed"):
      os.makedirs("Processed")

    mythdb = MythDB()

    for dir in glob.glob("DL/*"):
        title = os.path.basename(dir)
        print dir, title
        if os.path.isdir(dir):
            for jsonFile in glob.glob(dir + "/*.info.json"):
                print "   ", jsonFile
                with open(jsonFile) as j:
                    vid = json.load(j)
                vid["FileName"] = vid["display_id"] + "." + vid["ext"]
                vid["FullName"] = dir + "/" + vid["FileName"]
                print "        ", vid["FullName"]
                if os.path.exists(vid["FullName"]) and vid["display_id"] not in existingPrograms:
                    vid["EmailTitle"] = title
                    fstat = os.stat(vid["FullName"])
                    vid["modTime"] = fstat.st_mtime
                    vid["modTimeString"] = datetime.datetime.fromtimestamp(vid["modTime"]).strftime('%Y-%m-%d %H:%M:%S')
                    vid["FileSize"] = fstat.st_size
                    print "   ", vid["FileName"]
                    done = False
                    while not done:
                        try:
                            Recorded().create(dbRowFromJason(vid))
                            done = True
                        except MythTV.exceptions.MythDBError:
                            vid["modTime"] += 1
                    shutil.move(vid["FullName"], TargetDisk())
                    subprocess.call(["mythcommflag", "--rebuild", "--queue", "--file", vid["FileName"]])
                    #print dbRowFromJason(vid)
                    existingPrograms.update([vid["display_id"]])
                    for supportingFile in glob.glob(dir + "/" + vid["display_id"] + ".*"):
                        shutil.move(supportingFile, "Processed")


    # gmail doesn't delete with the Deleted flag, it archives
    for msgID in msgIDs:
        print "archive", msgID
        M.store(msgID, "+FLAGS", "\\Deleted")


    M.close()
M.logout()