Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # -*- coding: utf-8 -*-
- config_file="C://mediacore_dossier//mediacore_source//development.ini"
- softlog = 'C://mediacore_dossier//softlog.log'
- hardlog ='C://mediacore_dossier//mediacore_source//%s-%s.log'
- dbpass=''
- from lib_categories import get_genres
- import paste
- from paste.deploy import loadapp
- import random
- app = loadapp('config:%s' % config_file)
- import mediacore
- from mediacore.model import Media as MediacoreMedia
- from mediacore.model.authors import Author
- from mediacore.model.categories import Category
- from mediacore.model.meta import DBSession
- DBSession.autoflush = True
- from mediacore.model import meta
- from mediacore.lib.storage import add_new_media_file
- from mediacore.model.tags import Tag
- import datetime, time,winsound
- from mediacore_parseurs import Parseur
- import logging
- log = logging.getLogger("MyApffp")
- hdlr = logging.FileHandler(softlog)
- FORMAT='%(asctime)s\t%(levelname)s\t%(message)s'
- formatter = logging.Formatter(FORMAT)
- logging.basicConfig(format=FORMAT) # log sur console
- hdlr.setFormatter(formatter)
- log.addHandler(hdlr)
- log.setLevel(logging.ERROR)
- import MySQLdb
- conn = MySQLdb.connect (host = "localhost",
- user = "root",
- passwd = dbpass,
- db = "crawl")
- cursor = conn.cursor(cursorclass=MySQLdb.cursors.DictCursor)
- from ordering import to_order
- class Client(object):
- '''
- classdocs
- '''
- def commit(self):
- return
- DBSession.flush()
- def end(self):
- DBSession.commit()
- DBSession.flush()
- def random_commit(self):
- if random.choice([0,0,0,0,0,0,0,0,0,0,0,0,0,1]):
- self.end()
- def delete_media(self,id=None,all=None):
- print "test all"
- DBSession.query(MediacoreMedia).delete(synchronize_session=False)
- DBSession.query(Category).delete(synchronize_session=False)
- DBSession.query(Tag).delete(synchronize_session=False)
- print "tt est supprime,babyy!"
- self.end()
- def get_media_by_titre(self,titre):
- for element in MediacoreMedia.query.all():
- if element.title.strip() == titre.strip():
- return element
- return None
- try:
- return MediacoreMedia.query.filter(MediacoreMedia.title.hh).one()
- except:
- return None
- def get_genre_by_string1(self,genre_string):
- return get_genres(genre_string)
- def get_genre_by_string(self,genre_string):
- genre_string = get_genres(genre_string)
- liste=[]
- cats = Category.query.all()
- for genre in [x.strip() for x in genre_string]:
- flag=0
- for cat in cats:
- if genre.lower().strip() == cat.name.lower().strip():
- flag=1
- liste.append(cat.id)
- break
- if not flag:
- cat = Category()
- slug= mediacore.model.get_available_slug(Category, genre,)
- cat.name = genre
- cat.slug=slug
- DBSession.add(cat)
- self.end()
- liste.append(cat.id)
- return liste
- def attach_media_file(self,media,media_file,media_file_title):
- mf = add_new_media_file(media, None, media_file)
- mf.display_name =media_file_title
- self.commit()
- def create_media(self,title, author_email=None, tags=[],author_name=None, poster=None,liste_acteurs=None,liste_real=None,annee=None,resume=None,media_files=None,genres = None, duree=None,pays=None):
- categories= self.get_genre_by_string(genres)
- media=MediacoreMedia()
- idd = media.id
- media.title = title
- slug = mediacore.model.get_available_slug(MediacoreMedia, media.title,)
- media.slug=slug
- media.author = Author(author_name, author_email)
- if resume != "<br />":
- media.description = str(resume)
- media.set_tags(tags)
- media.set_categories(categories)
- media.meta["mediacore_amine_realisateur"] = "; ".join(liste_real)
- media.meta["mediacore_amine_acteurs"] = "; ".join(liste_acteurs)
- media.meta["mediacore_amine_poster"] = poster
- media.meta["mediacore_amine_annee"] = annee
- media.meta["mediacore_amine_pays"] = pays[0]
- DBSession.add(media)
- if media_files:
- for media_file,titre,url in media_files:
- mf = add_new_media_file(media, None, media_file)
- mf.display_name = titre
- media.duration = duree
- media.reviewed = True
- media.publishable = True
- media.publish_on = datetime.datetime.now()
- media.update_status()
- #self.random_commit()
- return media
- def __init__(self,config_file ):
- '''
- dpstream client
- '''
- pass
- client=Client("jh")
- import sys
- print "---->", sys.getdefaultencoding()
- def raz():
- cursor.execute ('SET NAMES \'utf8\';')
- cursor.execute ('SET CHARACTER SET utf8;')
- client.delete_media(all=1)
- requete_raz="update movies set done=0"
- cursor.execute (requete_raz)
- client.delete_media(all=1)
- time.sleep(30)
- raz()
- winsound.PlaySound("*", winsound.SND_ALIAS)
- print 'trop tard...'
- permited_hosts = []
- requete_recherche_movies="select id , UNCOMPRESS(html) as html , titre from movies where done=0 order by id desc limit 500 "
- requete_recherche_media_file="select id, UNCOMPRESS(html) as html , titre , unique_id, film_id,link from media_files where film_id = %s "
- requete_set_movie_done = "update movies set done=1 where id=%s limit 1"
- def fire(nombre):
- to_go = 0
- print "fired!!!"
- while nombre:
- daba = datetime.datetime.now()
- nombre -=1
- cursor.execute (requete_recherche_movies)
- movies = cursor.fetchall()
- if movies:
- to_go += 1
- for movie in movies:
- # log.debug("--------------------------")
- titre_movie = movie["titre"]
- # log.debug("traitement " + movie["titre"])
- cursor.execute (requete_recherche_media_file,movie["id"])
- media_files = cursor.fetchall()
- # log.debug("%s%s"% ("all mediafiles found: " , len(media_files),))
- parseur_film = Parseur(movie["html"])
- liste_medias= []
- for media_file in media_files:
- unique_id=""
- title_media_file=""
- # log.debug( "!!!unique id!!!")
- # log.debug( media_file["unique_id"])
- if media_file["unique_id"]:
- unique_id = media_file["unique_id"]
- title_media_file = media_file["titre"]
- else:
- # log.debug("ej dois parser...")
- parseur_media_file = Parseur(media_file["html"])
- unique_id = parseur_media_file.unique_ids
- title_media_file = parseur_media_file.title_media_file
- unique_ids = unique_id
- if unique_ids.count("videobb") or unique_ids.count("megavideo"):
- liste_medias.append((unique_id,title_media_file,media_file["link"]))
- else:
- pass
- liste_medias = to_order(liste_medias)
- tags = parseur_film.liste_real + parseur_film.liste_acteurs + [parseur_film.annee]
- # log.debug("%s%s" % ("mediafiles a vraiment traiter..",len(liste_medias),) )
- client.create_media(title = parseur_film.title.strip()
- , author_email="admin@admin.com",
- tags=[x for x in tags if x],
- author_name="Administrator",
- poster=parseur_film.poster,
- genres=parseur_film.genres,
- liste_acteurs = [x for x in parseur_film.liste_acteurs if x],
- liste_real = [x for x in parseur_film.liste_real if x],
- annee=parseur_film.annee,
- resume=parseur_film.resume,
- media_files=liste_medias,
- duree=parseur_film.duree,
- pays = parseur_film.pays
- )
- #log.debug("%s----------"% to_go)
- cursor.execute (requete_set_movie_done,movie['id'])
- client.end()
- winsound.PlaySound("*", winsound.SND_ALIAS)
- print "300 en %s" % (datetime.datetime.now() - daba)
- else:
- break
- #client.end()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement