Guest User

Untitled

a guest
Jun 20th, 2013
136
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. # -*- coding: utf-8 -*-
  2.  
  3. config_file="C://mediacore_dossier//mediacore_source//development.ini"
  4. softlog = 'C://mediacore_dossier//softlog.log'
  5. hardlog ='C://mediacore_dossier//mediacore_source//%s-%s.log'
  6. dbpass=''
  7.  
  8. from lib_categories import get_genres
  9. import paste
  10. from paste.deploy import loadapp
  11.  
  12. import random
  13.  
  14.  
  15. app = loadapp('config:%s' % config_file)
  16. import mediacore
  17. from mediacore.model import Media as MediacoreMedia
  18. from mediacore.model.authors import Author
  19. from mediacore.model.categories import Category
  20. from mediacore.model.meta import DBSession
  21. DBSession.autoflush = True
  22. from mediacore.model import meta
  23. from mediacore.lib.storage import add_new_media_file
  24. from mediacore.model.tags import Tag
  25. import datetime, time,winsound
  26. from mediacore_parseurs import Parseur
  27.  
  28. import logging
  29. log = logging.getLogger("MyApffp")
  30. hdlr = logging.FileHandler(softlog)
  31. FORMAT='%(asctime)s\t%(levelname)s\t%(message)s'
  32. formatter = logging.Formatter(FORMAT)
  33. logging.basicConfig(format=FORMAT) # log sur console
  34. hdlr.setFormatter(formatter)
  35. log.addHandler(hdlr)
  36. log.setLevel(logging.ERROR)
  37.  
  38. import MySQLdb
  39. conn = MySQLdb.connect (host = "localhost",
  40. user = "root",
  41. passwd = dbpass,
  42. db = "crawl")
  43.  
  44. cursor = conn.cursor(cursorclass=MySQLdb.cursors.DictCursor)
  45.  
  46. from ordering import to_order
  47.  
  48.  
  49. class Client(object):
  50. '''
  51. classdocs
  52. '''
  53. def commit(self):
  54. return
  55. DBSession.flush()
  56.  
  57.  
  58. def end(self):
  59.  
  60. DBSession.commit()
  61. DBSession.flush()
  62.  
  63. def random_commit(self):
  64. if random.choice([0,0,0,0,0,0,0,0,0,0,0,0,0,1]):
  65. self.end()
  66.  
  67. def delete_media(self,id=None,all=None):
  68.  
  69. print "test all"
  70. DBSession.query(MediacoreMedia).delete(synchronize_session=False)
  71. DBSession.query(Category).delete(synchronize_session=False)
  72. DBSession.query(Tag).delete(synchronize_session=False)
  73. print "tt est supprime,babyy!"
  74. self.end()
  75.  
  76.  
  77.  
  78.  
  79. def get_media_by_titre(self,titre):
  80. for element in MediacoreMedia.query.all():
  81. if element.title.strip() == titre.strip():
  82. return element
  83. return None
  84. try:
  85. return MediacoreMedia.query.filter(MediacoreMedia.title.hh).one()
  86. except:
  87. return None
  88.  
  89.  
  90.  
  91.  
  92. def get_genre_by_string1(self,genre_string):
  93. return get_genres(genre_string)
  94.  
  95.  
  96.  
  97. def get_genre_by_string(self,genre_string):
  98. genre_string = get_genres(genre_string)
  99. liste=[]
  100. cats = Category.query.all()
  101. for genre in [x.strip() for x in genre_string]:
  102. flag=0
  103. for cat in cats:
  104. if genre.lower().strip() == cat.name.lower().strip():
  105. flag=1
  106. liste.append(cat.id)
  107. break
  108. if not flag:
  109. cat = Category()
  110. slug= mediacore.model.get_available_slug(Category, genre,)
  111. cat.name = genre
  112. cat.slug=slug
  113. DBSession.add(cat)
  114. self.end()
  115. liste.append(cat.id)
  116.  
  117. return liste
  118.  
  119. def attach_media_file(self,media,media_file,media_file_title):
  120. mf = add_new_media_file(media, None, media_file)
  121.  
  122. mf.display_name =media_file_title
  123. self.commit()
  124.  
  125.  
  126. def create_media(self,title, author_email=None, tags=[],author_name=None, poster=None,liste_acteurs=None,liste_real=None,annee=None,resume=None,media_files=None,genres = None, duree=None,pays=None):
  127. categories= self.get_genre_by_string(genres)
  128. media=MediacoreMedia()
  129. idd = media.id
  130. media.title = title
  131. slug = mediacore.model.get_available_slug(MediacoreMedia, media.title,)
  132. media.slug=slug
  133. media.author = Author(author_name, author_email)
  134.  
  135.  
  136. if resume != "<br />":
  137. media.description = str(resume)
  138.  
  139. media.set_tags(tags)
  140. media.set_categories(categories)
  141.  
  142.  
  143.  
  144.  
  145. media.meta["mediacore_amine_realisateur"] = "; ".join(liste_real)
  146. media.meta["mediacore_amine_acteurs"] = "; ".join(liste_acteurs)
  147. media.meta["mediacore_amine_poster"] = poster
  148. media.meta["mediacore_amine_annee"] = annee
  149. media.meta["mediacore_amine_pays"] = pays[0]
  150. DBSession.add(media)
  151. if media_files:
  152. for media_file,titre,url in media_files:
  153.  
  154. mf = add_new_media_file(media, None, media_file)
  155. mf.display_name = titre
  156.  
  157.  
  158.  
  159.  
  160.  
  161. media.duration = duree
  162. media.reviewed = True
  163. media.publishable = True
  164. media.publish_on = datetime.datetime.now()
  165. media.update_status()
  166. #self.random_commit()
  167.  
  168. return media
  169.  
  170. def __init__(self,config_file ):
  171. '''
  172. dpstream client
  173. '''
  174.  
  175. pass
  176.  
  177.  
  178.  
  179.  
  180. client=Client("jh")
  181.  
  182.  
  183. import sys
  184. print "---->", sys.getdefaultencoding()
  185.  
  186. def raz():
  187. cursor.execute ('SET NAMES \'utf8\';')
  188. cursor.execute ('SET CHARACTER SET utf8;')
  189. client.delete_media(all=1)
  190. requete_raz="update movies set done=0"
  191.  
  192. cursor.execute (requete_raz)
  193. client.delete_media(all=1)
  194. time.sleep(30)
  195. raz()
  196.  
  197. winsound.PlaySound("*", winsound.SND_ALIAS)
  198. print 'trop tard...'
  199.  
  200.  
  201.  
  202. permited_hosts = []
  203. requete_recherche_movies="select id , UNCOMPRESS(html) as html , titre from movies where done=0 order by id desc limit 500 "
  204. requete_recherche_media_file="select id, UNCOMPRESS(html) as html , titre , unique_id, film_id,link from media_files where film_id = %s "
  205. requete_set_movie_done = "update movies set done=1 where id=%s limit 1"
  206.  
  207.  
  208.  
  209.  
  210. def fire(nombre):
  211. to_go = 0
  212. print "fired!!!"
  213. while nombre:
  214. daba = datetime.datetime.now()
  215. nombre -=1
  216. cursor.execute (requete_recherche_movies)
  217. movies = cursor.fetchall()
  218. if movies:
  219. to_go += 1
  220.  
  221.  
  222. for movie in movies:
  223.  
  224. # log.debug("--------------------------")
  225. titre_movie = movie["titre"]
  226. # log.debug("traitement " + movie["titre"])
  227. cursor.execute (requete_recherche_media_file,movie["id"])
  228. media_files = cursor.fetchall()
  229. # log.debug("%s%s"% ("all mediafiles found: " , len(media_files),))
  230. parseur_film = Parseur(movie["html"])
  231. liste_medias= []
  232.  
  233. for media_file in media_files:
  234. unique_id=""
  235. title_media_file=""
  236. # log.debug( "!!!unique id!!!")
  237. # log.debug( media_file["unique_id"])
  238. if media_file["unique_id"]:
  239. unique_id = media_file["unique_id"]
  240. title_media_file = media_file["titre"]
  241. else:
  242. # log.debug("ej dois parser...")
  243. parseur_media_file = Parseur(media_file["html"])
  244. unique_id = parseur_media_file.unique_ids
  245. title_media_file = parseur_media_file.title_media_file
  246.  
  247. unique_ids = unique_id
  248. if unique_ids.count("videobb") or unique_ids.count("megavideo"):
  249. liste_medias.append((unique_id,title_media_file,media_file["link"]))
  250. else:
  251. pass
  252.  
  253. liste_medias = to_order(liste_medias)
  254. tags = parseur_film.liste_real + parseur_film.liste_acteurs + [parseur_film.annee]
  255. # log.debug("%s%s" % ("mediafiles a vraiment traiter..",len(liste_medias),) )
  256. client.create_media(title = parseur_film.title.strip()
  257. , author_email="admin@admin.com",
  258. tags=[x for x in tags if x],
  259. author_name="Administrator",
  260. poster=parseur_film.poster,
  261. genres=parseur_film.genres,
  262. liste_acteurs = [x for x in parseur_film.liste_acteurs if x],
  263. liste_real = [x for x in parseur_film.liste_real if x],
  264. annee=parseur_film.annee,
  265. resume=parseur_film.resume,
  266. media_files=liste_medias,
  267.  
  268. duree=parseur_film.duree,
  269. pays = parseur_film.pays
  270. )
  271. #log.debug("%s----------"% to_go)
  272.  
  273. cursor.execute (requete_set_movie_done,movie['id'])
  274. client.end()
  275. winsound.PlaySound("*", winsound.SND_ALIAS)
  276. print "300 en %s" % (datetime.datetime.now() - daba)
  277. else:
  278. break
  279. #client.end()
RAW Paste Data