Guest User

Untitled

a guest
Nov 24th, 2017
80
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 5.32 KB | None | 0 0
  1. from optparse import OptionParser
  2. import os
  3. import sys
  4.  
  5. from django.core.management import setup_environ
  6. from cc_admin2 import settings
  7. setup_environ(settings)
  8.  
  9. from cc_admin2.mapping import map_lib
  10.  
  11. from cc_admin2.coupon.models import Category, Offer, Store
  12.  
  13. log = settings.LOGGING
  14.  
  15.  
  16. def map_in(model_arg, map_csv, query, db1):
  17.     """
  18.    map outside data source to django model using csv file
  19.  
  20.    In this case, "outside data source" is a CSV from our exports. -jough
  21.    """
  22.  
  23. #  batch_signal.send(sender=model_arg, model=model_arg, action='start')
  24.  
  25.     mappings2 = map_lib.get_mappings(model_arg, map_csv)
  26.     log.info("map_in : {0} ({1} columns)".format(model_arg, len(mappings2)))
  27.  
  28.     # query the old DB
  29.     results = map_lib.query_old_db(query, db1)
  30.  
  31.     log.info("query :" + query)
  32.     log.info("result rows : " + str(len(results)))
  33.  
  34.     # go through rows in the query result and apply xforms, if nessecary
  35.     object_save_count = 0
  36.     fail_save_count = 0
  37.  
  38.     for r1 in results:
  39.  
  40.         dj_obj = map_lib.get_existing_dj_obj(model_arg, r1, mappings2)
  41.  
  42.         d1 = {}
  43.         m2m_dict = {}
  44.  
  45.         # Apply mappings
  46.         for m1 in mappings2:
  47.  
  48.             # check if this is a M2M relationship
  49.             if m1['relationship']:
  50.                 exec_str = "dj_obj_list =  %s" % (m1['transform'])
  51.                 try:
  52.                     exec exec_str
  53.                 except:
  54.                     print >> sys.stderr, "--- xform error ----"
  55.                     print >> sys.stderr, sys.exc_info()
  56.                     print >> sys.stderr, "--- xform error ----"
  57.  
  58.                 # save to deal with later
  59.                 m2m_dict[m1['django']] = dj_obj_list
  60.  
  61.             # check if a transform has to be applied
  62.             elif m1['transform'] != '' and m1['transform'] != None:
  63.  
  64.                 exec_str = "d1[m1['django']] = %s" % (m1['transform'])
  65.                 exec exec_str
  66.  
  67.             # else just copy it over
  68.             elif r1[m1['field']] != None and r1[m1['field']] != '':
  69.                 d1[m1['django']] = r1[m1['field']]
  70.  
  71.  
  72.         # Merge and Save
  73.         try:
  74.             if dj_obj != None:
  75.  
  76.                 for field, value in d1.items():
  77.                     exec_str = "field_type = dj_obj.%s.__class__.__name__" % (field)
  78.                     exec exec_str
  79.  
  80.                     if field_type == "unicode" or field_type == "datetime" or field_type.lower() == 'nonetype':
  81.                         exec_str = "dj_obj.%s = '''%s'''" % (field, value)
  82.                     else:
  83.                         exec_str = "dj_obj.%s = %s" % (field, value)
  84.  
  85.          # print "EXEC_STR=[" + str(exec_str) + "] type [" + field_type + "]"
  86.                     exec exec_str
  87.  
  88.             else:
  89.                 exec_str = "dj_obj = %s(**d1)" % (model_arg)
  90.                 exec exec_str
  91.  
  92.             # print d1
  93.             dj_obj.save()
  94.  
  95.             # this fills in the M2M objects found above and save again
  96.             for field, objs in m2m_dict.items():
  97.  
  98.                 try:
  99.                     exec_str = "dj_obj." + field + ".clear()"
  100.                     exec exec_str
  101.                    
  102.                 except:
  103.                     pass
  104.  
  105.                 for obj in objs:
  106.                     try:
  107.                         exec_str = "dj_obj." + field + ".add(obj)"
  108.                         exec exec_str
  109.                     except:
  110.                         exec_str = "dj_obj." + field + "=obj"
  111.                         exec exec_str
  112.  
  113.             dj_obj.save()
  114.             object_save_count += 1
  115.  
  116.         except:
  117.             print >> sys.stderr, exec_str
  118.             fail_save_count += 1
  119.             print >> sys.stderr, str(sys.exc_value)
  120.             print >> sys.stderr, "error: " + str(d1) + " -- " + str(sys.exc_info()[1])
  121.  
  122.     log.info("saved : " + str(object_save_count) + "")
  123.     log.info("fail : " + str(fail_save_count) + "")
  124.     log.info("DONE : " + model_arg)
  125.  
  126. #  batch_signal.send(sender=model_arg, model=model_arg, action='stop')
  127.  
  128. #
  129. # map_in method
  130.  
  131.  
  132. ########
  133.  
  134. if __name__ == '__main__':
  135.  
  136.     script_name = sys.argv[0].split("/").pop()
  137.  
  138.     opts = OptionParser()
  139.     opts.add_option('-s', '--short', dest="short_update", default=False, action="store_true")
  140.     opts.add_option('-l', '--long', dest="all_update", default=False, action="store_true")
  141.     opts.add_option("-c", "--csv_map", dest="csv_file", type="string", help="CSV mapping file")
  142.     opts.add_option("-x", "--config_py", dest="config_py", type="string", help="Config python")
  143.     options, arguments = opts.parse_args()
  144.  
  145.     if not options.csv_file or not options.config_py or not os.path.exists(options.config_py):
  146.         print >> sys.stderr, "usage: %s --short(default)|--long --csv_map <map file> --config_py <config py>" % (script_name)
  147.         sys.exit(0)
  148.  
  149.     exec_str = "from cc_admin2.mapping.%s import *" % (os.path.basename(options.config_py.replace(r".py", "")))
  150.     exec exec_str
  151.  
  152.     if options.all_update:
  153.         log.info("Long Update")
  154. #     map_in("SubCategory", options.csv_file, ALL_SUB_CAT, db1)
  155. #     map_in("Category", options.csv_file, ALL_CAT, db1)
  156.         map_in("Store", options.csv_file, ALL_STORE, db1)
  157.         map_in("Offer", options.csv_file, ALL_OFFER, db1)
  158.  
  159.     else:
  160.         log.info("Short Update")
  161.         map_in("Offer", options.csv_file, ADDED_LAST_DAY, db1)
Add Comment
Please, Sign In to add comment