Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from optparse import OptionParser
- import os
- import sys
- from django.core.management import setup_environ
- from cc_admin2 import settings
- setup_environ(settings)
- from cc_admin2.mapping import map_lib
- from cc_admin2.coupon.models import Category, Offer, Store
- log = settings.LOGGING
- def map_in(model_arg, map_csv, query, db1):
- """
- map outside data source to django model using csv file
- In this case, "outside data source" is a CSV from our exports. -jough
- """
- # batch_signal.send(sender=model_arg, model=model_arg, action='start')
- mappings2 = map_lib.get_mappings(model_arg, map_csv)
- log.info("map_in : {0} ({1} columns)".format(model_arg, len(mappings2)))
- # query the old DB
- results = map_lib.query_old_db(query, db1)
- log.info("query :" + query)
- log.info("result rows : " + str(len(results)))
- # go through rows in the query result and apply xforms, if nessecary
- object_save_count = 0
- fail_save_count = 0
- for r1 in results:
- dj_obj = map_lib.get_existing_dj_obj(model_arg, r1, mappings2)
- d1 = {}
- m2m_dict = {}
- # Apply mappings
- for m1 in mappings2:
- # check if this is a M2M relationship
- if m1['relationship']:
- exec_str = "dj_obj_list = %s" % (m1['transform'])
- try:
- exec exec_str
- except:
- print >> sys.stderr, "--- xform error ----"
- print >> sys.stderr, sys.exc_info()
- print >> sys.stderr, "--- xform error ----"
- # save to deal with later
- m2m_dict[m1['django']] = dj_obj_list
- # check if a transform has to be applied
- elif m1['transform'] != '' and m1['transform'] != None:
- exec_str = "d1[m1['django']] = %s" % (m1['transform'])
- exec exec_str
- # else just copy it over
- elif r1[m1['field']] != None and r1[m1['field']] != '':
- d1[m1['django']] = r1[m1['field']]
- # Merge and Save
- try:
- if dj_obj != None:
- for field, value in d1.items():
- exec_str = "field_type = dj_obj.%s.__class__.__name__" % (field)
- exec exec_str
- if field_type == "unicode" or field_type == "datetime" or field_type.lower() == 'nonetype':
- exec_str = "dj_obj.%s = '''%s'''" % (field, value)
- else:
- exec_str = "dj_obj.%s = %s" % (field, value)
- # print "EXEC_STR=[" + str(exec_str) + "] type [" + field_type + "]"
- exec exec_str
- else:
- exec_str = "dj_obj = %s(**d1)" % (model_arg)
- exec exec_str
- # print d1
- dj_obj.save()
- # this fills in the M2M objects found above and save again
- for field, objs in m2m_dict.items():
- try:
- exec_str = "dj_obj." + field + ".clear()"
- exec exec_str
- except:
- pass
- for obj in objs:
- try:
- exec_str = "dj_obj." + field + ".add(obj)"
- exec exec_str
- except:
- exec_str = "dj_obj." + field + "=obj"
- exec exec_str
- dj_obj.save()
- object_save_count += 1
- except:
- print >> sys.stderr, exec_str
- fail_save_count += 1
- print >> sys.stderr, str(sys.exc_value)
- print >> sys.stderr, "error: " + str(d1) + " -- " + str(sys.exc_info()[1])
- log.info("saved : " + str(object_save_count) + "")
- log.info("fail : " + str(fail_save_count) + "")
- log.info("DONE : " + model_arg)
- # batch_signal.send(sender=model_arg, model=model_arg, action='stop')
- #
- # map_in method
- ########
- if __name__ == '__main__':
- script_name = sys.argv[0].split("/").pop()
- opts = OptionParser()
- opts.add_option('-s', '--short', dest="short_update", default=False, action="store_true")
- opts.add_option('-l', '--long', dest="all_update", default=False, action="store_true")
- opts.add_option("-c", "--csv_map", dest="csv_file", type="string", help="CSV mapping file")
- opts.add_option("-x", "--config_py", dest="config_py", type="string", help="Config python")
- options, arguments = opts.parse_args()
- if not options.csv_file or not options.config_py or not os.path.exists(options.config_py):
- print >> sys.stderr, "usage: %s --short(default)|--long --csv_map <map file> --config_py <config py>" % (script_name)
- sys.exit(0)
- exec_str = "from cc_admin2.mapping.%s import *" % (os.path.basename(options.config_py.replace(r".py", "")))
- exec exec_str
- if options.all_update:
- log.info("Long Update")
- # map_in("SubCategory", options.csv_file, ALL_SUB_CAT, db1)
- # map_in("Category", options.csv_file, ALL_CAT, db1)
- map_in("Store", options.csv_file, ALL_STORE, db1)
- map_in("Offer", options.csv_file, ALL_OFFER, db1)
- else:
- log.info("Short Update")
- map_in("Offer", options.csv_file, ADDED_LAST_DAY, db1)
Add Comment
Please, Sign In to add comment