Advertisement
Guest User

Untitled

a guest
Jul 19th, 2019
78
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 10.06 KB | None | 0 0
  1. import json
  2. import subprocess
  3. import logging
  4. import dateutil.parser
  5. from datetime import datetime, timedelta
  6.  
  7. logging.basicConfig(filename="intake.log",filemode='a+',level=logging.DEBUG)
  8. logger = logging.getLogger(__name__)
  9.  
  10. ROOT_CMD = "npm run intake -- "
  11. COMMANDS = ['stream', 'download', 'createIndex']
  12.  
  13. HISTORY = None
  14.  
  15. NODES = {
  16. "Order": {
  17. "tags": ["transactional"],
  18. },
  19. "OrderItem": {
  20. "tags": ["transactional"],
  21. "relationships": [
  22. "HAS_ORDER_ITEM",
  23. "CONTAINS_VARIANT"
  24. ]
  25. },
  26. "Review": {
  27. "tags": ["reviews"]
  28. },
  29. "Question": {
  30. "tags": ["reviews"]
  31. },
  32. "ClientResponse": {
  33. "tags": ["reviews"],
  34. "relationships":[
  35. "HAS_CLIENT_RESPONSE"
  36. ]
  37. },
  38. "Answer": {
  39. "tags": ["reviews"],
  40. "relationships": [
  41. "HAS_ANSWER"
  42. ]
  43. },
  44. "ABOM": {
  45. "tags": ["product", "material"],
  46. "relationships": [
  47. "HAS_PRIMARY_COLOR",
  48. "HAS_SECONDARY_COLOR",
  49. "HAS_LOGO_COLOR",
  50. "HAS_ARTICLE"
  51. ]
  52. },
  53. "Article": {
  54. "tags": ["product"],
  55. "relationships": [
  56. "INCLUDES_BOM_SECTION",
  57. "HAS_PRIMARY_COLOR",
  58. "HAS_SECONDARY_COLOR",
  59. "HAS_LOGO_COLOR",
  60. "HAS_ARTICLE"
  61. ]
  62. },
  63. "ArticleImageAsset": {
  64. "tags": ["product"],
  65. "relationships": [
  66. "HAS_ARTICLE_IMAGE"
  67. ]
  68. },
  69. "BOMSection": {
  70. "tags": ["product", "material"]
  71. },
  72. "CBOM": {
  73. "tags": ["product", "material"],
  74. "relationships":[
  75. "INCLUDES_ABOM",
  76. "INCLUDES_COLOR",
  77. "INCLUDES_ARTICLE",
  78. ]
  79. },
  80. "Color": {
  81. "tags": ["product", "material"]
  82. },
  83. "Fabric": {
  84. "tags": ["product", "material"]
  85. },
  86. "FeaturesAndBenefits": {
  87. "tags": ["product","test"]
  88. },
  89. "PlannedDevelopment": {
  90. "tags": ["product"]
  91. },
  92. "MiscPart": {
  93. "tags": ["product", "material"]
  94. },
  95. "ProductConfiguration": {
  96. "tags": ["product", "material"]
  97. },
  98. "ProductConfigurationChoice": {
  99. "tags": ["product", "material"],
  100. "relationships":[
  101. "HAS_PRODUCT_CONFIGURATION_CHOICE",
  102. "HAS_PLANNED_DEVELOPMENT"
  103. ]
  104. },
  105. "ProductPlan": {
  106. "tags": ["product", "material"]
  107. },
  108. "SeasonalPlan": {
  109. "tags": ["product"]
  110. },
  111. "SizeChart": {
  112. "tags": ["product"]
  113. },
  114. "SizeVariant": {
  115. "tags": ["product"]
  116. },
  117. "Style": {
  118. "tags": ["product"],
  119. "relationships":[
  120. "HAS_DIVISION",
  121. "HAS_FITTYPE",
  122. "HAS_GENDER",
  123. "HAS_SUBCATEGORY",
  124. ]
  125. },
  126. "RegionalPrice": {
  127. "tags": ["product","pricing"],
  128. "relationships": [
  129. "HAS_REGIONAL_PRICE"
  130. ]
  131. },
  132. "SubCategory": {
  133. "tags": ["product"]
  134. },
  135. "Royalty": {
  136. "tags": ["product"]
  137. },
  138. "Trim": {
  139. "tags": ["product", "material"]
  140. },
  141. "FitType": {
  142. "tags": ["product"],
  143. },
  144. "Gender": {
  145. "tags": ["product"]
  146. },
  147. "Division": {
  148. "tags": ["product"]
  149. },
  150. }
  151. RELATIONSHIPS = {
  152. "HAS_FEATURES_AND_BENEFITS": {
  153. "tags": ["product","test"]
  154. },
  155. "HAS_REVIEW": {
  156. "tags": ["reviews"]
  157. },
  158. "HAS_SIZE_CHART": {
  159. "tags": ["product"]
  160. },
  161. "INCLUDES_MISC_PART": {
  162. "tags": ["product"]
  163. },
  164. "HAS_QUESTION": {
  165. "tags": ["reviews"]
  166. },
  167. "HAS_ROYALTY": {
  168. "tags": ["product"]
  169. },
  170. "INCLUDES_FABRIC": {
  171. "tags": ["product"]
  172. },
  173. "INCLUDES_TRIM": {
  174. "tags": ["product"]
  175. },
  176. "PLANNED_FOR_SEASON": {
  177. "tags": ["product"]
  178. },
  179. }
  180.  
  181. FAKE_RUN = False
  182.  
  183.  
  184. def get_command(command=None, nodes=None, relationships=None):
  185. if command not in COMMANDS and command is not None:
  186. raise ("Command must be one of the following commands: " + ','.join(COMMANDS))
  187. cmd = ROOT_CMD
  188. if nodes:
  189. cmd += ' --nodes=' + ','.join(nodes.keys())
  190.  
  191. if relationships:
  192. cmd += ' --relationships=' + ','.join(
  193. relationships.keys() if isinstance(relationships,dict) else relationships)
  194.  
  195. if command:
  196. return ''.join([cmd, ' --', command])
  197. else:
  198. return cmd
  199.  
  200.  
  201. def run_command(cmd):
  202. if FAKE_RUN:
  203. print(cmd)
  204. return True
  205. else:
  206. try:
  207. logger.info("#####################################")
  208. logger.info(f"Running '{cmd}'")
  209. logger.info("#####################################")
  210. output = subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
  211. except subprocess.CalledProcessError as exc:
  212. logger.error(f"Failed with return code {exc.returncode}:\n{exc.output}\n\n")
  213. return False
  214. else:
  215. logger.info(f"Succeeded: \n{output}")
  216. return True
  217.  
  218.  
  219. def create_indices(nodes):
  220. return run_command(get_command("createIndex", nodes))
  221.  
  222.  
  223. def download_nodes(nodes, stream_to_db=False):
  224. return run_command(get_command("stream" if stream_to_db else "download", nodes=nodes))
  225.  
  226.  
  227. def download_relationships(relationships, nodes=None, stream_to_db=False):
  228. return run_command(get_command("stream" if stream_to_db else "download", nodes=nodes, relationships=relationships))
  229.  
  230.  
  231. def upload_nodes(nodes):
  232. return run_command(get_command(nodes=nodes))
  233.  
  234.  
  235. def upload_relationships(relationships, nodes=None):
  236. return run_command(get_command(nodes=nodes, relationships=relationships))
  237.  
  238.  
  239. def download_by_tags(tags, stream_to_db=False):
  240. if not isinstance(tags, list):
  241. tags = [tags]
  242.  
  243. # find the nodes to import (based on the tags given)
  244. nodes_to_import = {k: v for k, v in NODES.items() if list(set(tags) & set(v['tags']))}
  245. relationships_to_import = {k: v for k, v in RELATIONSHIPS.items() if list(set(tags) & set(v['tags']))}
  246.  
  247. # Run the command for one node at a time.
  248. action = "Streaming" if stream_to_db else "Downloading"
  249. action_past = "streamed" if stream_to_db else "downloaded"
  250. failures = []
  251. for name, info in nodes_to_import.items():
  252. print("###################################################")
  253. print("{0} {1}:".format(action,name))
  254.  
  255. if completed_recently(name):
  256. print("Skipping {0} because it was uploaded recently".format(name))
  257. continue
  258.  
  259. if stream_to_db:
  260. print(" > Creating index")
  261. if not create_indices(nodes={name: info}):
  262. print(" >> Create index on {0} failed. See logs for more info".format(name))
  263. failures.append("Create Index: {0}".format(name))
  264. else:
  265. print(" >> Successfully created")
  266.  
  267. print(" > {0} node only".format(action))
  268. if not download_nodes(nodes={name: info}, stream_to_db=stream_to_db):
  269. print(" >> {0} {1} failed. See logs for more info".format(action,name))
  270. failures.append("{1}: {0}".format(action,name))
  271. else:
  272. completed_now(name)
  273. print(" >> Successfully {0} {1}".format(action_past,name))
  274.  
  275. # We have to do the nested relationships for the nodes only after all the connected nodes have been
  276. # created so we have to do this in two separate loops.
  277. for name, info in nodes_to_import.items():
  278. if 'relationships' in info and isinstance(info['relationships'], list) and len(info['relationships']) > 0:
  279.  
  280. if completed_recently(name+"_relationships"):
  281. print("Skipping {0} relationships because they were uploaded recently".format(name))
  282. continue
  283.  
  284. print(" > {0} relationships for {1} only".format(action,name))
  285. if not download_relationships(nodes={name: info}, relationships=info['relationships'], stream_to_db=stream_to_db):
  286. print(" >> {0} relationships failed for {1}. See logs for more info".format(action, name))
  287. failures.append("{0} relationships for {1}: {2}".format(action, name, ','.join(info['relationships'])))
  288. else:
  289. completed_now(name+"_relationships")
  290. print(" >> Successfully {0} relationships for {1}".format(action_past, name))
  291.  
  292. # Now handle independent relationships
  293. if len(relationships_to_import) > 0:
  294. print(" > {0} independent relationships".format(action))
  295. for name,info in relationships_to_import.items():
  296. if completed_recently(name):
  297. print("Skipping {0} because it was uploaded recently".format(name))
  298. continue
  299.  
  300. print(" >> {0} {1} relationship".format(action, name))
  301. if not download_relationships(relationships={name: info}, stream_to_db=stream_to_db):
  302. print(" >> {0} independent relationship {1} failed . See logs for more info".format(action, name))
  303. failures.append("{0} relationship: {1}".format(action, name))
  304. else:
  305. completed_now(name)
  306. print(" >> Successfully {0} relationship {1}".format(action_past, name))
  307.  
  308. print("-----------------")
  309. if len(failures) > 0:
  310. print("The following actions failed during execution. See logs for more information")
  311. for f in failures:
  312. print("* " + f)
  313.  
  314. print("###############")
  315.  
  316.  
  317. def load_history():
  318. global HISTORY
  319.  
  320. if not HISTORY:
  321. with open('./intake_history.json', 'w+') as fp:
  322.  
  323. try:
  324. HISTORY = json.load(fp)
  325. except json.JSONDecodeError:
  326. HISTORY = {}
  327.  
  328. return HISTORY
  329.  
  330.  
  331. def save_history():
  332. h = load_history()
  333. if h:
  334. with open('./intake_history.json', 'w+') as fp:
  335. json.dump(HISTORY, fp)
  336.  
  337.  
  338. def completed_recently(node_or_relationship):
  339. h = load_history()
  340. if node_or_relationship in h:
  341. date_ob = dateutil.parser.parse(h['date'])
  342. if datetime.now() - date_ob < timedelta(days=7):
  343. return True
  344.  
  345. return False
  346.  
  347.  
  348. def completed_now(node_or_relationship):
  349. h = load_history()
  350. h[node_or_relationship] = {"date": datetime.now().isoformat()}
  351. save_history()
  352.  
  353.  
  354. download_by_tags(["product"], True)
  355.  
  356. exit(0)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement