API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Dec 2nd, 2016
131
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 17.66 KB | None | 0 0
raw download clone embed print report
  1. ##
  2. ##
  3. ## __________ .____ .___ _________
  4. ## \______ \| | | | / _____/
  5. ## | _/| | | | \_____ \
  6. ## | | \| |___ | | / \
  7. ## |____|_ /|_______ \|___|/_______ /
  8. ## \/ \/ \/
  9. ## _____ __ .___ __
  10. ## / \ ____ _/ |_ _____ __| _/_____ _/ |_ _____
  11. ## / \ / \ _/ __ \\ __\\__ \ / __ | \__ \ \ __\\__ \
  12. ## / Y \\ ___/ | | / __ \_/ /_/ | / __ \_| | / __ \_
  13. ## \____|__ / \___ >|__| (____ /\____ | (____ /|__| (____ /
  14. ## \/ \/ \/ \/ \/ \/
  15. #
  16. # Ben Sainsbury
  17. # 2013.08.08
  18. #
  19. # Creates XML metadata using the record returned from the
  20. # database.
  21. # Here is a rough sketch of the data model for the
  22. # XML export:
  23. #
  24. # - Root Meta Layer Info
  25. # - Tables
  26. # - Columns
  27. # - Lineage
  28. #
  29. # Note that the stored function
  30. # METADATA.GETDISCOVERYMETA returns 'stringified
  31. # JSON objects for the child elements = easily
  32. # parsed by Python into dict/list objects
  33. #
  34. ###################################################
  35.  
  36. import cx_Oracle, os, shutil, Image, base64
  37. from HTMLParser import HTMLParser
  38. from sys import path
  39. path.append("//alex/rlis/drcTools/Libraries/Python/")
  40. import MetroUtil as mu
  41. #from validation import validate
  42.  
  43. OUTPUT_DIRECTORY = r'\\alex\rlis\rlislite\DVD\ESRISHAPEFILES\\'
  44.  
  45. IMAGE_LIBRARY = r'\\atlas\www\prod\library\rlisdiscovery\browse_graphic'
  46.  
  47. def append_text(val):
  48. global f
  49. f.write(val+'\n')
  50.  
  51. def format(wrap, val):
  52. instr = val.replace("&nbsp", " ")
  53. if "<" in instr or "&" in instr:
  54. instr= "<![CDATA[" + instr + "]]>"
  55. instr=instr.replace("\n","")
  56. instr=instr.strip()
  57. return wrap.format(word=instr)
  58.  
  59. def encode_thumbnail(filename):
  60. im = Image.open(IMAGE_LIBRARY+os.sep+filename)
  61. im.convert('RGB').save(r"c:\temp\outfile.jpg")
  62. with open(r"c:\temp\outfile.jpg", "rb") as image_file:
  63. encoded_string = base64.b64encode(image_file.read())
  64. return encoded_string
  65.  
  66. def update_layer(layer_id, field, value):
  67. try:
  68. stringval = value
  69. numericval = -99
  70.  
  71. if type(value) == int or type(value) == float:
  72. numericval=value
  73. stringval=''
  74.  
  75. params=[layer_id, field, stringval, numericval]
  76. conn = cx_Oracle.connect(mu.METADATA_RLISPROD_CONN_STRING)
  77. curs = conn.cursor()
  78. out_parameter = curs.var(cx_Oracle.NUMBER)
  79. result = curs.callfunc('UPDATE_LAYER', out_parameter, params)
  80.  
  81. print result
  82. except Exception as e:
  83. print "Update failed: " + str(e)
  84. finally:
  85. conn.close()
  86.  
  87. def update_layer_date(layer_id):
  88.  
  89. _now = datetime.datetime.now()
  90.  
  91. update_layer(layer_id, 'TIME_PERIOD_OF_CONTENT', _now.strftime('%Y%m%d'))
  92. update_layer(layer_id, 'METADATA_DATE', _now.strftime('%Y%m%d'))
  93.  
  94. def create_metadata(layer_id, path=None, include_image=True):
  95. global f
  96.  
  97. try:
  98. conn = cx_Oracle.connect(mu.METADATA_RLISPROD_CONN_STRING)
  99.  
  100. curs = conn.cursor()
  101.  
  102. out_parameter = curs.var(cx_Oracle.CURSOR)
  103.  
  104. params =[]
  105.  
  106. if type(layer_id) is int:
  107. params=[layer_id, '']
  108. else:
  109. params=[0,layer_id]
  110.  
  111. data = curs.callfunc('GETDISCOVERYMETA', out_parameter, params)
  112.  
  113. fieldNames = map(lambda foo: foo[0], data.description)
  114.  
  115. row = dict(zip(fieldNames, data.fetchone()))
  116.  
  117. row["LINEAGE"] = row["LINEAGE"].read();
  118. row["TABLES"] = row["TABLES"].read();
  119.  
  120. #Quick little bit of defense here
  121. for k in row.keys():
  122. row[k]='' if row[k] is None else row[k]
  123. row[k]='' if row[k] =='none' else row[k]
  124.  
  125. lineage = eval(row["LINEAGE"])["lineage"]
  126. tables = eval(row["TABLES"])["tables"]
  127.  
  128. except Exception, e:
  129. print str(e)
  130. finally:
  131. conn.close()
  132.  
  133. file_name = row['FILE_NAME'].split('/')
  134. file_name = file_name[len(file_name)-1]
  135.  
  136. out_path = os.path.join(OUTPUT_DIRECTORY, str(row['CATEGORY']).upper())
  137.  
  138. if not os.path.exists(out_path):
  139. os.makedirs(out_path)
  140.  
  141. if path is None:
  142. path= os.path.join(out_path, file_name+'.xml')
  143.  
  144. f=open(path, 'w')
  145.  
  146. append_text('<?xml version="1.0"?>')
  147. append_text('<metadata>')
  148. append_text(' <idinfo>')
  149. append_text(' <citation>')
  150. append_text(' <citeinfo>')
  151. append_text(format(' <origin>{word}</origin>', row['ORIGINATOR']))
  152. append_text(format(' <pubdate>{word}</pubdate>', row['PUBLICATION_DATE']))
  153. append_text(format(' <title>{word}</title>', row['TITLE']))
  154. append_text(' </citeinfo>')
  155. append_text(' </citation>')
  156. append_text(' <descript>')
  157. append_text(format(' <abstract>{word}</abstract>', row['ABSTRACT']))
  158. append_text(format(' <purpose>{word}</purpose>', row['PURPOSE']))
  159. append_text(format(' <supplinf>{word}</supplinf>' , row['SUPPLEMENTAL_INFORMATION']))
  160. append_text(' </descript>')
  161. append_text(' <timeperd>')
  162. append_text(' <timeinfo>')
  163. append_text(' <sngdate>')
  164. append_text(format(' <caldate>{word}</caldate>', row['TIME_PERIOD_OF_CONTENT']))
  165. append_text(' </sngdate>')
  166. append_text(' </timeinfo>')
  167. append_text(' <current>Ground condition</current>')
  168. append_text(' </timeperd>')
  169. append_text(' <status>')
  170. append_text(format(' <progress>{word}</progress>', row['PROGRESS']))
  171. append_text(format(' <update>{word}</update>', row['MAINT_UPDATE_FREQUENCY']))
  172. append_text(' </status>')
  173. append_text(' <spdom>')
  174. append_text(' <bounding>')
  175. append_text(format(' <westbc>{word}</westbc>', row['WEST_BOUNDING_COORDINATE']))
  176. append_text(format(' <eastbc>{word}</eastbc>', row['EAST_BOUNDING_COORDINATE']))
  177. append_text(format(' <northbc>{word}</northbc>', row['NORTH_BOUNDING_COORDINATE']))
  178. append_text(format(' <southbc>{word}</southbc>', row['SOUTH_BOUNDING_COORDINATE']))
  179. append_text(' </bounding>')
  180. append_text(' </spdom>')
  181. append_text(' <keywords>')
  182. append_text(' <theme>')
  183. append_text(format(' <themekt>{word}</themekt>', row['THEME_KEYWORD_THESAURUS']))
  184.  
  185. for keyword in row['THEME_KEYWORD'].split(','):
  186. append_text(format(' <themekey>{word}</themekey>', keyword.strip()))
  187.  
  188. append_text(' </theme>')
  189. append_text(' <place>')
  190. append_text(format(' <placekt>{word}</placekt>', row['PLACE_KEYWORD_THESAURUS']))
  191.  
  192. for place in row['PLACE_KEYWORD'].split(','):
  193. append_text(format(' <placekey>{word}</placekey>', place.strip()))
  194.  
  195. append_text(' </place>')
  196. append_text(' </keywords>')
  197. append_text(format(' <accconst>{word}</accconst>', row['ACCESS_CONSTRAINTS']))
  198. append_text(format(' <useconst>{word}</useconst>', row['USE_CONSTRAINTS']))
  199.  
  200. append_text(' <browse>')
  201. append_text(format(' <browsen>http://library.oregonmetro.gov/rlisdiscovery/browse_graphic/{word}</browsen>', row['BROWSE_GRAPHIC_FILE_NAME']))
  202. append_text(format(' <browsed>{word}</browsed>', row['BROWSE_GRAPHIC_FILE_DESC']))
  203. append_text(format(' <browset>{word}</browset>', row['BROWSE_GRAPHIC_FILE_TYPE']))
  204. append_text(' </browse>')
  205. append_text(' </idinfo>')
  206. append_text(' <dataqual>')
  207. append_text(' <logic></logic>')
  208. append_text(' <complete></complete>')
  209. append_text(' <posacc>')
  210. append_text(' <horizpa>')
  211.  
  212. if row['HORIZONTAL_POS_ACC_LINK'] == "Taxlots":
  213. append_text('\t\t\t\t<horizpar>Registered to tax lots. Street center lines were digitized from aerial orthophotography at +/- 10 feet accuracy. Tax lots were fit to the street centerlines. Beaverton, Oregon City and Tigard planning areas were improved to +/- 5 feet accuracy. Multnomah County was created with coordinate geometry using better control.</horizpar>')
  214. elif row['HORIZONTAL_POS_ACC_LINK'] == "Streets":
  215. append_text('\t\t\t\t<horizpar>Registered to streets. The street linework is contained within the street rights-of-way as shown on the tax lot layer. As the tax lot horizonal accuracy is improved, streets are adjusted accordingly. The tax lots are digitized at +/- 10 feet accuracy. Beaverton, Oregon City and Tigard planning areas are improved to +/- 5 feet accuracy. Multnomah County was created with coordinate geometry using better control.</horizpar>')
  216. else:
  217. append_text('\t\t\t\t<horizpar>'+row['HORIZONTAL_POS_ACCURACY']+'</horizpar>')
  218.  
  219. append_text(' </horizpa>')
  220. append_text(' </posacc>')
  221. append_text(' <lineage>')
  222.  
  223. for process in lineage:
  224. append_text(' <procstep>')
  225. append_text(format(' <procdesc>{word}</procdesc>', process['desc']))
  226. append_text(format(' <procdate>{word}</procdate>', process['date']))
  227. append_text(' </procstep>')
  228.  
  229. append_text(' </lineage>')
  230. append_text(' </dataqual>')
  231. append_text(' <spdoinfo>')
  232. append_text(format(' <direct>{word}</direct>', row['DIRECT_SPATIAL_REF_METHOD']))
  233.  
  234. if row['DIRECT_SPATIAL_REF_METHOD'] == "Vector" or row['DIRECT_SPATIAL_REF_METHOD'] == "Point":
  235. append_text(' <ptvctinf>')
  236. append_text(' <sdtsterm>')
  237. append_text(format(' <sdtstype>{word}</sdtstype>', row['SDTS_PT_AND_VEC_OBJ_TYPE']))
  238. append_text(' </sdtsterm>')
  239. append_text(' </ptvctinf>')
  240. elif row['DIRECT_SPATIAL_REF_METHOD'] == "Raster":
  241. append_text(' <rastinfo>')
  242. append_text(format(' <rasttype>{word}</rasttype>', row['RASTER_OBJECT_TYPE']))
  243. append_text(' </rastinfo>')
  244.  
  245. append_text(' </spdoinfo>')
  246. append_text(' <spref>')
  247. append_text(' <horizsys>')
  248. append_text(' <planar>')
  249. append_text(' <gridsys>')
  250. append_text(format(' <gridsysn>{word}</gridsysn>', row['GRID_COORD_SYSTEM_NAME']))
  251. append_text(' <spcs>')
  252. append_text(format(' <spcszone>{word}</spcszone>', row['SPCS_ZONE_IDENTIFIER']))
  253. append_text(' </spcs>')
  254. append_text(' </gridsys>')
  255. append_text(' <planci>')
  256. append_text(' <plance></plance>')
  257. append_text(format(' <plandu>{word}</plandu>', row['PLANAR_DISTANCE_UNITS']))
  258. append_text(' </planci>')
  259. append_text(' </planar>')
  260. append_text(' <geodetic>')
  261. append_text(format(' <horizdn>{word}</horizdn>', row['HORIZONTAL_DATUM_NAME']))
  262. append_text(' <ellips></ellips>')
  263. append_text(' <semiaxis></semiaxis>')
  264. append_text(' <denflat></denflat>')
  265. append_text(' </geodetic>')
  266. append_text(' </horizsys>')
  267. append_text(' </spref>')
  268.  
  269. append_text(' <eainfo>')
  270.  
  271. if tables != None:
  272. for table in tables:
  273. append_text(' <detailed>')
  274. append_text(' <enttyp>')
  275. append_text(format(' <enttypl>{word}</enttypl>', tables[0]['name']))
  276. append_text(format(' <enttypd>{word}</enttypd>', tables[0]['desc']))
  277. append_text(' <enttypds></enttypds>')
  278. append_text(' </enttyp>')
  279.  
  280. for col in table['cols']:
  281. append_text(' <attr>')
  282. append_text(format(' <attrlabl>{word}</attrlabl>', col['name']))
  283. append_text(format(' <attrdef>{word}</attrdef>', col['desc']))
  284. append_text(' <attrdefs></attrdefs>')
  285. append_text(' <attrdomv>')
  286.  
  287. for domain_val in col['domain_values']:
  288. append_text(' <edom>')
  289. append_text(format(' <edomv>{word}</edomv>', domain_val['value']))
  290. append_text(format(' <edomvd>{word}</edomvd>', domain_val['desc']))
  291. append_text(' <edomvds></edomvds>')
  292. append_text(' </edom>')
  293. else:
  294. append_text(' <udom>NA</udom>')
  295. append_text(' </attrdomv>')
  296. append_text(' </attr>')
  297.  
  298. append_text(' </detailed>')
  299. append_text(' </eainfo>')
  300. append_text(' <distinfo>')
  301. append_text(' <distrib>')
  302. append_text(' <cntinfo>')
  303. append_text(' <cntorgp>')
  304. append_text(format(' <cntorg>{word}</cntorg>', row['CONTACT_ORGANIZATION']))
  305. append_text(' </cntorgp>')
  306. append_text(' <cntaddr>')
  307. append_text(' <addrtype></addrtype>')
  308. append_text(format(' <address>{word}</address>', row['ADDRESS']))
  309. append_text(format(' <city>{word}</city>', row['CITY']))
  310. append_text(format(' <state>{word}</state>',row['STATE_OR_PROVINCE']))
  311. append_text(format(' <postal>{word}</postal>',row['POSTAL_CODE']))
  312. append_text(' </cntaddr>')
  313. append_text(format(' <cntvoice>{word}</cntvoice>', row['CONTACT_VOICE_TELEPHONE']))
  314. append_text(format(' <cntfax>{word}</cntfax>', row['CONTACT_FACSIMILE_TELEPHONE']))
  315. append_text(format(' <cntemail>{word}</cntemail>', row['CONTACT_E_MAIL_ADDRESS']))
  316. append_text(' </cntinfo>')
  317. append_text(' </distrib>')
  318. append_text(format(' <distliab>{word}</distliab>' ,row['DISTRIBUTION_LIABILITY']))
  319. append_text(' <stdorder>')
  320. append_text(' <digform>')
  321. append_text(' <digtinfo>')
  322. append_text(format(' <formname>{word}</formname>', row['FORMAT_NAME']))
  323.  
  324. try:
  325. append_text(format(' <transize>{word}</transize>', str(row['TRANSFER_SIZE']/1000)))
  326. except Exception as e:
  327. append_text(' <transize>0</transize>')
  328.  
  329. append_text(' </digtinfo>')
  330. append_text(' <digtopt>')
  331. append_text(' <offoptn>')
  332. append_text(format(' <offmedia>{word}</offmedia>', row['OFFLINE_MEDIA']))
  333. append_text(format(' <recfmt>{word}</recfmt>', row['RECORDING_FORMAT']))
  334. append_text(' </offoptn>')
  335. append_text(' <onlinopt>')
  336. append_text(' <computer>')
  337. append_text(' <networka>')
  338. append_text(format(' <networkr>{word}</networkr>', row['NETWORK_RESOURCE_NAME']))
  339. append_text(' </networka>')
  340. append_text(' </computer>')
  341. append_text(format(' <accinstr>{word}</accinstr>', row['ACCESS_INSTRUCTIONS']))
  342. append_text(' </onlinopt>')
  343. append_text(' </digtopt>')
  344. append_text(' </digform>')
  345. append_text(format(' <fees>{word}</fees>', row['FEES']))
  346. append_text(' </stdorder>')
  347. append_text(' </distinfo>')
  348. append_text(' <metainfo>')
  349. append_text(format(' <metd>{word}</metd>', row['METADATA_DATE']))
  350. append_text(' <metc>')
  351. append_text(' <cntinfo>')
  352. append_text(' <cntorgp>')
  353. append_text(' <cntorg>Metro Data Resource Center</cntorg>')
  354. append_text(format(' <cntper>{word}</cntper>', row['CONTACT_PERSON']))
  355. append_text(' </cntorgp>')
  356. append_text(' <cntaddr>')
  357. append_text(' <addrtype>mailing and physical address</addrtype>')
  358. append_text(' <address>600 NE Grand Ave.</address>')
  359. append_text(' <city>Portland</city>')
  360. append_text(' <state>OR</state>')
  361. append_text(' <postal>97232</postal>')
  362. append_text(' </cntaddr>')
  363. append_text(format(' <cntvoice>{word}</cntvoice>', row['CONTACT_VOI_TELE']))
  364. append_text(format(' <cntfax>{word}</cntfax> ', row['CONTACT_FAX_TELE']))
  365. append_text(format(' <cntemail>{word}</cntemail>', row['CONTACT_E_MAIL']))
  366. append_text(' </cntinfo>')
  367. append_text(' </metc>')
  368. append_text(format(' <metstdn>{word}</metstdn>', row['METADATA_STANDARD_NAME']))
  369. append_text(format(' <metstdv>{word}</metstdv>', row['METADATA_STANDARD_VERSION']))
  370. append_text(' </metainfo>')
  371.  
  372. if include_image:
  373. if row['BROWSE_GRAPHIC_FILE_NAME']:
  374. encoded_string = encode_thumbnail(row['BROWSE_GRAPHIC_FILE_NAME'])
  375. append_text(' <Binary>')
  376. append_text(' <Thumbnail>')
  377. append_text(' <Data EsriPropertyType="PictureX">'+encoded_string+'</Data>')
  378. append_text(' </Thumbnail>')
  379. append_text(' </Binary>')
  380.  
  381. append_text('</metadata>')
  382.  
  383. f.close()
  384.  
  385. if not path: # using this as a proxy for module-level access to this method vs main.
  386. print "completed metadata file for "+row["TITLE"]+ " (LayerID: "+str(row["LAYER_ID"])+")"
  387. #validate(out_path+'/'+ file_name+'.xml', "D:/scratch/meta_test/VALIDATION")
  388. #print "completed metadata validation for "+row["TITLE"]+ " (LayerID: "+str(row["LAYER_ID"])+")"
  389. return
  390.  
  391. if __name__ == "__main__":
  392.  
  393. layer_id = raw_input("enter layer id (enter 0 to export all):")
  394. #database = raw_input("enter database to export from (PUB, TEST,DEV, PROD)[Usually prod]")
  395.  
  396. if layer_id=='0':
  397. try:
  398. conn = cx_Oracle.connect(mu.METADATA_RLISPROD_CONN_STRING)
  399.  
  400. curs = conn.cursor()
  401.  
  402. x=curs.execute("SELECT distinct l.layer_id FROM metadata_layer l INNER JOIN metadata_layer_format mlf on l.layer_id = mlf.layer_id inner join metadata_category mlc on l.CATEGORY_ID = Mlc.Category_Id where mlf.transfer_on_id = 1 and l.file_name not like 'hillshd%' and Upper(mlc.category) not in ('PHOTO', 'SPECIAL') ORDER by l.layer_id")
  403.  
  404. layers_to_export = x.fetchall()
  405.  
  406. for layer_id in layers_to_export:
  407. create_metadata(int(layer_id[0]))
  408.  
  409. except Exception as e:
  410. print "Metadata publish process has experienced an error:"+str(e)
  411. finally:
  412. conn.close()
  413. else:
  414. create_metadata(int(layer_id))
  415.  
  416. print 'process complete'
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!