API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Mar 3rd, 2017
101
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 16.89 KB | None | 0 0
raw download clone embed print report
  1. #!/bin/bash
  2. #------------------------------------------------------------------------------#
  3. # Demo script illustrating some examples using the OData interface #
  4. # of the Data Hub Service (DHuS) #
  5. #------------------------------------------------------------------------------#
  6. # GAEL Systems 2014 #
  7. #------------------------------------------------------------------------------#
  8. # Change log #
  9. # 2014-11-07 v1.0 First version listing collections, products, matching name #
  10. # or ingestion date. Download of the Manifest file. #
  11. # 2014-11-11 v1.1 Add search by AOI, product type and last <n> days. Get #
  12. # Polarisation & relative orbit values, Download quick-look #
  13. # or full product. #
  14. # 2014-11-18 v1.2 Add case listing products from specific acquisition date #
  15. # Date operations are now forced in UTC #
  16. # Date operations supporting both Linux and Max OSX syntax #
  17. # -V command line option added to display script version #
  18. #------------------------------------------------------------------------------#
  19.  
  20. # Define default options and variables
  21. VERSION="1.2"
  22. DHUS_SERVER_URL="https://scihub.esa.int/dhus"
  23. DHUS_USER="username"
  24. DHUS_PASSWD="password"
  25. JSON_OPT=false
  26. VERBOSE=false
  27. RESULT_LIST=""
  28.  
  29. # Display help
  30. function show_help()
  31. {
  32. echo "USAGE: odata-demo.sh [OPTIONS]... "
  33. echo "This script illustrates sample scenarios using the OData inteface of the Data Hub Service (DHuS)."
  34. echo "OPTIONS are:"
  35. echo " -h, --help display this help message"
  36. echo " -j, --json use json output format for OData (default is xml)"
  37. echo " -p, --password=PASSWORD use PASSWORD as password for the Data Hub"
  38. echo " -s, --server=SERVER use SERVER as URL of the Data Hub Server"
  39. echo " -u, --user=NAME use NAME as username for the Data Hub"
  40. echo " -v, --verbose display curl command lines and results"
  41. echo " -V, --version display the current version of the script"
  42. }
  43.  
  44. # Display version
  45. function show_version()
  46. {
  47. echo "odata-demo.sh $VERSION"
  48. }
  49.  
  50. # Display a banner with the passed text (limited to 20 lines)
  51. function show_text()
  52. {
  53. echo "--------------------------------------------------"
  54. echo "$1" | head -20
  55. [ $(echo "$1" | wc -l) -gt 20 ] && echo "[Truncated to 20 lines]..."
  56. echo "--------------------------------------------------"
  57. }
  58.  
  59. # Return list of values for the passed field name from the result file depending on its json or xml format
  60. function get_field_values()
  61. {
  62. field_name="$1"
  63. if [ "$USE_JQ" = "true" ]
  64. then
  65. RESULT_LIST=$(jq ".d.results[].$field_name" "$OUT_FILE" | tr -d '"')
  66. else
  67. RESULT_LIST=$(cat "$OUT_FILE" | xmlstarlet sel -T -t -m "//*[local-name()='entry']//*[local-name()='$field_name']" -v '.' -n)
  68. fi
  69. }
  70.  
  71. # Display numbered list of items from a multiple lines variable
  72. function show_numbered_list()
  73. {
  74. # Get number of items in the list
  75. LIST="$1"
  76. if [ ! "$LIST" ]
  77. then
  78. nb_items=0
  79. echo "Result list is empty."
  80. return
  81. fi
  82. # Loop on list and add number as prefix
  83. nb_items=$(echo "$LIST" | wc -l | tr -d ' ')
  84. echo "Result list has $nb_items item(s):"
  85. OLD_IFS=$IFS
  86. IFS=$'\n'
  87. i=0
  88. for item in $LIST
  89. do
  90. i=$(expr $i + 1)
  91. echo " $i. $item"
  92. done
  93. IFS=$OLD_IFS
  94. }
  95.  
  96. # Query the server and return json or xml output, with optional verbose mode
  97. # Args are URL JSON_FILTER XML_FILTER
  98. function query_server()
  99. {
  100. # Get URL and filter space characters
  101. URL="${1// /%20}"
  102.  
  103. # Version using JSON output and jq parsing
  104. if [ "$USE_JQ" = "true" ]
  105. then
  106. # Add "?" to URL if not yet present, or "&" with json format option
  107. if (echo $URL | grep "?" > /dev/null)
  108. then URL="${URL}&\$format=json"
  109. else URL="${URL}?\$format=json"
  110. fi
  111. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX \"$URL\""
  112. $CURL_PREFIX "$URL" > "$OUT_FILE"
  113. [ "$VERBOSE" = "true" ] && show_text "$(jq "." "$OUT_FILE")"
  114.  
  115. # Version using XML output and xmlstarlet parsing
  116. else
  117. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX \"$URL\""
  118. $CURL_PREFIX "$URL" > "$OUT_FILE"
  119. [ "$VERBOSE" = "true" ] && show_text "$(xmlstarlet fo "$OUT_FILE")"
  120. fi
  121. }
  122.  
  123. # Parse command line arguments
  124. for arg in "$@"
  125. do
  126. case "$arg" in
  127. -h | --help) show_help; exit 0 ;;
  128. -j | --json) JSON_OPT=true ;;
  129. -p=* | --password=*) DHUS_PASSWD="${arg#*=}" ;;
  130. -s=* | --server=*) DHUS_SERVER_URL="${arg#*=}" ;;
  131. -u=* | --user=*) DHUS_USER="${arg#*=}" ;;
  132. -v | --verbose) VERBOSE=true ;;
  133. -V | --version) show_version; exit 0 ;;
  134. *) echo "Invalid option: $arg" >&2; show_help; exit 1 ;;
  135. esac
  136. done
  137.  
  138. # Set variables depending to optional command line arguments
  139. ROOT_URL_ODATA="$DHUS_SERVER_URL/odata/v1"
  140. ROOT_URL_SEARCH="$DHUS_SERVER_URL/search"
  141. CURL_PREFIX="curl -gu $DHUS_USER:$DHUS_PASSWD"
  142.  
  143. # Check if needed commands are present (date (differs on Linux and OSX), curl, then jq or xmlstarlet)
  144. USE_JQ=false
  145. USE_XMLST=false
  146. USE_DATEV=false
  147. if $(date -v-1d &> /dev/null)
  148. then USE_DATEV=true
  149. fi
  150. if ! $(type curl &> /dev/null)
  151. then echo "Command \"curl\" is missing, please install it first!"; exit 1
  152. fi
  153. if [ "$JSON_OPT" = "true" ]
  154. then
  155. if ! $(type jq &> /dev/null)
  156. then echo "Command \"jq\" is missing, please install it first!"; exit 1
  157. fi
  158. USE_JQ=true
  159. OUT_FILE="/tmp/result.json"
  160. else
  161. if ! $(type xmlstarlet &> /dev/null)
  162. then echo "Command \"xmlstarlet\" is missing, please install it first!"; exit 1
  163. fi
  164. OUT_FILE="/tmp/result.xml"
  165. fi
  166.  
  167. # Menu: Ask which scenario to start
  168. while true
  169. do
  170. echo ""
  171. echo "Choose a sample demo:"
  172. echo " 1. List the collections"
  173. echo " 2. List <n> products from a specified collection"
  174. echo " 3. List first 10 products matching part of product name"
  175. echo " 4. List first 10 products matching a specific ingestion date"
  176. echo " 5. List first 10 products matching a specific aquisition date"
  177. echo " 6. List first 10 products since last <n> days, by product type and intersecting an AOI"
  178. echo " 7. Get product id from product name"
  179. echo " 8. Get polarisation from a product id"
  180. echo " 9. Get relative orbit from a product id"
  181. echo " 10. Download Manifest file from a product id"
  182. echo " 11. Download quick-look from a product id"
  183. echo " 12. Download full product from its id"
  184. echo " q. Quit"
  185. echo -n "Please enter the selected item number: "
  186. read answer
  187. case $answer in
  188. 1) # List the collections
  189. # Build URL and query server to get result list
  190. query_server "${ROOT_URL_ODATA}/Collections"
  191. get_field_values "Name"
  192. # Display result list
  193. show_numbered_list "$RESULT_LIST"
  194. ;;
  195.  
  196. 2) # List <n> products from a specified collection
  197. # Ask for a collection name and filter potential quotes
  198. echo -n "Please enter the name of a collection (e.g. one from step 1., default=none): "
  199. read colname; colname=${colname//\"/}
  200. # Ask for top and skip limiters
  201. echo -n "How many products to list [1-n], default=10: "
  202. read nbtop; [ -z "$nbtop" ] && nbtop=10
  203. echo -n "Starting from [0-p], default=0: "
  204. read nbskip; [ -z "$nbskip" ] && nbskip=0
  205. # Build URL and query server to get result list
  206. if [ -z "$colname" ]
  207. then query_server "${ROOT_URL_ODATA}/Products?\$skip=$nbskip&\$top=$nbtop"
  208. else query_server "${ROOT_URL_ODATA}/Collections('$colname')/Products?\$skip=$nbskip&\$top=$nbtop"
  209. fi
  210. get_field_values "Name"
  211. # Display result list
  212. show_numbered_list "$RESULT_LIST"
  213. ;;
  214.  
  215. 3) # List first 10 products matching part of product name
  216. # Ask for a product name part and remove potential quotes
  217. echo -n "Please enter the name part to match (e.g. GRD, EW, 201410, default=SLC): "
  218. read namepart; namepart=${namepart//\"/}; [ -z "$namepart" ] && namepart="SLC"
  219. # Build URL and query server to get result list
  220. query_server "${ROOT_URL_ODATA}/Products?\$select=Id&\$filter=substringof('$namepart',Name)&\$top=10"
  221. get_field_values "Name"
  222. # Display result list
  223. show_numbered_list "$RESULT_LIST"
  224. ;;
  225.  
  226. 4) # List first 10 products matching a specific ingestion date
  227. # Ask for ingestion date, default is today
  228. echo -n "Please enter the ingestion date (YYYYMMDD format, default today=$(date -u +%Y%m%d)): "
  229. read idate; [ -z "$idate" ] && idate=$(date -u +%Y%m%d)
  230. year=${idate:0:4}
  231. month=${idate:4:2}
  232. day=${idate:6:2}
  233. # Build URL and query server to get result list
  234. query_server "${ROOT_URL_ODATA}/Products?\$filter=year(IngestionDate)+eq+$year+and+month(IngestionDate)+eq+$month+and+day(IngestionDate)+eq+$day&\$top=10"
  235. get_field_values "Name"
  236. # Display result list
  237. show_numbered_list "$RESULT_LIST"
  238. ;;
  239.  
  240. 5) # List first 10 products matching a specific aquisition date
  241. # Display a notice due to the current limitations
  242. echo "PLEASE NOTE: the current version is getting a list of 1000 products from the server and filters the results locally. Getting this list may take some time. Additional filter at server level will be available in future evolutions. Another way to do this is to use demo case 3. with the following date pattern YYYYMMDD, which is part of the product name."
  243. # Ask for acquisition date, default is yesterday, manage date command syntax for Linux or Mac OSX
  244. if [ "$USE_DATEV" = "true" ]
  245. then yesterday=$(date -u -v-1d +%Y%m%d)
  246. else yesterday=$(date -u -d 'yesterday' +%Y%m%d)
  247. fi
  248. echo -n "Please enter the acquisition date (YYYYMMDD format, default yesterday=$yesterday): "
  249. read acq_date; [ -z "$acq_date" ] && acq_date=$yesterday
  250. # Build URL and query server to get result list
  251. query_server "${ROOT_URL_ODATA}/Products?\$top=1000"
  252. if [ "$USE_DATEV" = "true" ]
  253. then
  254. millistart=$(date -u -j -f %Y%m%d-%H%M%S "$acq_date-000000" +%s000)
  255. millistop=$(expr $millistart \+ 86399999)
  256. acq_date2=$(date -u -j -f %Y%m%d "$acq_date" +%Y-%m-%d)
  257. else
  258. millistart=$(date -u -d "$acq_date" +%s000)
  259. millistop=$(date -u -d "$acq_date+1day-1sec" +%s999)
  260. acq_date2=$(date -u -d "$acq_date" +%Y-%m-%d)
  261. fi
  262. if [ "$USE_JQ" = "true" ]
  263. then
  264. RESULT_LIST=$(jq ".d.results | map(select(.ContentDate.Start >= \"/Date($millistart)/\" and .ContentDate.Start <= \"/Date($millistop)/\")) | .[].Name" "$OUT_FILE" | tr -d '"')
  265. else
  266. RESULT_LIST=$(xmlstarlet sel -T -t -m '//_:entry' -i "contains(.//d:ContentDate/d:Start/text(),\"$acq_date2\")" -c './_:title/text()' -n "$OUT_FILE")
  267. fi
  268. # Display result list
  269. show_numbered_list "$RESULT_LIST"
  270. ;;
  271.  
  272. 6) # List first 10 products since last <n> days, by product type and intersecting an AOI
  273. # Display a notice due to the current use of /search
  274. echo "PLEASE NOTE: the current syntax is using the /search api instead of pure OData. Additional functions including geographical search will be also available via the OData API in future evolutions."
  275. # Ask for the query parameters
  276. echo -n "Please enter the number of days from today (default=1): "
  277. read lastdays; [ -z "$lastdays" ] && lastdays=1
  278. echo -n "Please enter the selected product type (e.g. SLC, default=GRD): "
  279. read ptype; [ -z "$ptype" ] && ptype="GRD"
  280. polygon_default="POLYGON((-15.0 47.0,5.5 47.0,5.5 60.0,-15.5 60.0,-15.50 47.0,-15.0 47.0))"
  281. echo -n "Please enter the AOI polygon, first and last points shall be the same. Defaults is $polygon_default: "
  282. read polygon; [ -z "$polygon" ] && polygon="$polygon_default"
  283. # Build query and replace blanks spaces by '+'
  284. query="ingestiondate:[NOW-${lastdays}DAYS TO NOW] AND producttype:${ptype} AND footprint:\"Intersects(${polygon})\""
  285. query_server "${ROOT_URL_SEARCH}?q=${query// /+}"
  286. if [ "$USE_JQ" = "true" ]
  287. then
  288. RESULT_LIST=$(jq ".feed.entry[].id" "$OUT_FILE" | tr -d '"')
  289. else
  290. RESULT_LIST=$(cat "$OUT_FILE" | xmlstarlet sel -T -t -m '//_:entry/_:id/text()' -v '.' -n)
  291. fi
  292. # Display result list
  293. show_numbered_list "$RESULT_LIST"
  294. ;;
  295.  
  296. 7) # Get product id from product name
  297. # Ask for a product name and remove potential quotes
  298. echo -n "Please enter the name of the product (e.g. one from previous steps): "
  299. read prodname; prodname=${prodname//\"/}
  300. # Build URL and query server to get result list
  301. query_server "${ROOT_URL_ODATA}/Products?\$filter=Name+eq+'$prodname'"
  302. get_field_values "Id"
  303. # Display result list
  304. show_numbered_list "$RESULT_LIST"
  305. ;;
  306.  
  307. 8) # Get polarisation from a product id
  308. # Ask for a product id
  309. echo -n "Please enter the id of the product (e.g. one from step 5.): "
  310. read prodid; prodid=${prodid//\"/}
  311. # Build URL to get polarisation
  312. URL="${ROOT_URL_ODATA}/Products('$prodid')/Attributes('Polarisation')/Value/\$value"
  313. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX \"$URL\""
  314. value=$($CURL_PREFIX "$URL")
  315. show_text "Polarisation = $value"
  316. ;;
  317.  
  318. 9) # Get relative orbit from a product id
  319. # Ask for a product id
  320. echo -n "Please enter the id of the product (e.g. one from step 5.): "
  321. read prodid; prodid=${prodid//\"/}
  322. # Build URL to get relative orbit
  323. URL="${ROOT_URL_ODATA}/Products('$prodid')/Attributes('Relative%20orbit%20(start)')/Value/\$value"
  324. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX \"$URL\""
  325. value=$($CURL_PREFIX "$URL")
  326. show_text "Relative orbit (start) = $value"
  327. ;;
  328.  
  329. 10) # Download Manifest file from a product id
  330. # Ask for a product id
  331. echo -n "Please enter the id of the product (e.g. one from step 5.): "
  332. read prodid; prodid=${prodid//\"/}
  333. # Build URL to get product name
  334. URL="${ROOT_URL_ODATA}/Products('$prodid')/Name/\$value"
  335. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX \"$URL\""
  336. prodname=$($CURL_PREFIX "$URL")
  337. [ "$VERBOSE" = "true" ] && show_text "$prodname"
  338. # Build URL to get Manifest node
  339. URL="${ROOT_URL_ODATA}/Products('$prodid')/Nodes('$prodname.SAFE')/Nodes('manifest.safe')/\$value"
  340. output_file="$prodname.manifest.safe"
  341. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX -o "$output_file" \"$URL\""
  342. $CURL_PREFIX -o "$output_file" "$URL" && echo "Manifest file saved as \"$output_file\""
  343. [ "$VERBOSE" = "true" ] && show_text "$(cat "$output_file")"
  344. ;;
  345.  
  346. 11) # Download quick-look from a product id
  347. # Ask for a product id
  348. echo -n "Please enter the id of the product (e.g. one from step 5.): "
  349. read prodid; prodid=${prodid//\"/}
  350. # Build URL to get product name
  351. URL="${ROOT_URL_ODATA}/Products('$prodid')/Name/\$value"
  352. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX \"$URL\""
  353. prodname=$($CURL_PREFIX "$URL")
  354. [ "$VERBOSE" = "true" ] && show_text "$prodname"
  355. # Build URL to get quick-look
  356. URL="${ROOT_URL_ODATA}/Products('$prodid')/Nodes('$prodname.SAFE')/Nodes('preview')/Nodes('quick-look.png')/\$value"
  357. output_file="$prodname.quick-look.png"
  358. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX -o "$output_file" \"$URL\""
  359. $CURL_PREFIX -o "$output_file" "$URL" && echo "Quick-look file saved as \"$output_file\""
  360. ;;
  361.  
  362. 12) # Download full product from its id
  363. # Ask for a product id
  364. echo -n "Please enter the id of the product (e.g. one from step 5.): "
  365. read prodid; prodid=${prodid//\"/}
  366. # Build URL to get product
  367. URL="${ROOT_URL_ODATA}/Products('$prodid')/\$value"
  368. [ "$VERBOSE" = "true" ] && show_text "$CURL_PREFIX -JO \"$URL\""
  369. $CURL_PREFIX -JO "$URL"
  370. ;;
  371.  
  372. q) echo "Bye."
  373. exit 0;;
  374.  
  375. *) echo "Invalid selection \"$answer\"!" ;;
  376. esac
  377. #echo -n "Press ENTER to continue..."; read key
  378. done
  379.  
  380. # Exit
  381. exit 0
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!