API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Sep 6th, 2018
101
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 8.51 KB | None | 0 0
raw download clone embed print report
  1. import requests
  2. import numpy as np
  3. import rasterio
  4. import matplotlib.pyplot as plt
  5. import time
  6. from time import sleep
  7. from meeo_sso_cli import login, getToken
  8. import os
  9. import threading
  10. from functools import partial
  11. import random
  12.  
  13. dic_years = ["2000", "2001", "2002", "2003", "2004", "2005", "2006", "2007", "2008", "2009", "2010"]
  14. dic_m = ["JAN","FEB","MAR","APR","MAY","JUN","JUL","AUG","SEP","OCT","NOV","DEC"]
  15. dic_months = ["00","01","02","03","04","05","06","07","08","09","10","11","12"]
  16. dic_weeks = ["01","02","03","04","05","06","07","08","09","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52"]
  17.  
  18. collection = 'ERA-Interim_temp2m_4326_05'
  19. username = 'premia'
  20. password = '4gr1cult'
  21.  
  22. year_ini = 2000#request.inputs['year_ini'][0].data
  23. year_fin = 2002#request.inputs['year_fin'][0].data
  24. month_ini = 1#request.inputs['month_ini'][0].data
  25. month_fin = 4#request.inputs['month_fin'][0].data
  26.  
  27. #username = str(request.inputs['user'][0].data) #'figuera'
  28. #password = str(request.inputs['pass'][0].data) #'110586ra'
  29.  
  30. #aggr = str(request.inputs['aggr'][0].data)
  31. aggr = 'm'
  32.  
  33. class myThread (threading.Thread):
  34.  
  35.     def __init__(self, access_token,time_t):
  36.         threading.Thread.__init__(self)
  37.         self.access_token = access_token
  38.         self.time_t = time_t
  39.         self.result = None
  40.  
  41.     def run(self):
  42.         print ("Starting " + self.name)
  43.         for i in range(0,len(self.time_t)):
  44.             download_file_paral(self.access_token, self.time_t[i])
  45.         print ("Exiting " + self.name)
  46.  
  47.     def download_file_paral(self):
  48.         #print(access_token)
  49.         #print(time_t)
  50.         #for i in range(0,len(time_t)):
  51.         #print(str(time_t))
  52.         st_mean = []
  53.         #fname = '/home/meeo/wps/processes/result/'+str(random.randint(0,9999))+'.tif'
  54.         fname = str(random.randint(0,9999))+'.tif'
  55.         url = 'https://eodataservice.org/dar05?service=WCS&Request=GetCoverage&version=2.0.0&subset=unix('+str(self.time_t)+')&format=image/tiff&CoverageId=ERA-Interim_temp2m_4326_05'
  56.         params = {'geometry': 'geometry=POLYGON ((11.930252963058445 45.055792724312049,11.986114233195178 45.100873749334674,12.019434990820596 45.075393169974056,12.035115347350205 45.038152323216238,12.005714678857188 45.003851543307718,11.965533765250065 44.989151209061205,11.921432762510539 44.995031342759809,11.97533398808107 45.027372078102132,11.930252963058445 45.055792724312049))',
  57.                           'format':'image/tiff',
  58.                           'CoverageId':'ERA-Interim_temp2m_4326_05',
  59.                           'token': self.access_token}
  60.  
  61.         #execute query
  62.         result = requests.post(url, params=params)
  63.         #print(result)
  64.         with open(fname, 'wb') as f:
  65.             #print(fname)
  66.             f.write(result.content)
  67.             #read data and compute mean value
  68.             src = rasterio.open(fname)
  69.             array = src.read()
  70.             print(np.mean(array))
  71.             st_mean.append(np.mean(array))
  72.         #print(np.mean(array))
  73.         return st_mean
  74.  
  75.     def get_result( self ):
  76.         return self.result
  77.  
  78. def monthly_compute(year_ini, year_fin, month_ini, month_fin, dic_months, collection, access_token):
  79.  
  80.     times = []
  81.     start = time.time()
  82.     #compute time steps
  83.     #fname = "/home/meeo/wps/processes/result/image.tif"
  84.     st_mean = np.zeros((year_fin-year_ini+1)*(month_fin-month_ini+1)) #60
  85.     #x = np.zeros(len(dic_days)*delta_month)
  86.     x = []
  87.  
  88.     #assume aggregation is every hour for one daydic_hours_e
  89.  
  90.     for y in range(year_ini, year_fin+1):
  91.         for m in range(month_ini,month_fin+1):
  92.             if m%2 == 0 and m != 2: #that means pair months
  93.                 time_t = str(y)+'-'+dic_months[m]+'-01T00:00:00,'+str(y)+'-'+dic_months[m]+'-30T23:59:59'
  94.             elif m == 2:
  95.                 time_t = str(y)+'-'+dic_months[m]+'-01T00:00:00,'+str(y)+'-'+dic_months[m]+'-28T23:59:59'
  96.             else:
  97.                 time_t = str(y)+'-'+dic_months[m]+'-01T00:00:00,'+str(y)+'-'+dic_months[m]+'-31T23:59:59'
  98.  
  99.             x.append(str(y) + '-' +dic_m[m-1] )
  100.             times.append(time_t)
  101.     return x, times
  102.  
  103.  
  104. def weekly_compute(year_ini, year_fin, month_ini, month_fin, dic_months, collection, access_token):
  105.  
  106.     times = []
  107.     start = time.time()
  108.     #compute time steps
  109.     dic_days_s = ["01","07","14","21"]
  110.  
  111.     dic_days_31 = ["08","15","22","31"]
  112.     dic_days_30 = ["08","15","22","30"]
  113.     dic_days_28 = ["08","15","22","28"]
  114.  
  115.     #fname = "/home/meeo/wps/processes/result/image.tif"
  116.     st_mean = np.zeros((year_fin-year_ini+1)*(month_fin-month_ini+1)*len(dic_days_31)) #60
  117.     #x = np.zeros(len(dic_days)*delta_month)
  118.     x = []
  119.  
  120.  
  121.     #assume aggregation is every hour for one daydic_hours_e
  122.  
  123.     for y in range(year_ini, year_fin+1):
  124.         for m in range(month_ini,month_fin+1):
  125.             for d in range(0,4):
  126.  
  127.                 if m%2 == 0 and m != 2: #that means pair months
  128.                     time_t = str(y)+'-'+dic_months[m]+'-'+dic_days_s[d]+'T00:00:00,'+str(y)+'-'+dic_months[m]+'-'+dic_days_30[d]+'T23:59:59'
  129.                 elif m == 2:
  130.                     time_t = str(y)+'-'+dic_months[m]+'-'+dic_days_s[d]+'T00:00:00,'+str(y)+'-'+dic_months[m]+'-'+dic_days_28[d]+'T23:59:59'
  131.                 else:
  132.                     time_t = str(y)+'-'+dic_months[m]+'-'+dic_days_s[d]+'T00:00:00,'+str(y)+'-'+dic_months[m]+'-'+dic_days_30[d]+'T23:59:59'
  133.  
  134.                 x.append(str(y) + '-' +dic_m[m-1] )
  135.                 times.append(time_t)
  136.     return x, times
  137.  
  138. def parallel_runs(x, time_t):
  139.     start = time.time()
  140.     print('Start processing in parallel ...')
  141.     #cores = multiprocessing.cpu_count()
  142.     #pool = multiprocessing.Pool(processes=cores) #processes max = n of cores
  143.     #part = partial(download_file_paral, access_token)
  144.     threadLock = threading.Lock()
  145.     threads = []
  146.     result_list = []
  147.     thread1 = myThread(access_token,time_t[0:int(len(time_t)/2)])
  148.     thread2 = myThread(access_token,time_t[int(len(time_t)/2):int(len(time_t))])
  149.     thread1.start()
  150.     thread2.start()
  151.  
  152.     threads.append(thread1)
  153.     threads.append(thread2)
  154.  
  155.     #thread1.join()
  156.     #thread2.join()
  157.     for t in threads:
  158.         t.join()
  159.  
  160.     for thread in threads:
  161.         if thread.get_result() is not None:
  162.             result_list.append( thread.get_result() )
  163.  
  164.  
  165.  
  166.     #result_list = pool.map(unwrap_self_f, zip([part]*len(part), time_t))
  167.     #result_list = pool.map(part, time_t)
  168.     os.system('rm *.tif')
  169.     #print(result_list)
  170.     print("elapsed time: ", time.time()-start, "seconds")
  171.     return result_list
  172.  
  173.  
  174. # def download_file_paral(access_token,time_t):
  175. #     print(access_token)
  176. #     #print(time_t)
  177. #     #for i in range(0,len(time_t)):
  178. #     print(str(time_t))
  179. #     st_mean = []
  180. #     #fname = '/home/meeo/wps/processes/result/'+str(random.randint(0,9999))+'.tif'
  181. #     fname = str(random.randint(0,9999))+'.tif'
  182. #     url = 'https://eodataservice.org/dar05?service=WCS&Request=GetCoverage&version=2.0.0&subset=unix('+str(time_t)+')&format=image/tiff&CoverageId=ERA-Interim_temp2m_4326_05'
  183. #     params = {'geometry': 'geometry=POLYGON ((11.930252963058445 45.055792724312049,11.986114233195178 45.100873749334674,12.019434990820596 45.075393169974056,12.035115347350205 45.038152323216238,12.005714678857188 45.003851543307718,11.965533765250065 44.989151209061205,11.921432762510539 44.995031342759809,11.97533398808107 45.027372078102132,11.930252963058445 45.055792724312049))',
  184. #                       'format':'image/tiff',
  185. #                       'CoverageId':'ERA-Interim_temp2m_4326_05',
  186. #                       'token': access_token}
  187. #
  188. #     #execute query
  189. #     result = requests.post(url, params=params)
  190. #     #print(result)
  191. #     with open(fname, 'wb') as f:
  192. #         #print(fname)
  193. #         f.write(result.content)
  194. #         #read data and compute mean value
  195. #         src = rasterio.open(fname)
  196. #         array = src.read()
  197. #         print(np.mean(array))
  198. #         st_mean.append(np.mean(array))
  199. #     #print(np.mean(array))
  200. #     return st_mean
  201. ##### MAIN #######
  202.  
  203. access_token = login( username, password )[ 'access_token' ]
  204.  
  205. if aggr == 'm':
  206.     x,time_t = monthly_compute(year_ini,year_fin,month_ini,month_fin,dic_months,collection, access_token)
  207. else:
  208.     x,time_t = weekly_compute(year_ini,year_fin,month_ini,month_fin,dic_months,collection, access_token)
  209.  
  210. air_temp = parallel_runs(x,time_t)
  211. print(x)
  212. print(air_temp)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!