daily pastebin goal
43%
SHARE
TWEET

Untitled

a guest May 30th, 2017 70 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/python3
  2. # -*- coding: utf-8 -*-
  3. from bs4 import BeautifulSoup
  4. import requests
  5. import pymysql
  6.  
  7. db = pymysql.connect(host='127.0.0.1', user='spaceapp2017',
  8.         password='5PGfPUMlNqt7HyFe', db='spaceapp2017', charset='utf8')
  9.  
  10. cursor = db.cursor()
  11.  
  12. # Prepare SQL query to INSERT a record into the database.
  13. sql = "INSERT INTO rain_10min(batch, area, subarea, 10min, 1hr, 3hr, 6hr, 12hr, 24hr, today, day1, day2) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
  14.  
  15. r = requests.get('http://www.cwb.gov.tw/V7/observe/rainfall/A136.htm')
  16. r.encoding = 'utf-8'
  17. soup = BeautifulSoup(r.text, 'html.parser')
  18.  
  19. data = []
  20. table = soup.find('table', attrs={'class':'tablesorter'})
  21. table_body = table.find('tbody')
  22.  
  23. rows = table_body.find_all('tr')
  24. for row in rows:
  25.     cols = row.find_all('td')
  26.     cols = [ele.get_text() for ele in cols]
  27.     data.append([ele for ele in cols if ele]) # Get rid of empty values
  28. #print(data)
  29.  
  30. batch = 0
  31. cols = soup.select('table.description tr td') # , attrs={'class':'description'}
  32. batch = cols[3]
  33. batch = batch.get_text()
  34. tag = batch.index(':')
  35. batch = batch[tag+2:len(batch)-3]
  36. batch = batch.replace("/", "")
  37. batch = batch.replace(" ", "")
  38. batch = batch.replace(":", "")
  39.  
  40. for i in range(len(data)):
  41.     val = []
  42.     for j in range(len(data[0])):
  43.         if(data[i][j] == '-' or data[i][j] == 'X'):
  44.             val.append(None)
  45.         elif(j != 0 and j != 1):
  46.             val.append(float(data[i][j]))
  47.     tag = data[i][1].index(' (')
  48.     cursor.execute(sql, (batch, data[i][0], data[i][1][:tag], val[0], val[1],
  49.                     val[2], val[3], val[4], val[5], val[6], val[7], val[8]))
  50.  
  51. '''path = './10minrain.txt'
  52. rain = open(path,'w')
  53. for i in data:
  54.    for j in i:
  55.        rain.write(str(j) + ' ')
  56.    rain.write('\n')'''
  57.  
  58. db.commit()
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top