SHARE
TWEET

OtoMotoV5

lancernik Apr 25th, 2019 103 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. # -*- coding: utf-8 -*-
  2. """
  3. Created on Thu Apr 25 09:06:22 2019
  4.  
  5. @author: lancernik
  6. """
  7.  
  8. # -*- coding: utf-8 -*-
  9. """
  10. Created on Wed Apr 24 23:35:43 2019
  11.  
  12. @author: lancernik
  13. """
  14.  
  15.  
  16. from requests import get
  17. from requests.exceptions import RequestException
  18. from contextlib import closing
  19. from bs4 import BeautifulSoup
  20. import string
  21. import re
  22. from itertools import groupby
  23. import pandas as pd
  24. import time
  25. import matplotlib.pyplot as plt
  26. import numpy as np
  27. from scipy.stats import gaussian_kde
  28. from scipy.stats import kde
  29. import seaborn as sns
  30.  
  31. from sklearn.linear_model import LinearRegression
  32. from sklearn.model_selection import train_test_split
  33. import matplotlib.pyplot as plt
  34.  
  35.  
  36. def simple_get(url):
  37.     #Zwraca none, w przypadku problemu z pobraniem danych
  38.     try:
  39.         with closing(get(url, stream=True)) as resp:
  40.             if is_good_response(resp):
  41.                 return resp.content
  42.             else:
  43.                 return None
  44.  
  45.     except RequestException as e:
  46.         log_error('Error during requests to {0} : {1}'.format(url, str(e)))
  47.         return None
  48.    
  49. def is_good_response(resp):
  50.     #Zwaraca True, jeżeli HTMl
  51.     content_type = resp.headers['Content-Type'].lower()
  52.     return (resp.status_code == 200
  53.             and content_type is not None
  54.             and content_type.find('html') > -1)
  55.    
  56. def log_error(e):
  57.     print(e)
  58.  
  59. def lastpage(page):
  60.     lastepage_out=0
  61.     lastpage = str(page.find_all(class_="page"))
  62.     lastpage_all = [int(s) for s in re.findall(r'\b\d+\b',lastpage)]
  63.     lastpage_out = lastpage_all[-1]
  64.     return lastepage_out
  65.    
  66. def scrappy(page):  #Pobiera dane z konretnej strony
  67.    
  68.     datadict = {'Milage':[0],'Age':[0],'Price':[0]}
  69.     dataset = pd.DataFrame(data=datadict)
  70.    
  71.    
  72.     #Zdobywa numer ostatniej strony
  73.    
  74.     lastpage = str(page.find_all(class_="page"))
  75.     lastpage_all = [int(s) for s in re.findall(r'\b\d+\b',lastpage)]
  76.     lastpage_out = lastpage_all[-1]
  77.        
  78.     #Scrapowanie przebiegu
  79.  
  80.     milage_from_page = ''.join(map(str,(page.find_all("li", {"data-code" : "mileage"}))))
  81.     milage_from_page_nospace = milage_from_page.translate({ord(c): None for c in string.whitespace})
  82.     milage_page_out = [int(''.join(i)) for is_digit, i in groupby(milage_from_page_nospace, str.isdigit) if is_digit]
  83.    
  84.     #Scrapowanie roku z danej strony
  85.    
  86.     age_from_page = str(page.find_all(class_="offer-item__params-item"))
  87.     age_from_page_nospace = age_from_page.translate({ord(c): None for c in string.whitespace})
  88.     age_from_page_out = [int(s) for s in re.findall(r'\b\d+\b',age_from_page_nospace)]
  89.    
  90.     # Scrapowanie cen z danej strony
  91.    
  92.     price_from_page = str(page.find_all(class_="offer-price__number"))
  93.     price_from_page_nospace = price_from_page.translate({ord(c): None for c in string.whitespace})
  94.     price_from_page_out = [int(s) for s in re.findall(r'\b\d+\b',price_from_page_nospace)]
  95.    
  96.     df = pd.DataFrame(
  97.     {'Milage':milage_page_out,
  98.      'Age': age_from_page_out,
  99.      'Price': price_from_page_out})
  100.    
  101.     dataset = dataset.append(df,ignore_index=True)
  102.    
  103.     return dataset
  104.  
  105.  
  106. def ScrapPage(marka,model,start,stop):  #Oczyszcza dane, wyznacza zares stron
  107.     datadict = {'Milage':[0],'Age':[0],'Price':[0]}
  108.     dataset_out = pd.DataFrame(data=datadict)
  109.     for i in range(start,stop):  #Docelowo 1, lastpage
  110.         time.sleep(2)
  111.        
  112.         #To w formacie beda kolejne argumenty, tj za opel i corsa
  113.         url = simple_get('https://www.otomoto.pl/osobowe/{}/{}/?page={}'.format(marka,model,i))
  114.         page = BeautifulSoup(url, 'html.parser')
  115.         print(scrappy(page))
  116.         dataset_out = dataset_out.append(scrappy(page), ignore_index=True)
  117.         print(dataset_out)
  118.    
  119.    
  120.     #Usuwanie danych odstających
  121.    
  122.     clear = dataset_out.Milage[((dataset_out.Milage - dataset_out.Milage.mean()) / dataset_out.Milage.std()).abs() > 2]
  123.     clear = clear.append(dataset_out.Age[((dataset_out.Age - dataset_out.Age.mean()) / dataset_out.Age.std()).abs() > 2])
  124.     clear = clear.append(dataset_out.Price[((dataset_out.Price - dataset_out.Price.mean()) / dataset_out.Price.std()).abs() > 3])
  125.     test = clear.index.get_values()
  126.    
  127.     for i in range(0,len(test)):
  128.         dataset_out = dataset_out.drop(test[i],axis=0)
  129.  
  130.     return dataset_out
  131.  
  132.  
  133. def ClearCarData():
  134.     clear = dataset_out.Milage[((dataset_out.Milage - dataset_out.Milage.mean()) / dataset_out.Milage.std()).abs() > 2]
  135.     clear = clear.append(dataset_out.Age[((dataset_out.Age - dataset_out.Age.mean()) / dataset_out.Age.std()).abs() > 2])
  136.     clear = clear.append(dataset_out.Price[((dataset_out.Price - dataset_out.Price.mean()) / dataset_out.Price.std()).abs() > 3])
  137.     test = clear.index.get_values()
  138.    
  139.     for i in range(0,len(test)):
  140.         dataset_out = dataset_out.drop(test[i],axis=0)
  141.     return dataset_out    
  142.  
  143. def LoadCarData(filename):
  144.     dataset_out = pd.read_csv('{}.csv'.format(filename))  #9-45
  145.     clear = dataset_out.Milage[((dataset_out.Milage - dataset_out.Milage.mean()) / dataset_out.Milage.std()).abs() > 2]
  146.     clear = clear.append(dataset_out.Age[((dataset_out.Age - dataset_out.Age.mean()) / dataset_out.Age.std()).abs() > 2])
  147.     clear = clear.append(dataset_out.Price[((dataset_out.Price - dataset_out.Price.mean()) / dataset_out.Price.std()).abs() > 3])
  148.     test = clear.index.get_values()
  149.    
  150.     for i in range(0,len(test)):
  151.         dataset_out = dataset_out.drop(test[i],axis=0)
  152.     return dataset_out
  153.  
  154. def regress(x,y):
  155.     model = LinearRegression()
  156.     model.fit(x,y)
  157.     model.predict([[100]])
  158.    
  159.     x_test = np.linspace(min(x),400000)
  160.     y_pred = model.predict(x_test[:,None])
  161.    
  162.     plt.scatter(x,y)
  163.     plt.plot(x_test,y_pred,'r')
  164.     plt.legend(['Regresja', 'Kropeczki'])
  165.     plt.show()
  166.  
  167. def Plot1(x,y):
  168.     # Evaluate a gaussian kde on a regular grid of nbins x nbins over data extents
  169.  
  170.  
  171.    
  172.     nbins=300
  173.     k = kde.gaussian_kde([x,y])
  174.     xi, yi = np.mgrid[x.min():x.max():nbins*1j, y.min():y.max():nbins*1j]
  175.     zi = k(np.vstack([xi.flatten(), yi.flatten()]))
  176.      
  177.     # Make the plot
  178.     plt.pcolormesh(xi, yi, zi.reshape(xi.shape))
  179.     plt.show()
  180.      
  181.     # Change color palette
  182.     plt.pcolormesh(xi, yi, zi.reshape(xi.shape), cmap=plt.cm.Greens_r)
  183.     plt.show()
  184. def Plot2(x,y):
  185.     # Make the plot
  186.     plt.hexbin(x, y, gridsize=(15,15) )
  187.     plt.show()
  188.      
  189.     # We can control the size of the bins:
  190.     plt.hexbin(x, y, gridsize=(150,150) )
  191.     plt.show()
  192. def Plot3(x,y):          
  193.     sns.jointplot(x, y, kind='scatter')
  194.     sns.jointplot(x, y, kind='hex')
  195.     sns.jointplot(x, y, kind='kde')
  196.        
  197.     # Then you can pass arguments to each type:
  198.     sns.jointplot(x, y, kind='scatter', s=200, color='m', edgecolor="skyblue", linewidth=2)
  199.      
  200.     # Custom the color
  201.     sns.set(style="white", color_codes=True)
  202.     sns.jointplot(x, y, kind='kde', color="skyblue",xlim={-30000,300000})
  203.    
  204.    
  205.  
  206.  
  207.    
  208.  
  209.    
  210. #LoadCarData(filename):  Wczytuje dane z pliku CSV stworzonego przez funkcje ScrapPage,
  211. #dodatkowo oczyszcza z danych odstajcych  
  212.  
  213. #ClearCarData():   Oczyszcza z danych odstajacych, zdiala tylko dla df o nazwie dataset_out
  214.  
  215.  
  216.  
  217.  
  218. # 1)   Scrapuje dane
  219.  
  220. #         Marka,   model, start, stop
  221. #ScrapPage("opel" ,"corsa",  1     ,3)
  222. #dataset_out.to_csv('dataset1.csv')
  223. #dataset_out = pd.read_csv('dataset1.csv')  #9-45
  224.  
  225.  
  226. # 2)   Wczytuje zeskrapowane dane
  227.  
  228. dataset_out = LoadCarData("dataset1")
  229.  
  230.  
  231. #Rozne ploty  
  232.  
  233. #x=dataset_out['Milage']
  234. #y=dataset_out['Age']
  235. #
  236. #
  237. #Plot1(x,y)
  238. #Plot2(x,y)
  239. #Plot3(x,y)
  240.  
  241.  
  242.  
  243.  
  244.  
  245.  
  246.  
  247. #Regresja przebiegu względem czasu
  248. #
  249. #a=np.array(dataset_out['Milage'].tolist()).reshape(-1,1)
  250. #b=np.array(dataset_out['Age'].tolist()).reshape(-1,1)
  251. #regress(a,b)
  252.  
  253.  
  254.  
  255.  
  256.  
  257.  
  258. #To sie przyda do wojewodztw
  259. #for i, li in enumerate(page.select('li')):  #To się przyda do wojedzowtw
  260. #    print(i, li.text)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top