daily pastebin goal
40%
SHARE
TWEET

Untitled

a guest Apr 28th, 2017 556 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. cc <- rxSparkConnect(reset = TRUE, nameNode="wasb://benchmark@airlinecranrscaler.blob.core.windows.net/", driverMem="4g", numExecutors=4, executorCores=5, executorMem="16g", executorOverheadMem="4g")
  2.  
  3. # prepare hive data 1M, 2M, 5M, 10M, 100M
  4. hdfs <- RxHdfsFileSystem(hostName="wasb://benchmark@airlinecranrscaler.blob.core.windows.net/")
  5.  
  6. airText <- RxTextData("wasb://benchmark@airlinecranrscaler.blob.core.windows.net/airline/csv/airOT1M", firstRowIsColNames=T, fileSystem=hdfs)
  7. airHive <- RxHiveData(table="airOT1M")
  8. rxDataStep(airText, airHive)
  9.  
  10. airText <- RxTextData("wasb://benchmark@airlinecranrscaler.blob.core.windows.net/airline/csv/airOT2M", firstRowIsColNames=T, fileSystem=hdfs)
  11. airHive <- RxHiveData(table="airOT2M")
  12. rxDataStep(airText, airHive)
  13.  
  14. airText <- RxTextData("wasb://benchmark@airlinecranrscaler.blob.core.windows.net/airline/csv/airOT5M", firstRowIsColNames=T, fileSystem=hdfs)
  15. airHive <- RxHiveData(table="airOT5M")
  16. rxDataStep(airText, airHive)
  17.  
  18. airText <- RxTextData("wasb://benchmark@airlinecranrscaler.blob.core.windows.net/airline/csv/airOT10M", firstRowIsColNames=T, fileSystem=hdfs)
  19. airHive <- RxHiveData(table="airOT10M")
  20. rxDataStep(airText, airHive)
  21.  
  22. airText <- RxTextData("wasb://benchmark@airlinecranrscaler.blob.core.windows.net/airline/csv/airOT100M", firstRowIsColNames=T, fileSystem=hdfs)
  23. airHive <- RxHiveData(table="airOT100M")
  24. rxDataStep(airText, airHive)
  25.  
  26. rxSparkDisconnect(cc)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top