Guest User

Untitled

a guest
Dec 12th, 2019
806
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. val appID = ""
  2. val password = ""
  3. val tenantID = ""
  4. val fileSystemName = "";
  5. var storageAccountName = "";
  6.  
  7. val configs =  Map("fs.azure.account.auth.type" -> "OAuth",
  8.        "fs.azure.account.oauth.provider.type" -> "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
  9.        "fs.azure.account.oauth2.client.id" -> appID,
  10.        "fs.azure.account.oauth2.client.secret" -> password,
  11.        "fs.azure.account.oauth2.client.endpoint" -> ("https://login.microsoftonline.com/" + tenantID + "/oauth2/token"),
  12.        "fs.azure.createRemoteFileSystemDuringInitialization"-> "true")
  13.  
  14. dbutils.fs.mount(
  15. source = "abfss://" + fileSystemName + "@" + storageAccountName + ".dfs.core.windows.net/",
  16. mountPoint = "/mnt/data",
  17. extraConfigs = configs)
  18.  
  19.  
  20. val df = spark.read.csv("/mnt/data/demo/movies.csv")
  21.  
  22. display(df)
  23.  
  24. val df = spark.read.option("header", "true").csv("/mnt/data/demo/movies.csv")
  25.  
  26.  
  27. val selected = df.select("movieId", "title")
  28.  
  29. selected.save.csv("/mnt/data/demo/movies-2.csv")
RAW Paste Data