Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def SaveData(row):
- ...
- # read csv string
- df = spark.read
- .option("header", True)
- .option("delimiter","|")
- .option("quote", """)
- .option("nullValue", "\N")
- .schema(schemaMapping)
- .csv(csvData)
- df.write.format("delta").mode("append").save(tableLocation)
- #df.write.saveAsTable(tableName)
- #df.saveAsTable(tableName, format='parquet', mode='append')
- query = dfDEHubStream.writeStream.foreach(SaveData).start()
- CREATE TABLE tableName
- USING DELTA
- LOCATION tableLocation
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement