Guest User

Untitled

a guest
Dec 1st, 2017
100
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.99 KB | None | 0 0
  1. //Declare the target Sink for storing test data
  2. val mongoDB = MongoSink("mongodb://${mongo.server.host}")
  3.  
  4. //Init the test data
  5. val testData = Seq(
  6. """{"_id":"1000","articleId":"a001","comment":{"content":"Nice article","author":"a001"}}""",
  7. """{"_id":"1001","articleId":"a002","comment":{"content":"Good One","author":"a002"}}""",
  8. """{"_id":"1002","articleId":"a003","comment":{"content":"That's way too long","author":"a003"}}"""
  9. )
  10.  
  11. val sparkCmd = "spark-submit --master yarn --deploy-mode cluster --name MongoJob --class SparkApp /path/to/app.jar"
  12.  
  13. //Remote server ssh config
  14. val sshConfig = SSHConfig(user = "admin", password = "admin", hostName = "xx.xx.xx.xx")
  15.  
  16. //Define the scenario
  17. val setup = scenario("Mongo Spark Test Suite")
  18. .exec(
  19. mongoDB.createCollection(db = "articles", collection = "comments", indexFields = Seq("comment.author"))
  20. )
  21. .exec(
  22. mongoDB.insertDocuments(db = "articles", collection = "comments", rows = docs)
  23. )
  24. .ssh(sshConfig, cmd = sparkCmd)
Add Comment
Please, Sign In to add comment