Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- //Get all the calls for the last 24 hours for each MSISDN in the hour
- val sCallsPlaced = (grouped24HourCallsPlaS).join(distinctMSISDNs)
- val oCallsPlaced = (grouped24HourCallsPlaO).join(distinctMSISDNs)
- val sCallsReceived = grouped24HourCallsRecS.join(distinctMSISDNs)
- val oCallsReceived = grouped24HourCallsRecO.join(distinctMSISDNs)
- val callsToProcess = sCallsPlaced.union(oCallsPlaced).union(sCallsReceived).union(oCallsReceived)
- spark.driver.memory=16g
- spark.driver.cores=1
- spark.driver.maxResultSize=2g
- spark.executor.memory=24g
- spark.executor.cores=10
- spark.default.parallelism=256
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement