Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- commit log file:
- v1
- {"nextBatchWatermarkMs":0}
- {
- "batchWatermarkMs": 0,
- "batchTimestampMs": 1542634173589,
- "conf": {
- "spark.sql.shuffle.partitions": "200",
- "spark.sql.streaming.stateStore.providerClass": "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider",
- "spark.sql.streaming.multipleWatermarkPolicy": "min",
- "spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion": "2"
- }
- }
- {
- "poc-test01": {
- "1": 3666,
- "0": 3664
- }
- }
- {
- "path": "dbfs:/test02/part-00000-98c5def2-bdf3-4e56-af9f-ff0796344b01-c000.snappy.parquet",
- "size": 59180,
- "isDir": false,
- "modificationTime": 1542214122000,
- "blockReplication": 1,
- "blockSize": 536870912,
- "action": "add"
- }
- {
- "path": "dbfs:/test02/part-00001-a38dbf8b-240a-48d0-8438-f81f0506a79c-c000.snappy.parquet",
- "size": 59462,
- "isDir": false,
- "modificationTime": 1542214122000,
- "blockReplication": 1,
- "blockSize": 536870912,
- "action": "add"
- }
Add Comment
Please, Sign In to add comment