Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- {"json_data":"{"table":"TEST.FUBAR","op_type":"I","op_ts":"2019-03-14 15:33:50.031848","current_ts":"2019-03-14T15:33:57.479002","pos":"1111","after":{"COL1":949494949494949494,"COL2":99,"COL3":2,"COL4":" 99999","COL5":9999999,"COL6":90,"COL7":42478,"COL8":"I","COL9":null,"COL10":"2019-03-14 15:33:49","COL11":null,"COL12":null,"COL13":null,"COL14":"x222263 ","COL15":"2019-03-14 15:33:49","COL16":"x222263 ","COL17":"2019-03-14 15:33:49","COL18":"2020-09-10 00:00:00","COL19":"A","COL20":"A","COL21":0,"COL22":null,"COL23":"2019-03-14 15:33:47","COL24":2,"COL25":2,"COL26":"R","COL27":"2019-03-14 15:33:49","COL28":" ","COL29":"PBU67H ","COL30":" 20000","COL31":2,"COL32":null}}"}
- df = sqlContext.read.json("test_data.json")
- {"json_data":"{"table":"TEST.FUBAR","op_type":"I","op_ts":"2019-03-14 15:33:50.031848","current_ts":"2019-03-14T15:33:57.479002","pos":"1111","after":{"COL1":949494949494949494,"COL2":99,"COL3":2,"COL4":" 99999","COL5":9999999,"COL6":90,"COL7":42478,"COL8":"I","COL9":null,"COL10":"2019-03-14 15:33:49","COL11":null,"COL12":null,"COL13":null,"COL14":"x222263 ","COL15":"2019-03-14 15:33:49","COL16":"x222263 ","COL17":"2019-03-14 15:33:49","COL18":"2020-09-10 00:00:00","COL19":"A","COL20":"A","COL21":0,"COL22":null,"COL23":"2019-03-14 15:33:47","COL24":2,"COL25":2,"COL26":"R","COL27":"2019-03-14 15:33:49","COL28":" ","COL29":"PBU67H ","COL30":" 20000","COL31":2,"COL32":null}}"}
- {"json_data":{"table":"TEST.FUBAR","op_type":"I","op_ts":"2019-03-14 15:33:50.031848","current_ts":"2019-03-14T15:33:57.479002","pos":"1111","after":{"COL1":949494949494949494,"COL2":99,"COL3":2,"COL4":" 99999","COL5":9999999,"COL6":90,"COL7":42478,"COL8":"I","COL9":null,"COL10":"2019-03-14 15:33:49","COL11":null,"COL12":null,"COL13":null,"COL14":"x222263 ","COL15":"2019-03-14 15:33:49","COL16":"x222263 ","COL17":"2019-03-14 15:33:49","COL18":"2020-09-10 00:00:00","COL19":"A","COL20":"A","COL21":0,"COL22":null,"COL23":"2019-03-14 15:33:47","COL24":2,"COL25":2,"COL26":"R","COL27":"2019-03-14 15:33:49","COL28":" ","COL29":"PBU67H ","COL30":" 20000","COL31":2,"COL32":null}}}
- Welcome to
- ____ __
- / __/__ ___ _____/ /__
- _ / _ / _ `/ __/ '_/
- /__ / .__/_,_/_/ /_/_ version 1.6.0
- /_/
- Using Python version 2.7.13 (default, Dec 20 2016 23:09:15)
- SparkContext available as sc, HiveContext available as sqlContext.
- >>> filePath = "/user/no_quote_json.json"
- >>> df = sqlContext.read.json(filePath)
- >>> df.printSchema()
- root
- |-- json_data: struct (nullable = true)
- | |-- after: struct (nullable = true)
- | | |-- COL1: long (nullable = true)
- | | |-- COL10: string (nullable = true)
- | | |-- COL11: string (nullable = true)
- | | |-- COL12: string (nullable = true)
- | | |-- COL13: string (nullable = true)
- | | |-- COL14: string (nullable = true)
- | | |-- COL15: string (nullable = true)
- | | |-- COL16: string (nullable = true)
- | | |-- COL17: string (nullable = true)
- | | |-- COL18: string (nullable = true)
- | | |-- COL19: string (nullable = true)
- | | |-- COL2: long (nullable = true)
- | | |-- COL20: string (nullable = true)
- | | |-- COL21: long (nullable = true)
- | | |-- COL22: string (nullable = true)
- | | |-- COL23: string (nullable = true)
- | | |-- COL24: long (nullable = true)
- | | |-- COL25: long (nullable = true)
- | | |-- COL26: string (nullable = true)
- | | |-- COL27: string (nullable = true)
- | | |-- COL28: string (nullable = true)
- | | |-- COL29: string (nullable = true)
- | | |-- COL3: long (nullable = true)
- | | |-- COL30: string (nullable = true)
- | | |-- COL31: long (nullable = true)
- | | |-- COL32: string (nullable = true)
- | | |-- COL4: string (nullable = true)
- | | |-- COL5: long (nullable = true)
- | | |-- COL6: long (nullable = true)
- | | |-- COL7: long (nullable = true)
- | | |-- COL8: string (nullable = true)
- | | |-- COL9: string (nullable = true)
- | |-- current_ts: string (nullable = true)
- | |-- op_ts: string (nullable = true)
- | |-- op_type: string (nullable = true)
- | |-- pos: string (nullable = true)
- | |-- table: string (nullable = true)
- >>> df.select("json_data.after.col29").show()
- +---------+
- | col29|
- +---------+
- |PBU67H |
- +---------+
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement