Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def flatten(lst):
- return [item for sublist in lst for item in sublist]
- def timestamp_fields(field, prefix=''):
- """ Find all fields with Timestamp metadata """
- if isinstance(field, StructType):
- return flatten(timestamp_fields(f, prefix) for f in field.fields)
- if isinstance(field, StructField):
- if field.metadata and field.metadata.get('encoding') == 'timestamp':
- return [prefix + '.' + field.name if prefix else field.name]
- return timestamp_fields(field.dataType, field.name)
- return []
- fields = timestamp_fields(df.schema)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement