Advertisement
JBOliveira

Untitled

Dec 22nd, 2021 (edited)
74
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 0.59 KB | None | 0 0
  1. def flatten(lst):
  2.     return [item for sublist in lst for item in sublist]
  3.  
  4.  
  5. def timestamp_fields(field, prefix=''):
  6.     """ Find all fields with Timestamp metadata """
  7.     if isinstance(field, StructType):
  8.         return flatten(timestamp_fields(f, prefix) for f in field.fields)
  9.  
  10.     if isinstance(field, StructField):
  11.         if field.metadata and field.metadata.get('encoding') == 'timestamp':
  12.             return [prefix + '.' + field.name if prefix else field.name]
  13.         return timestamp_fields(field.dataType, field.name)
  14.  
  15.     return []
  16.  
  17.  
  18. fields = timestamp_fields(df.schema)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement