daily pastebin goal
59%
SHARE
TWEET

Untitled

a guest Jan 18th, 2019 49 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. from numbers import Integral
  2.  
  3. import pandas as pd
  4.  
  5.  
  6. def get_colnames(cursor):
  7.     """ Get column names for cursor's result set """
  8.     return [coldesc[0] for coldesc in cursor.description]
  9.  
  10.  
  11. # TODO: have the batched dict and pd methods emit batches instead of flattening it out (otherwise
  12. # what's the point of the batching....)
  13.  
  14. def read_cursor(cursor, batch_size='all'):
  15.     """ Consistent interface to a DBAPI-compliant Cursor object """
  16.     if batch_size == 'all':
  17.         yield from cursor.fetchall()
  18.     elif batch_size == 1:
  19.         while True:
  20.             row = cursor.fetchone()
  21.             if row is not None:
  22.                 yield row
  23.             else:
  24.                 break
  25.     elif isinstance(batch_size, Integral) and batch_size > 0:
  26.         while True:
  27.             rows = cursor.fetchmany(batch_size)
  28.             if len(rows):
  29.                 yield from rows
  30.             else:
  31.                 break
  32.     else:
  33.         raise ValueError('Invalid batch size')
  34.  
  35.  
  36. def read_cursor_dicts(cursor, batch_size='all'):
  37.     """ Read cursor, with each row as a dictionary """
  38.     colnames = get_colnames(cursor)
  39.     for row in read_cursor(cursor, batch_size):
  40.         yield dict(zip(colnames, row))
  41.  
  42.  
  43. def read_cursor_pandas(cursor, batch_size='all'):
  44.     colnames = get_colnames(cursor)
  45.     return pd.DataFrame(list(read_cursor(cursor, batch_size)), columns=colnames)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top