Guest User

Untitled

a guest
May 11th, 2024
123
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 5.29 KB | None | 0 0
  1. import psycopg
  2. from psycopg.rows import dict_row
  3. import datetime
  4. import csv
  5. import pandas
  6.  
  7. ########################################################################################################################
  8. # database connection parameters
  9.  
  10. dbname = ''
  11. user = ''
  12. host = 'mach3db.com'
  13. password = ''
  14. connection_string = f"dbname={dbname} user={user} host={host} password={password} sslmode=require"
  15.  
  16. ########################################################################################################################
  17. # table and query
  18.  
  19. table = ''
  20.  
  21. query = f"""
  22.            
  23.            SELECT * FROM {table}
  24.            
  25. """
  26. ########################################################################################################################
  27.  
  28.  
  29. def get_column_names():
  30.     column_query = f"""
  31.        SELECT *
  32.        FROM {table}
  33.        LIMIT 0
  34.    """
  35.  
  36.     with psycopg.connect(connection_string) as conn:
  37.         with conn.cursor() as cur:
  38.             cur.execute(column_query)
  39.             return [desc[0] for desc in cur.description]
  40.  
  41.  
  42. def time_to_fetch_and_output_csv(fetchall=False):
  43.     base_dir = 'C:\\Users\\shaughnessy software\\Downloads\\'
  44.  
  45.     date_string = datetime.datetime.now().strftime('%c').replace(':', '-')
  46.     filename_string = f'mach3db-benchmarket-output-{date_string}.csv'
  47.  
  48.     output_filename = f'{base_dir}{filename_string}'
  49.  
  50.     with psycopg.connect(connection_string) as conn:
  51.         with conn.cursor() as cur:
  52.             with open(output_filename, 'w', newline='') as f:
  53.                 writer = csv.writer(f)
  54.  
  55.                 writer.writerow(
  56.                     get_column_names()
  57.                 )
  58.  
  59.                 if fetchall:
  60.                     start_time = datetime.datetime.now()
  61.  
  62.                     cur.execute(query)
  63.  
  64.                     writer.writerows(
  65.                         cur.fetchall()
  66.                     )
  67.  
  68.                     end_time = datetime.datetime.now()
  69.  
  70.                     time_taken = (end_time - start_time).total_seconds()
  71.  
  72.                     return time_taken
  73.                 else:
  74.                     start_time = datetime.datetime.now()
  75.  
  76.                     cur.execute(query)
  77.  
  78.                     row = cur.fetchone()
  79.  
  80.                     while row:
  81.                         writer.writerow(row)
  82.                         row = cur.fetchone()
  83.  
  84.                     end_time = datetime.datetime.now()
  85.  
  86.                     time_taken = (end_time - start_time).total_seconds()
  87.  
  88.                     return time_taken
  89.  
  90.  
  91. def time_to_fetch_and_return(as_dict=False, as_df=False):
  92.     if as_dict:
  93.         row_factory = dict_row
  94.     else:
  95.         row_factory = None
  96.  
  97.     with psycopg.connect(connection_string, row_factory=row_factory) as conn:
  98.         with conn.cursor() as cur:
  99.             start_time = datetime.datetime.now()
  100.  
  101.             cur.execute(query)
  102.  
  103.             results = cur.fetchall()
  104.  
  105.             end_time = datetime.datetime.now()
  106.  
  107.             time_taken = (end_time - start_time).total_seconds()
  108.  
  109.             if as_df:
  110.                 if as_dict:
  111.                     results = pandas.DataFrame(results)
  112.                 else:
  113.                     results = pandas.DataFrame(results, columns=get_column_names())
  114.  
  115.             return results, time_taken
  116.  
  117.  
  118. def time_to_fetch():
  119.     with psycopg.connect(connection_string) as conn:
  120.         with conn.cursor() as cur:
  121.             start_time = datetime.datetime.now()
  122.  
  123.             cur.execute(query)
  124.  
  125.             end_time = datetime.datetime.now()
  126.  
  127.             time_taken = (end_time - start_time).total_seconds()
  128.  
  129.             return time_taken
  130.  
  131.  
  132. if __name__ == '__main__':
  133.     ####################################################################################################################
  134.  
  135.     # test parameters
  136.  
  137.     MODE_CHOICES = 'query_only', 'return_results', 'output_csv'
  138.  
  139.     SELECTED_MODE = 'query_only'
  140.  
  141.     # return_results sub-parameters
  142.     as_dict = False
  143.     as_df = True
  144.     rows_to_print = 5
  145.     set_trace = True
  146.  
  147.     # output_csv sub-parameters
  148.     fetch_all = False
  149.  
  150.     ####################################################################################################################
  151.  
  152.     if SELECTED_MODE == 'query_only':
  153.         print(
  154.             f'Query time taken in seconds: {time_to_fetch()}'
  155.         )
  156.     elif SELECTED_MODE == 'output_csv':
  157.         print(
  158.             f'Query time taken in seconds: {time_to_fetch_and_output_csv(fetchall=fetch_all)}'
  159.         )
  160.     elif SELECTED_MODE == 'return_results':
  161.         results, time_taken_in_seconds = time_to_fetch_and_return(
  162.             as_dict=as_dict,
  163.             as_df=as_df
  164.         )
  165.  
  166.         if as_df:
  167.             print(
  168.                 results.head(
  169.                     rows_to_print
  170.                 )
  171.             )
  172.         else:
  173.             if not as_dict:
  174.                 print(
  175.                     get_column_names()
  176.                 )
  177.  
  178.             for row_to_print in results[:rows_to_print]:
  179.                 print(
  180.                     row_to_print
  181.                 )
  182.  
  183.         print(
  184.             f'Query time taken in seconds: {time_taken_in_seconds}'
  185.         )
  186.  
  187.         if set_trace:
  188.             import pdb; pdb.set_trace()
  189.     else:
  190.         print(f'INVALID MODE SELECTED! CHOICES ARE: {MODE_CHOICES}')
  191.  
Advertisement
Add Comment
Please, Sign In to add comment