View difference between Paste ID: aybPTd3b and n31cuE1q
SHOW: | | - or go back to the newest paste.
1
import psycopg
2
from psycopg.rows import dict_row
3
import datetime
4
import csv
5
import pandas
6
7
########################################################################################################################
8
# database connection parameters
9
10
dbname = ''
11
user = ''
12
host = 'mach3db.com'
13
password = ''
14
connection_string = f"dbname={dbname} user={user} host={host} password={password} sslmode=require"
15
16
########################################################################################################################
17
# table and query
18
19
table = ''
20
21
query = f"""
22
            
23
            SELECT * FROM {table} 
24
            
25
"""
26
########################################################################################################################
27
28
29
def get_column_names():
30
    column_query = f"""
31
        SELECT *
32
        FROM {table}
33
        LIMIT 0
34
    """
35
36
    with psycopg.connect(connection_string) as conn:
37
        with conn.cursor() as cur:
38
            cur.execute(column_query)
39
            return [desc[0] for desc in cur.description]
40
41
42
def time_to_fetch_and_output_csv(fetchall=False):
43-
    base_dir = 'C:\\Users\\shaughnessy software\\Downloads\\'
43+
    base_dir = ''
44
45
    date_string = datetime.datetime.now().strftime('%c').replace(':', '-')
46
    filename_string = f'mach3db-benchmarket-output-{date_string}.csv'
47
48
    output_filename = f'{base_dir}{filename_string}'
49
50
    with psycopg.connect(connection_string) as conn:
51
        with conn.cursor() as cur:
52
            with open(output_filename, 'w', newline='') as f:
53
                writer = csv.writer(f)
54
55
                writer.writerow(
56
                    get_column_names()
57
                )
58
59
                if fetchall:
60
                    start_time = datetime.datetime.now()
61
62
                    cur.execute(query)
63
64
                    writer.writerows(
65
                        cur.fetchall()
66
                    )
67
68
                    end_time = datetime.datetime.now()
69
70
                    time_taken = (end_time - start_time).total_seconds()
71
72
                    return time_taken
73
                else:
74
                    start_time = datetime.datetime.now()
75
76
                    cur.execute(query)
77
78
                    row = cur.fetchone()
79
80
                    while row:
81
                        writer.writerow(row)
82
                        row = cur.fetchone()
83
84
                    end_time = datetime.datetime.now()
85
86
                    time_taken = (end_time - start_time).total_seconds()
87
88
                    return time_taken
89
90
91
def time_to_fetch_and_return(as_dict=False, as_df=False):
92
    if as_dict:
93
        row_factory = dict_row
94
    else:
95
        row_factory = None
96
97
    with psycopg.connect(connection_string, row_factory=row_factory) as conn:
98
        with conn.cursor() as cur:
99
            start_time = datetime.datetime.now()
100
101
            cur.execute(query)
102
103
            results = cur.fetchall()
104
105
            end_time = datetime.datetime.now()
106
107
            time_taken = (end_time - start_time).total_seconds()
108
109
            if as_df:
110
                if as_dict:
111
                    results = pandas.DataFrame(results)
112
                else:
113
                    results = pandas.DataFrame(results, columns=get_column_names())
114
115
            return results, time_taken
116
117
118
def time_to_fetch():
119
    with psycopg.connect(connection_string) as conn:
120
        with conn.cursor() as cur:
121
            start_time = datetime.datetime.now()
122
123
            cur.execute(query)
124
125
            end_time = datetime.datetime.now()
126
127
            time_taken = (end_time - start_time).total_seconds()
128
129
            return time_taken
130
131
132
if __name__ == '__main__':
133
    ####################################################################################################################
134
135
    # test parameters
136
137
    MODE_CHOICES = 'query_only', 'return_results', 'output_csv'
138
139
    SELECTED_MODE = 'query_only'
140
141
    # return_results sub-parameters
142
    as_dict = False
143
    as_df = True
144
    rows_to_print = 5
145
    set_trace = True
146
147
    # output_csv sub-parameters
148
    fetch_all = False
149
150
    ####################################################################################################################
151
152
    if SELECTED_MODE == 'query_only':
153
        print(
154
            f'Query time taken in seconds: {time_to_fetch()}'
155
        )
156
    elif SELECTED_MODE == 'output_csv':
157
        print(
158
            f'Query time taken in seconds: {time_to_fetch_and_output_csv(fetchall=fetch_all)}'
159
        )
160
    elif SELECTED_MODE == 'return_results':
161
        results, time_taken_in_seconds = time_to_fetch_and_return(
162
            as_dict=as_dict,
163
            as_df=as_df
164
        )
165
166
        if as_df:
167
            print(
168
                results.head(
169
                    rows_to_print
170
                )
171
            )
172
        else:
173
            if not as_dict:
174
                print(
175
                    get_column_names()
176
                )
177
178
            for row_to_print in results[:rows_to_print]:
179
                print(
180
                    row_to_print
181
                )
182
183
        print(
184
            f'Query time taken in seconds: {time_taken_in_seconds}'
185
        )
186
187
        if set_trace:
188
            import pdb; pdb.set_trace()
189
    else:
190
        print(f'INVALID MODE SELECTED! CHOICES ARE: {MODE_CHOICES}')
191