Project

General

Profile

exposure_reader.py

Elizabeth Buckley-Geer, 12/13/2017 10:42 AM

 
1
#!/usr/bin/env python
2
# A simple module to retrieve and record image quality data
3
# 
4
#                query_exposures                        - queries the SISPI database for IQ data,
5
#                                                                  returns selected data as a python dictionary
6
#                write_exposures                        - writes out the result of query_exposures 
7
#                                                                 in an IQ catalog format
8
#                sql_query                                - returns the raw result of an sql query to 
9
#                                                                 the database. Can optionally write the result
10
#                                                                 to a CSV file.
11

    
12
import sys
13
import os
14
import sqlite3 as lite
15
import psycopg2
16
from optparse import OptionParser
17

    
18
exposure_properties = ['id', 'date', 'propid', 'guider', 'filter', 'airmass', 'aos', 'az', 'dec_offset', 'dimmsee', 'domeflor', 'domehigh', 'domelow', 'exptime', 'focus[1]', 'ha', 'hexapod_time', 'mairtemp', 'moonangl', 'msurtemp', 'program',  'ra_offset', 'readout_time', 'slewangl', 'slew_time', 'teldec', 'telra', 'time_between_exposures', 'winddir', 'windspd', 'zd']
19

    
20
exposure_properties_renamed = ['expid', 'date-obs', 'propid', 'guider', 'filter', 'airmass', 'aos', 'az', 'dec_offset', 'dimmsee', 'domeflor', 'domehigh', 'domelow', 'exptime', 'focus', 'ha', 'hexapod_time', 'mairtemp', 'moonangl', 'msurtemp', 'program', 'ra_offset', 'readout_time', 'slewangl', 'slew_time', 'teldec', 'telra', 'time_between_exposures', 'winddir', 'windspd', 'zd']
21

    
22
image_health_properties = ['angle', 'covxx', 'covxy', 'covyy', 'e1', 'e2', 'ellipticity', 'fwhm', 'median_data', 'nstars', 'r50', 'w1', 'w2', 'whisker', 'whisker_rms']
23

    
24
guider_properties = ['autocorx', 'autocory', 'maxx', 'maxy', 'meanx', 'meanx2', 'meanxy', 'meany', 'meany2', 'jumpx', 'jumpy']
25

    
26
guider_properties_renamed = ['autocorx', 'autocory', 'gmaxx', 'gmaxy', 'gx', 'gxx', 'gxy', 'gy', 'gyy', 'jumpx', 'jumpy']
27

    
28
tcs_properties = ['dec_mean', 'dec_mean_f0', 'dec_mean_f1', 'dec_mean_f2', 'dec_mean_f20', 'dec_mean_p1', 'dec_mean_p2', 'dec_mean_p3', 'dec_mean_p4', 'dec_mean_s20', 'dec_mean_t20', 'dec_std', 'dec_std_f0', 'dec_std_f1', 'dec_std_f2', 'dec_std_f20', 'dec_std_p1', 'dec_std_p2', 'dec_std_p3', 'dec_std_p4', 'dec_std_s20', 'dec_std_t20', 'ellip', 'ellip2', 'ge1', 'ge2', 'gellip', 'gellip2', 'ha_mean', 'ha_mean_f0', 'ha_mean_f1', 'ha_mean_f2', 'ha_mean_f20', 'ha_mean_p1', 'ha_mean_p2', 'ha_mean_p3', 'ha_mean_p4', 'ha_mean_s20', 'ha_mean_t20', 'ha_std', 'ha_std_f0', 'ha_std_f1', 'ha_std_f2', 'ha_std_f20', 'ha_std_p1', 'ha_std_p2', 'ha_std_p3', 'ha_std_p4', 'ha_std_s20', 'ha_std_t20', 'mjd', 'e1', 'e2', 'exptime', 'gxx', 'gxy', 'gyy', 'xx', 'xy', 'yy']
29

    
30
tcs_properties_renamed = ['tcs_dec_mean', 'tcs_dec_mean_f0', 'tcs_dec_mean_f1', 'tcs_dec_mean_f2', 'tcs_dec_mean_f20', 'tcs_dec_mean_p1', 'tcs_dec_mean_p2', 'tcs_dec_mean_p3', 'tcs_dec_mean_p4', 'tcs_dec_mean_s20', 'tcs_dec_mean_t20', 'tcs_dec_std', 'tcs_dec_std_f0', 'tcs_dec_std_f1', 'tcs_dec_std_f2', 'tcs_dec_std_f20', 'tcs_dec_std_p1', 'tcs_dec_std_p2', 'tcs_dec_std_p3', 'tcs_dec_std_p4', 'tcs_dec_std_s20', 'tcs_dec_std_t20', 'tcs_ellip', 'tcs_ellip2', 'tcs_ge1', 'tcs_ge2', 'tcs_gellip', 'tcs_gellip2', 'tcs_ha_mean', 'tcs_ha_mean_f0', 'tcs_ha_mean_f1', 'tcs_ha_mean_f2', 'tcs_ha_mean_f20', 'tcs_ha_mean_p1', 'tcs_ha_mean_p2', 'tcs_ha_mean_p3', 'tcs_ha_mean_p4', 'tcs_ha_mean_s20', 'tcs_ha_mean_t20', 'tcs_ha_std', 'tcs_ha_std_f0', 'tcs_ha_std_f1', 'tcs_ha_std_f2', 'tcs_ha_std_f20', 'tcs_ha_std_p1', 'tcs_ha_std_p2', 'tcs_ha_std_p3', 'tcs_ha_std_p4', 'tcs_ha_std_s20', 'tcs_ha_std_t20', 'tcs_mjd', 'tcs_e1', 'tcs_e2', 'tcs_exptime', 'tcs_gxx', 'tcs_gxy', 'tcs_gyy', 'tcs_xx', 'tcs_xy', 'tcs_yy']
31

    
32

    
33
donut_properties = ['dodzerr', 'dodx', 'dody', 'dodz', 'doxt', 'doyt', 'ndonuts_used', 'z5delta', 'z5thetax', 'z5thetay', 'z6delta', 'z6thetax', 'z6thetay']
34

    
35
donut_properties_renamed = ['dodzerr', 'donut_dodx', 'donut_dody', 'donut_dodz', 'donut_doxt', 'donut_doyt', 'ndonuts_used', 'z5delta', 'z5thetax', 'z5thetay', 'z6delta', 'z6thetax', 'z6thetay']
36

    
37
hexapod_properties = ['hexapod_tilt', 'hexapod_tip', 'hexapod_x', 'hexapod_y', 'hexapod_z', 'hexapod_zt', 'trim_dx', 'trim_dy', 'trim_dz', 'trim_xt', 'trim_yt', 'tweak_dx', 'tweak_dy', 'tweak_dz', 'tweak_xt', 'tweak_yt']
38

    
39
hexapod_properties_renamed = ['hexapod_tilt', 'hexapod_tip', 'hexapod_x', 'hexapod_y', 'hexapod_z', 'hexapod_zt', 'trim_dx', 'trim_dy', 'trim_dz', 'trim_xt', 'trim_yt', 'tweak_dx', 'tweak_dy', 'tweak_dz', 'tweak_xt', 'tweak_yt']
40

    
41
aos_summary = ['bcdx', 'bcdy', 'bcxt', 'bcyt', 'bczt', 'dodx', 'dody', 'dodz', 'doxt', 'doyt', 'fadx', 'fady', 'fadz', 'faxt', 'fayt']
42

    
43
aos_summary_renamed = ['bcdx', 'bcdy', 'bcxt', 'bcyt', 'bczt', 'dodx', 'dody', 'dodz', 'doxt', 'doyt', 'fadx', 'fady', 'fadz', 'faxt', 'fayt']
44

    
45
donut_summary = ['n_donuts']
46

    
47
donut_summary_renamed = ['n_donuts']
48

    
49
telescope_data = ['dome_az', 'tel_az', 'tel_el']
50

    
51
telescope_data_renamed = ['dome_az', 'tel_az', 'tel_el']
52

    
53
all_properties_renamed = exposure_properties_renamed + image_health_properties + guider_properties_renamed + tcs_properties_renamed + donut_properties_renamed + hexapod_properties_renamed + aos_summary_renamed + donut_summary_renamed + telescope_data_renamed
54

    
55
                
56
        
57
#Query to SISPI, returns data from specified exposures in dictionary
58
def query_exposures(exposure_id_min=None, exposure_id_max=None, exposure_date_min = None, exposure_date_max =None, exposure_flavor = 'object', exposure_delivered = True, exposure_guider_low_bound = 1, exposure_guider_high_bound = 2, exposure_exptime_min = 30, conditions = '', none_type = None):
59
        #information to connect to database        - Please contact buckley@fnal.gov if you want to use this script and I will give you the username and password
60
        db_user = 'username'
61
        db_password = 'password'
62
        db_host = 'des20.fnal.gov'
63
        db_name = 'decam_prd'
64
        db_port = '5443'
65
        
66
        #connection to database
67
        con = psycopg2.connect(user=db_user, password=db_password, host=db_host, dbname=db_name, port=db_port)
68

    
69
        #query to database
70
        cur = con.cursor()
71

    
72
        #fix dates
73
        if exposure_date_min is not None:
74
                exposure_date_min = str(exposure_date_min).replace(' ','T')
75
        if exposure_date_max is not None:
76
                exposure_date_max = str(exposure_date_max).replace(' ','T')
77

    
78
        condition_statement = 'WHERE '
79
        sql = 'SELECT '
80
        for prop in exposure_properties:
81
                sql = sql + ' exposure.' + prop + ','
82
        for prop in image_health_properties:
83
                sql = sql + ' image_health_fp.' + prop + '[1],'
84
        for prop in guider_properties:
85
                sql = sql + ' guider_summary.' + prop + ','
86
        for prop in tcs_properties:
87
                sql = sql + ' tcs_telemetry.' + prop + ','
88
        for prop in donut_properties:
89
                sql = sql + ' donut_ana.' + prop + ','
90
        for prop in hexapod_properties:
91
                sql = sql + ' hexapod_lut.' + prop + ','
92
        for prop in aos_summary:
93
                sql = sql + ' aos_summary.' + prop + ','
94
        for prop in donut_summary:
95
                sql = sql + ' donut_summary.' + prop + ','
96
        for prop in telescope_data:
97
                sql = sql + ' telescope_data.' + prop + ','
98
        sql = sql[:-1]
99
        sql = sql + ' '
100
        sql = sql + ' FROM exposure.exposure FULL JOIN telemetry.image_health_fp ON image_health_fp.exposure_id = exposure.id FULL JOIN telemetry.guider_summary ON guider_summary.expid = exposure.id FULL JOIN telemetry.donut_ana ON donut_ana.expid = exposure.id FULL JOIN telemetry.tcs_telemetry ON tcs_telemetry.expid = exposure.id FULL JOIN telemetry.hexapod_lut ON hexapod_lut.expid = exposure.id FULL JOIN telemetry.aos_summary ON aos_summary.expid = exposure.id FULL JOIN telemetry.donut_summary ON donut_summary.expid = exposure.id FULL JOIN telemetry.telescope_data ON telemetry.telescope_data.expid = exposure.id '  
101

    
102
        more = ''
103

    
104
        if exposure_id_min is not None:
105
                condition_statement = condition_statement + more + 'exposure.id >= ' + str(exposure_id_min)
106
                more = ' AND '
107

    
108
        if exposure_id_max is not None:
109
                condition_statement = condition_statement + more + ' exposure.id <= ' + str(exposure_id_max)
110
                more = ' AND '
111

    
112
        if exposure_date_min is not None:
113
                condition_statement = condition_statement + more + "exposure.date >= '" + str(exposure_date_min) + "' "
114
                more = ' AND '
115

    
116
        if exposure_date_max is not None:
117
                condition_statement = condition_statement + more + "exposure.date <= '" + str(exposure_date_max) + "' "
118
                more = ' AND '
119

    
120
        if more == '':
121
                condition_statement = condition_statement  + " exposure.flavor = '" + str(exposure_flavor) + "' AND exposure.delivered = " + str(exposure_delivered) + ' AND exposure.guider >= ' + str(exposure_guider_low_bound) + ' AND exposure.guider <= ' + str(exposure_guider_high_bound) + ' AND exposure.exptime >= ' + str(exposure_exptime_min)
122
        else:
123
                condition_statement = condition_statement  + " AND exposure.flavor = '" + str(exposure_flavor) + "' AND exposure.delivered = " + str(exposure_delivered) + ' AND exposure.guider >= ' + str(exposure_guider_low_bound) + ' AND exposure.guider <= ' + str(exposure_guider_high_bound) + ' AND exposure.exptime >= ' + str(exposure_exptime_min)
124

    
125
        if conditions != '':
126
                condition_statement = condition_statement + ' AND ' + conditions
127

    
128
        sql = sql + condition_statement +' ORDER BY exposure.id ASC'
129
        cur.execute(sql)
130

    
131
        #fill a list of the rows of returned data
132
        data = [r for r in cur.fetchall()]
133
        #fill dictionary
134
        dictionary = {x:[] for x in all_properties_renamed}
135

    
136

    
137
        #records data into dictionary        
138
        for n in range(len(data)):
139
                if n == 0:
140
                        for i in range(len(all_properties_renamed)):
141
                                v = data[n][i]
142
                                if v is None:
143
                                        v = none_type
144
                                dictionary[all_properties_renamed[i]].append(v)
145
                        continue
146
                if data[n][0] != data[n-1][0]:
147
                        for i in range(len(all_properties_renamed)):
148
                                v = data[n][i]
149
                                if v is None:
150
                                        v = none_type
151
                                dictionary[all_properties_renamed[i]].append(v)
152
        
153
        #close connection
154
        cur.close()
155
        con.close()
156
        
157
        return dictionary
158

    
159

    
160
def write_exposures(file_name = None, id_min = None, id_max = None, date_min = None, date_max = None, flavor = 'object', delivered = True, guider_low_bound = 1, guider_high_bound = 2, exptime_min = 30, user_conditions = '', none_type_written = None):
161
        """
162
        Records data from selected exposures in specified file
163

164
        Will only return "good" exposures. (exptime>=30, dilivered=true, etc.) unless
165
        the user specifies other criteria.
166

167
        id_min, id_max, date_min, date_max limit the results (should use at least
168
        1)
169

170
        user_conditions limits results of SQL query, string should take form of
171
        limitations after 'WHERE' in query. Ex: " winddir >= 235 "
172
        
173
        Dates must be in the following format: YYYY-MM-DDTHH:MM:SS with the T
174
        being the character 'T', or YY-MM-DD HH:MM:SS
175

176
        none_type_written is what is written to the file to denote a None value. Ex: -99999
177
        """
178

    
179
        #collect data to record
180
        data_dictionary = query_exposures(exposure_id_min = id_min, exposure_id_max = id_max, exposure_date_min = date_min, exposure_date_max = date_max, exposure_flavor = flavor, exposure_delivered = delivered, exposure_guider_low_bound = guider_low_bound, exposure_guider_high_bound = guider_high_bound, exposure_exptime_min = exptime_min, conditions = user_conditions, none_type = none_type_written)
181
        
182
        #open file to record data in
183
        out_file = open(file_name,'w')
184
        
185
        #assemble header and record it
186
        header = ", ".join(all_properties_renamed)
187
        header.replace("'",'')
188
        header = '#' + header
189
        out_file.write(header + '\n')
190
        
191
        #record data in file
192
        for i in range(len(data_dictionary['expid'])):
193
                data = []
194
                for index in (all_properties_renamed):
195
                        if index == 'date-obs':
196
                                data.append(str(data_dictionary[index][i]).replace(' ','T').strip("+00:00"))
197
                        else:
198
                                data.append(str(data_dictionary[index][i]))
199
                out_string = ', '.join(data)
200
                out_string = out_string.replace("'",'')
201
                out_file.write(out_string + '\n')
202
        #close file connection
203
        out_file.close()
204

    
205
# Returns the raw result of an sql query to the database.
206
# Optionally writes the result to a csv file, using a header
207
# given by the user if an IQ catalog is wanted.
208
def sql_query(sql = '', file_name = None, header = None):
209
        #information to connect to database        
210
        db_user = 'decam_reader'
211
        db_password = 'reader'
212
        db_host = 'des20.fnal.gov'
213
        db_name = 'decam_prd'
214
        db_port = '5443'
215
        
216
        #connection to databaseselect id, date from exposure.exposure where id = 176839
217
        con = psycopg2.connect(user=db_user, password=db_password, host=db_host, dbname=db_name, port=db_port)
218

    
219
        #query to database
220
        cur = con.cursor()
221
        
222
        cur.execute(sql)
223
        data = cur.fetchall()
224

    
225
        if file_name is None:
226
                return data
227

    
228
        out_file = open(file_name, 'w')
229
        
230
        if type(header) == list:
231
                header_string = '#'
232
                for item in header:
233
                        header_string = header_string + str(item) + ', '
234
                header_string = str(header_string[:-2]) + '\n'
235
                out_file.write(header_string)
236
        
237
        for row in data:
238
                row_string = ''
239
                for item in row:
240
                        row_string = row_string + str(item).strip().strip("+00:00") + ', '
241
                row_string = row_string[:-2] + '\n'
242
                out_file.write(row_string)
243

    
244
        return data
245

    
246
#Edit this section to change the script results
247
if __name__ == "__main__":
248
        parser = OptionParser(usage = "usage: %prog [options]")
249
        parser.add_option('-m', '--min', dest = 'id_minimum', type = int, default = None, help = 'Minimum exposure id')
250
        parser.add_option('-M', '--max', dest = 'id_maximum', type = int, default = None, help = 'Maximum exposure id')
251
        parser.add_option('-d', '--date-min', dest = 'date_minimum', type = str, default = None, help = 'Minimum date (to inculde time use the format YYY-MM-DDThh:mm:ss)')
252
        parser.add_option('-D', '--date-max', dest = 'date_maximum', type = str, default = None, help = 'Maximum date (to inculde time use the format YYY-MM-DDThh:mm:ss)')
253
        
254
        # parse args
255
        (options, args) = parser.parse_args()
256

    
257
        if len(args) != 1:
258
                parser.error('Need to specify file location')
259
                exit()
260

    
261
        write_exposures(file_name = args[0], id_min = options.id_minimum, id_max = options.id_maximum, date_min = options.date_minimum, date_max = options.date_maximum, flavor = 'object', delivered = True, guider_low_bound = 1, guider_high_bound = 2, exptime_min = 30, user_conditions = '')
262

    
263
        
264

    
265
        
266

    
267