Coverage for python/lsst/ap/association/assoc_db_sqlite.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# This file is part of ap_association. # # Developed for the LSST Data Management System. # This product includes software developed by the LSST Project # (https://www.lsst.org). # See the COPYRIGHT file at the top-level directory of this distribution # for details of code ownership. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>.
DIASources. This class is mainly for testing purposes in the context of ap_pipe/ap_verify. """
"AssociationDBSqliteTask", "SqliteDBConverter"]
make_minimal_dia_object_schema, \ make_minimal_dia_source_schema, \ getCcdVisitSchemaSql, \ add_dia_source_aliases_to_catalog, \ get_ccd_visit_info_from_exposure, \ make_overwrite_dict
"""Class for defining conversions to and from an sqlite database and afw SourceRecord objects.
Parameters ---------- schema : `lsst.afw.table.Schema` Schema defining the SourceRecord objects to be converted. table_name : `str` Name of the sqlite table this converter is to be used for. """
self._schema = schema self._table_name = table_name self._afw_to_db_types = { "Angle": "REAL", "D": "REAL", "L": "INTEGER", "String": "TEXT", }
def table_name(self): """Return name of the sqlite table this catalog is for """ return self._table_name
def schema(self): """Return the internal catalog schema. """ return self._schema
"""Convert the schema into a sqlite CREATE TABLE command.
Parameters ---------- table_name : `str` Name of the new table to create
Returns ------- sql_query : `str` A string of the query command to create the new table in sqlite. """ name_type_string = "" for sub_schema in self._schema: tmp_name = sub_schema.getField().getName() tmp_type = self._afw_to_db_types[ sub_schema.getField().getTypeString()] if tmp_name == 'id': tmp_type += " PRIMARY KEY" name_type_string += "%s %s," % (tmp_name, tmp_type) name_type_string = name_type_string[:-1]
return "CREATE TABLE IF NOT EXISTS %s (%s)" % (table_name, name_type_string)
"""Create a source record from the values stored in a database row.
Parameters ---------- db_row : `list` of ``values`` Retrieved values from the database to convert into a SourceRecord.
Returns ------- record : `lsst.afw.table.SourceRecord` Converted source record. """
output_source_record = afwTable.SourceTable.makeRecord( afwTable.SourceTable.make(self._schema))
for sub_schema, value in zip(self._schema, db_row): if value is None: if sub_schema.getField().getTypeString() == 'L': value = -1 elif sub_schema.getField().getTypeString() == 'String': value = 'NULL' else: continue if sub_schema.getField().getTypeString() == 'Angle': output_source_record.set( sub_schema.getKey(), value * afwGeom.degrees) else: output_source_record.set(sub_schema.getKey(), value) return output_source_record
"""Convert a source record object into a list of its internal values.
Parameters ---------- source_record : `lsst.afw.table.SourceRecord` SourceRecord to convert. overwrite_dict : `dict` (optional) Mapping specifying the names of columns to overwrite with specified values.
Returns ------- source_list : `list` of ``values`` Extracted values from ``source_record`` in `list` form. """ values = [] for sub_schema in self._schema: field_name = sub_schema.getField().getName() if field_name in overwrite_dict: values.append(overwrite_dict[field_name]) elif field_name in source_record.getSchema().getNames(): if sub_schema.getField().getTypeString() == 'Angle': values.append( source_record.get(sub_schema.getKey()).asDegrees()) else: values.append(source_record.get(sub_schema.getKey())) else: values.append(None) return values
"""Configuration parameters for the AssociationDBSqliteTask """ dtype=str, doc='Location on disk and name of the sqlite3 database for storing ' 'and loading DIASources and DIAObjects.', default=':memory:' ) dtype=str, doc='List of filter names to store and expect from in this DB.', default=[], ) doc='Select the spatial indexer to use within the database.', default='HTM' )
"""Enable storage of and reading of DIAObjects and DIASources from a sqlite database.
Create a simple sqlite database and implement wrappers to store and retrieve DIAObjects and DIASources from within that database. This task functions as a testing ground for the L1 database and should mimic this database's eventual functionality. This specific database implementation is useful for the verification packages which may not be run with access to L1 database. """
pipeBase.Task.__init__(self, **kwargs) self.indexer = IndexerRegistry[self.config.indexer.name]( self.config.indexer.active) self._db_connection = sqlite3.connect(self.config.db_name, timeout=60)
self._dia_object_converter = SqliteDBConverter( make_minimal_dia_object_schema(self.config.filter_names), "dia_objects") self._dia_source_converter = SqliteDBConverter( make_minimal_dia_source_schema(), "dia_sources") self._ccd_visit_schema = getCcdVisitSchemaSql()
"""Save changes to the sqlite database. """ self._db_connection.commit()
"""Close the connection to the sqlite database. """ self._db_connection.close()
"""If no sqlite database with the correct tables exists we can create one using this method.
Returns ------- succeeded : `bool` Successfully created a new database with specified tables. """ # Create tables to store the individual DIAObjects and DIASources with self._db_connection as conn: conn.execute("BEGIN") conn.execute( self._dia_object_converter.make_table_from_afw_schema( "dia_objects")) conn.execute( "CREATE INDEX IF NOT EXISTS indexer_id_index ON " "dia_objects(pixelId)") conn.execute( self._dia_source_converter.make_table_from_afw_schema( "dia_sources")) conn.execute( "CREATE INDEX IF NOT EXISTS diaObjectId_index ON " "dia_sources(diaObjectId)")
table_schema = ",".join( "%s %s" % (key, self._ccd_visit_schema[key]) for key in self._ccd_visit_schema.keys()) conn.execute( "CREATE TABLE IF NOT EXISTS CcdVisit (%s)" % table_schema)
return True
def load_dia_objects(self, exposure): """Load all DIAObjects within the exposure.
Parameters ---------- exposure : `lsst.afw.image.Exposure` An exposure with a solved WCS representing the area on the sky to load DIAObjects.
Returns ------- dia_objects : `lsst.afw.table.SourceCatalog` Catalog of DIAObjects that are contained with the the bounding box defined by expMd. """ bbox = afwGeom.Box2D(exposure.getBBox()) wcs = exposure.getWcs()
ctr_coord = wcs.pixelToSky(bbox.getCenter()) max_radius = max( ctr_coord.separation(wcs.pixelToSky(pp)) for pp in bbox.getCorners())
indexer_indices, on_boundry = self.indexer.getShardIds( ctr_coord, max_radius)
dia_objects = self._get_dia_object_catalog(indexer_indices, bbox, wcs)
return dia_objects
def load_dia_sources(self, dia_obj_ids): """Retrieve all DIASources associated with this collection of DIAObject ids.
Parameters ---------- dia_obj_ids : array-like of `int`s Id of the DIAObject that is associated with the DIASources of interest.
Returns ------- dia_sources : `lsst.afw.table.SourceCatalog` SourceCatalog of DIASources """
dia_source_schema = make_minimal_dia_source_schema() output_dia_sources = afwTable.SourceCatalog(dia_source_schema)
rows = self._query_dia_sources(dia_obj_ids) for row in rows: output_dia_sources.append( self._dia_source_converter.source_record_from_db_row(row))
return output_dia_sources.copy(deep=True)
dia_objects, compute_spatial_index=False, exposure=None): """Store all DIAObjects in this SourceCatalog.
Parameters ---------- dia_objects : `lsst.afw.table.SourceCatalog` Catalog of DIAObjects to store. compute_spatial_index : `bool` If True, compute the spatial search indices using the indexer specified at class instantiation. exposure: `lsst.afw.image.Exposure` (optional) CcdExposure associated with these DIAObjects being inserted. Inserts the CcdVisitInfo for this exposure in the CcdVisitTable. """ if compute_spatial_index: for dia_object in dia_objects: sphPoint = dia_object.getCoord() dia_object.set('pixelId', self.compute_indexer_id(sphPoint)) self._store_catalog(dia_objects, self._dia_object_converter, None)
if exposure is not None: self.store_ccd_visit_info(exposure)
"""Compute the pixel index of the given point.
Parameters ---------- sphere_point : `lsst.afw.geom.SpherePoint` Point to compute pixel index for.
Returns ------- index : `int` Index of the pixel the point is contained in. """ return self.indexer.indexPoints( [sphere_point.getRa().asDegrees()], [sphere_point.getDec().asDegrees()])[0]
dia_sources, associated_ids=None, exposure=None): """Store all DIASources in this SourceCatalog.
Parameters ---------- dia_sources : `lsst.afw.table.SourceCatalog` Catalog of DIASources to store. associated_ids : array-like of `int`s (optional) DIAObject ids that have been associated with these DIASources exposure : `lsst.afw.image.Exposure` Exposure object the DIASources were detected in. """ self._store_catalog(dia_sources, self._dia_source_converter, associated_ids, exposure)
"""Store information describing the exposure for this ccd, visit.
Paramters --------- exposure : `lsst.afw.image.Exposure` Exposure to store information from. """
values = get_ccd_visit_info_from_exposure(exposure) with self._db_connection as conn: conn.execute( "INSERT OR REPLACE INTO CcdVisit VALUES (%s)" % ",".join("?" for idx in range(len(self._ccd_visit_schema))), [values[key] for key in self._ccd_visit_schema.keys()])
"""Retrieve the DIAObjects from the database whose indexer indices are with the specified list of indices.
Retrieves a list of DIAObjects that are covered by the pixels with indices, indexer_indices. Use this to retrieve complete DIAObjects.
Parameters ---------- indexer_indices : array-like of `int`s Pixelized indexer indices from which to load. bbox : `lsst.geom.Box2D` Bounding box of exposure. wcs : `lsst.geom.SkyWcs` WCS of exposure
Returns ------- dia_objects : `lsst.afw.table.SourceCatalog` Catalog of DIAObjects with the specified indexer index and contained within the bounding box. """ dia_object_rows = self._query_dia_objects(indexer_indices)
output_dia_objects = afwTable.SourceCatalog( self._dia_object_converter.schema)
for row in dia_object_rows: dia_object_record = \ self._dia_object_converter.source_record_from_db_row(row) if self._check_dia_object_position(dia_object_record, bbox, wcs): output_dia_objects.append(dia_object_record)
return output_dia_objects.copy(deep=True)
"""Query the database for the stored DIAObjects given a set of indices in the indexer.
Parameters ---------- indexer_indices : array-like of `int`s Spatial indices in the indexer specifying the area on the sky to load DIAObjects for.
Returns ------- dia_objects : `list` of `tuples` containing ``dia_object`` ``values`` Query result containing the catalog values of the DIAObject. """ with self._db_connection as conn: conn.execute("BEGIN") conn.execute( "CREATE TEMPORARY TABLE tmp_indexer_indices " "(pixelId INTEGER PRIMARY KEY)")
conn.executemany( "INSERT OR REPLACE INTO tmp_indexer_indices VALUES (?)", [(int(indexer_index),) for indexer_index in indexer_indices])
cursor = conn.execute( "SELECT o.* FROM dia_objects AS o " "INNER JOIN tmp_indexer_indices AS i " "ON o.pixelId = i.pixelId")
output_rows = cursor.fetchall()
conn.execute("DROP TABLE tmp_indexer_indices")
return output_rows
"""Check the RA, DEC position of the current dia_object_record against the bounding box of the exposure.
Parameters ---------- dia_object_record : `lsst.afw.table.SourceRecord` A SourceRecord object containing the DIAObject we would like to test against our bounding box. bbox : `lsst.geom.Box2D` Bounding box of exposure. wcs : `lsst.geom.SkyWcs` WCS of exposure.
Return ------ is_contained : `bool` Object position is contained within the bounding box of expMd. """ point = wcs.skyToPixel(dia_object_record.getCoord()) return bbox.contains(point)
"""Query the database for the stored DIASources given a set of DIAObject ids.
Parameters ---------- dia_object_ids : array-like of `int`s Spatial indices in the indexer specifying the area on the sky to load DIAObjects for.
Return ------ dia_objects : `list` of `tuples` Query result containing the values representing DIASources """ with self._db_connection as conn: conn.execute("BEGIN") conn.execute( "CREATE TEMPORARY TABLE tmp_object_ids " "(diaObjectId INTEGER PRIMARY KEY)")
conn.executemany( "INSERT OR REPLACE INTO tmp_object_ids VALUES (?)", [(int(dia_object_id),) for dia_object_id in dia_object_ids])
cursor = conn.execute( "SELECT s.* FROM dia_sources AS s " "INNER JOIN tmp_object_ids AS o " "ON s.diaObjectId = o.diaObjectId")
output_rows = cursor.fetchall()
conn.execute("DROP TABLE tmp_object_ids")
return output_rows
""" Store a SourceCatalog into the database.
Parameters ---------- source_catalog : `lsst.afw.table.SourceCatalog` SourceCatalog to store in the database table specified by converter. converter : `lsst.ap.association.SqliteDBConverter` A converter object specifying the correct database table to write into. obj_id : array-like of `int`s (optional) Ids of the DIAObjects these objects are associated with. Use only when storing DIASources. exposure : `lsst.afw.image.Exposure` (optional) Exposure that the sources in source_catalog were detected in. If set the fluxes are calibrated and stored using the exposure Calib object. The filter the exposure was taken in as well as the ccdVisitId are also stored. """ values = []
if converter.table_name == 'dia_sources': # Create aliases to appropriate flux fields if they exist. add_dia_source_aliases_to_catalog(source_catalog)
if exposure is None: exp_dict = None else: exp_dict = get_ccd_visit_info_from_exposure(exposure)
for src_idx, source_record in enumerate(source_catalog): if obj_ids is None: obj_id = None else: obj_id = obj_ids[src_idx]
overwrite_dict = make_overwrite_dict(source_record, obj_id, exp_dict)
values.append(converter.source_record_to_value_list( source_record, overwrite_dict))
insert_string = ("?," * len(values[0]))[:-1]
with self._db_connection as conn: conn.executemany( "INSERT OR REPLACE INTO %s VALUES (%s)" % (converter.table_name, insert_string), values)
def dia_object_afw_schema(self): """Retrieve the Schema of the DIAObjects in this database.
Returns ------- schema : `lsst.afw.table.Schema` Schema of the DIAObjects in this database. """ return self._dia_object_converter.schema
def dia_source_afw_schema(self): """Retrieve the Schema of the DIASources in this database.
Returns ------- schema : `lsst.afw.table.Schema` Schema of the DIASources in this database. """ return self._dia_source_converter.schema |