Coverage for python/lsst/ap/association/mapApData.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# Developed for the LSST Data Management System.
3# This product includes software developed by the LSST Project
4# (http://www.lsst.org).
5# See the COPYRIGHT file at the top-level directory of this distribution
6# for details of code ownership.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License
19# along with this program. If not, see <http://www.gnu.org/licenses/>.
20#
22"""Classes for taking science pipeline outputs and creating data products for
23use in ap_association and the alert production database (APDB).
24"""
26__all__ = ["MapApDataConfig", "MapApDataTask",
27 "MapDiaSourceConfig", "MapDiaSourceTask",
28 "UnpackApdbFlags"]
30import numpy as np
31import os
32import yaml
34import lsst.afw.table as afwTable
35from lsst.daf.base import DateTime
36import lsst.pipe.base as pipeBase
37import lsst.pex.config as pexConfig
38from lsst.pex.exceptions import RuntimeError
39from lsst.utils import getPackageDir
40from .afwUtils import make_dia_source_schema
43class MapApDataConfig(pexConfig.Config):
44 """Configuration for the generic MapApDataTask class.
45 """
46 copyColumns = pexConfig.DictField(
47 keytype=str,
48 itemtype=str,
49 doc="Mapping of input SciencePipelines columns to output DPDD "
50 "columns.",
51 default={"id": "id",
52 "parent": "parent",
53 "coord_ra": "coord_ra",
54 "coord_dec": "coord_dec"}
55 )
58class MapApDataTask(pipeBase.Task):
59 """Generic mapper class for copying values from a science pipelines catalog
60 into a product for use in ap_association or the APDB.
61 """
62 ConfigClass = MapApDataConfig
63 _DefaultName = "mapApDataTask"
65 def __init__(self, inputSchema, outputSchema, **kwargs):
66 pipeBase.Task.__init__(self, **kwargs)
67 self.inputSchema = inputSchema
68 self.outputSchema = outputSchema
70 self.mapper = afwTable.SchemaMapper(inputSchema, outputSchema)
72 for inputName, outputName in self.config.copyColumns.items():
73 self.mapper.addMapping(
74 self.inputSchema.find(inputName).key,
75 outputName,
76 True)
78 def run(self, inputCatalog, exposure=None):
79 """Copy data from the inputCatalog into an output catalog with
80 requested columns.
82 Parameters
83 ----------
84 inputCatalog: `lsst.afw.table.SourceCatalog`
85 Input catalog with data to be copied into new output catalog.
87 Returns
88 -------
89 outputCatalog: `lsst.afw.table.SourceCatalog`
90 Output catalog with data copied from input and new column names.
91 """
92 outputCatalog = afwTable.SourceCatalog(self.outputSchema)
93 outputCatalog.extend(inputCatalog, self.mapper)
95 if not outputCatalog.isContiguous():
96 raise RuntimeError("Output catalogs must be contiguous.")
98 return outputCatalog
101class MapDiaSourceConfig(pexConfig.Config):
102 """Config for the DiaSourceMapperTask
103 """
104 copyColumns = pexConfig.DictField(
105 keytype=str,
106 itemtype=str,
107 doc="Mapping of input SciencePipelines columns to output DPDD "
108 "columns.",
109 default={"id": "id",
110 "parent": "parent",
111 "coord_ra": "coord_ra",
112 "coord_dec": "coord_dec",
113 "slot_Centroid_x": "x",
114 "slot_Centroid_xErr": "xErr",
115 "slot_Centroid_y": "y",
116 "slot_Centroid_yErr": "yErr",
117 "slot_ApFlux_instFlux": "apFlux",
118 "slot_ApFlux_instFluxErr": "apFluxErr",
119 "slot_PsfFlux_instFlux": "psFlux",
120 "slot_PsfFlux_instFluxErr": "psFluxErr",
121 "ip_diffim_DipoleFit_orientation": "dipAngle",
122 "ip_diffim_DipoleFit_chi2dof": "dipChi2",
123 "ip_diffim_forced_PsfFlux_instFlux": "totFlux",
124 "ip_diffim_forced_PsfFlux_instFluxErr": "totFluxErr",
125 "ip_diffim_DipoleFit_flag_classification": "isDipole",
126 "slot_Shape_xx": "ixx",
127 "slot_Shape_xxErr": "ixxErr",
128 "slot_Shape_yy": "iyy",
129 "slot_Shape_yyErr": "iyyErr",
130 "slot_Shape_xy": "ixy",
131 "slot_Shape_xyErr": "ixyErr",
132 "slot_PsfShape_xx": "ixxPSF",
133 "slot_PsfShape_yy": "iyyPSF",
134 "slot_PsfShape_xy": "ixyPSF"}
135 )
136 calibrateColumns = pexConfig.ListField(
137 dtype=str,
138 doc="Flux columns in the input catalog to calibrate.",
139 default=["slot_ApFlux", "slot_PsfFlux", "ip_diffim_forced_PsfFlux"]
140 )
141 flagMap = pexConfig.Field(
142 dtype=str,
143 doc="Yaml file specifying SciencePipelines flag fields to bit packs.",
144 default=os.path.join(getPackageDir("ap_association"),
145 "data",
146 "association-flag-map.yaml"),
147 )
148 dipFluxPrefix = pexConfig.Field(
149 dtype=str,
150 doc="Prefix of the Dipole measurement column containing negative and "
151 "positive flux lobes.",
152 default="ip_diffim_DipoleFit",
153 )
154 dipSepColumn = pexConfig.Field(
155 dtype=str,
156 doc="Column of the separation of the negative and positive poles of "
157 "the dipole.",
158 default="ip_diffim_DipoleFit_separation"
159 )
162class MapDiaSourceTask(MapApDataTask):
163 """Task specific for copying columns from science pipelines catalogs,
164 calibrating them, for use in ap_association and the APDB.
166 This task also copies information from the exposure such as the ExpsoureId
167 and the exposure date as specified in the DPDD.
168 """
170 ConfigClass = MapDiaSourceConfig
171 _DefaultName = "mapDiaSourceTask"
173 def __init__(self, inputSchema, **kwargs):
174 MapApDataTask.__init__(self,
175 inputSchema=inputSchema,
176 outputSchema=make_dia_source_schema(),
177 **kwargs)
178 self._create_bit_pack_mappings()
180 def _create_bit_pack_mappings(self):
181 """Setup all flag bit packings.
182 """
183 self.bit_pack_columns = []
184 with open(self.config.flagMap) as yaml_stream:
185 table_list = list(yaml.safe_load_all(yaml_stream))
186 for table in table_list:
187 if table['tableName'] == 'DiaSource':
188 self.bit_pack_columns = table['columns']
189 break
191 # Test that all flags requested are present in both the input and
192 # output schemas.
193 for outputFlag in self.bit_pack_columns:
194 try:
195 self.outputSchema.find(outputFlag['columnName'])
196 except KeyError:
197 raise KeyError(
198 "Requested column %s not found in MapDiaSourceTask output "
199 "schema. Please check that the requested output column "
200 "exists." % outputFlag['columnName'])
201 bitList = outputFlag['bitList']
202 for bit in bitList:
203 try:
204 self.inputSchema.find(bit['name'])
205 except KeyError:
206 raise KeyError(
207 "Requested column %s not found in MapDiaSourceTask input "
208 "schema. Please check that the requested input column "
209 "exists." % outputFlag['columnName'])
211 def run(self, inputCatalog, exposure, return_pandas=False):
212 """Copy data from the inputCatalog into an output catalog with
213 requested columns.
215 Parameters
216 ----------
217 inputCatalog : `lsst.afw.table.SourceCatalog`
218 Input catalog with data to be copied into new output catalog.
219 exposure: `lsst.afw.image.Exposure`
220 Exposure with containing the PhotoCalib object relevant to this
221 catalog.
222 return_pandas : `bool`
223 Return `pandas.DataFrame` instead of `lsst.afw.table.SourceCatalog`
225 Returns
226 -------
227 outputCatalog: `lsst.afw.table.SourceCatalog` or `pandas.DataFrame`
228 Output catalog with data copied from input and new column names.
229 """
230 visit_info = exposure.getInfo().getVisitInfo()
231 ccdVisitId = visit_info.getExposureId()
232 midPointTaiMJD = visit_info.getDate().get(system=DateTime.MJD)
233 filterId = exposure.getFilter().getId()
234 # canonical name should always be the abstract filter (in Gen 3 sense)
235 # TODO DM-21333: Remove [0] (first character only) workaround
236 filterName = exposure.getFilter().getCanonicalName()[0]
237 wcs = exposure.getWcs()
239 photoCalib = exposure.getPhotoCalib()
241 outputCatalog = afwTable.SourceCatalog(self.outputSchema)
242 outputCatalog.reserve(len(inputCatalog))
244 for inputRecord in inputCatalog:
245 outputRecord = outputCatalog.addNew()
246 outputRecord.assign(inputRecord, self.mapper)
247 self.calibrateFluxes(inputRecord, outputRecord, photoCalib)
248 self.computeDipoleFluxes(inputRecord, outputRecord, photoCalib)
249 self.computeDipoleSep(inputRecord, outputRecord, wcs)
250 self.bitPackFlags(inputRecord, outputRecord)
251 self.computeBBoxSize(inputRecord, outputRecord)
252 outputRecord.set("ccdVisitId", ccdVisitId)
253 outputRecord.set("midPointTai", midPointTaiMJD)
254 outputRecord.set("filterId", filterId)
255 outputRecord.set("filterName", filterName)
257 if not outputCatalog.isContiguous():
258 raise RuntimeError("Output catalogs must be contiguous.")
260 if return_pandas:
261 return self._convert_to_pandas(outputCatalog)
262 return outputCatalog
264 def calibrateFluxes(self, inputRecord, outputRecord, photoCalib):
265 """Copy flux values into an output record and calibrate them.
267 Parameters
268 ----------
269 inputRecord : `lsst.afw.table.SourceRecord`
270 Record to copy flux values from.
271 outputRecord : `lsst.afw.table.SourceRecord`
272 Record to copy and calibrate values into.
273 photoCalib : `lsst.afw.image.PhotoCalib`
274 Calibration object from the difference exposure.
275 """
276 for col_name in self.config.calibrateColumns:
277 meas = photoCalib.instFluxToNanojansky(inputRecord, col_name)
278 outputRecord.set(self.config.copyColumns[col_name + "_instFlux"],
279 meas.value)
280 outputRecord.set(
281 self.config.copyColumns[col_name + "_instFluxErr"],
282 meas.error)
284 def computeDipoleFluxes(self, inputRecord, outputRecord, photoCalib):
285 """Calibrate and compute dipole mean flux and diff flux.
287 Parameters
288 ----------
289 inputRecord : `lsst.afw.table.SourceRecord`
290 Record to copy flux values from.
291 outputRecord : `lsst.afw.table.SourceRecord`
292 Record to copy and calibrate values into.
293 photoCalib `lsst.afw.image.PhotoCalib`
294 Calibration object from the difference exposure.
295 """
297 neg_meas = photoCalib.instFluxToNanojansky(
298 inputRecord, self.config.dipFluxPrefix + "_neg")
299 pos_meas = photoCalib.instFluxToNanojansky(
300 inputRecord, self.config.dipFluxPrefix + "_pos")
301 outputRecord.set(
302 "dipMeanFlux",
303 0.5 * (np.abs(neg_meas.value) + np.abs(pos_meas.value)))
304 outputRecord.set(
305 "dipMeanFluxErr",
306 0.5 * np.sqrt(neg_meas.error ** 2 + pos_meas.error ** 2))
307 outputRecord.set(
308 "dipFluxDiff",
309 np.abs(pos_meas.value) - np.abs(neg_meas.value))
310 outputRecord.set(
311 "dipFluxDiffErr",
312 np.sqrt(neg_meas.error ** 2 + pos_meas.error ** 2))
314 def computeDipoleSep(self, inputRecord, outputRecord, wcs):
315 """Convert the dipole separation from pixels to arcseconds.
317 Parameters
318 ----------
319 inputRecord : `lsst.afw.table.SourceRecord`
320 Record to copy flux values from.
321 outputRecord : `lsst.afw.table.SourceRecord`
322 Record to copy and calibrate values into.
323 wcs : `lsst.afw.geom.SkyWcs`
324 Wcs of image inputRecords was observed.
325 """
326 pixScale = wcs.getPixelScale(inputRecord.getCentroid())
327 dipSep = pixScale * inputRecord.get(self.config.dipSepColumn)
328 outputRecord.set("dipLength", dipSep.asArcseconds())
330 def bitPackFlags(self, inputRecord, outputRecord):
331 """Pack requested flag columns in inputRecord into single columns in
332 outputRecord.
334 Parameters
335 ----------
336 inputRecord : `lsst.afw.table.SourceRecord`
337 Record to copy flux values from.
338 outputRecord : `lsst.afw.table.SourceRecord`
339 Record to copy and calibrate values into.
340 """
341 for outputFlag in self.bit_pack_columns:
342 bitList = outputFlag['bitList']
343 value = 0
344 for bit in bitList:
345 value += inputRecord[bit['name']] * 2 ** bit['bit']
346 outputRecord.set(outputFlag['columnName'], value)
348 def computeBBoxSize(self, inputRecord, outputRecord):
349 """Compute the size of a square bbox that fully contains the detection
350 footprint.
352 Parameters
353 ----------
354 inputRecord : `lsst.afw.table.SourceRecord`
355 Record to copy flux values from.
356 outputRecord : `lsst.afw.table.SourceRecord`
357 Record to copy and calibrate values into.
358 """
359 footprintBBox = inputRecord.getFootprint().getBBox()
360 # Compute twice the size of the largest dimension of the footprint
361 # bounding box. This is the largest footprint we should need to cover
362 # the complete DiaSource assuming the centroid is withing the bounding
363 # box.
364 maxSize = 2 * np.max([footprintBBox.getWidth(),
365 footprintBBox.getHeight()])
366 recX = inputRecord.getCentroid().x
367 recY = inputRecord.getCentroid().y
368 bboxSize = int(
369 np.ceil(2 * np.max(np.fabs([footprintBBox.maxX - recX,
370 footprintBBox.minX - recX,
371 footprintBBox.maxY - recY,
372 footprintBBox.minY - recY]))))
373 if bboxSize > maxSize:
374 bboxSize = maxSize
375 outputRecord.set("bboxSize", bboxSize)
377 def _convert_to_pandas(self, inputCatalog):
378 """Convert input afw table to pandas.
380 Using afwTable.toAstropy().to_pandas() alone is not sufficient to
381 properly store data in the Apdb. We must also convert the RA/DEC values
382 from radians to degrees and rename several columns.
384 Parameters
385 ----------
386 inputCatalog : `lsst.afw.table.SourceCatalog`
387 Catalog to convert to panads and rename columns.
389 Returns
390 -------
391 catalog : `pandas.DataFrame`
392 """
393 catalog = inputCatalog.asAstropy().to_pandas()
394 catalog.rename(columns={"coord_ra": "ra",
395 "coord_dec": "decl",
396 "id": "diaSourceId",
397 "parent": "parentDiaSourceId"},
398 inplace=True)
399 catalog["ra"] = np.degrees(catalog["ra"])
400 catalog["decl"] = np.degrees(catalog["decl"])
402 return catalog
405class UnpackApdbFlags:
406 """Class for unpacking bits from integer flag fields stored in the Apdb.
408 Attributes
409 ----------
410 flag_map_file : `str`
411 Absolute or relative path to a yaml file specifiying mappings of flags
412 to integer bits.
413 table_name : `str`
414 Name of the Apdb table the integer bit data are coming from.
415 """
417 def __init__(self, flag_map_file, table_name):
418 self.bit_pack_columns = []
419 with open(flag_map_file) as yaml_stream:
420 table_list = list(yaml.safe_load_all(yaml_stream))
421 for table in table_list:
422 if table['tableName'] == table_name:
423 self.bit_pack_columns = table['columns']
424 break
426 self.output_flag_columns = {}
428 for column in self.bit_pack_columns:
429 names = []
430 for bit in column["bitList"]:
431 names.append((bit["name"], np.bool))
432 self.output_flag_columns[column["columnName"]] = names
434 def unpack(self, input_flag_values, flag_name):
435 """Determine individual boolean flags from an input array of unsigned
436 ints.
438 Parameters
439 ----------
440 input_flag_values : array-like of type uint
441 Input integer flags to unpack.
442 flag_name : `str`
443 Apdb column name of integer flags to unpack. Names of packed int
444 flags are given by the flag_map_file.
446 Returns
447 -------
448 output_flags : `numpy.ndarray`
449 Numpy named tuple of booleans.
450 """
451 bit_names_types = self.output_flag_columns[flag_name]
452 output_flags = np.zeros(len(input_flag_values), dtype=bit_names_types)
454 for bit_idx, (bit_name, dtypes) in enumerate(bit_names_types):
455 masked_bits = np.bitwise_and(input_flag_values, 2 ** bit_idx)
456 output_flags[bit_name] = masked_bits
458 return output_flags