Coverage for python/lsst/ap/association/mapApData.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# Developed for the LSST Data Management System.
3# This product includes software developed by the LSST Project
4# (http://www.lsst.org).
5# See the COPYRIGHT file at the top-level directory of this distribution
6# for details of code ownership.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License
19# along with this program. If not, see <http://www.gnu.org/licenses/>.
20#
22"""Classes for taking science pipeline outputs and creating data products for
23use in ap_association and the alert production database (APDB).
24"""
26__all__ = ["MapApDataConfig", "MapApDataTask",
27 "MapDiaSourceConfig", "MapDiaSourceTask",
28 "UnpackApdbFlags"]
30import numpy as np
31import os
32import yaml
34import lsst.afw.table as afwTable
35from lsst.daf.base import DateTime
36import lsst.pipe.base as pipeBase
37import lsst.pex.config as pexConfig
38from lsst.pex.exceptions import RuntimeError
39from lsst.utils import getPackageDir
40from .afwUtils import make_dia_source_schema
43class MapApDataConfig(pexConfig.Config):
44 """Configuration for the generic MapApDataTask class.
45 """
46 copyColumns = pexConfig.DictField(
47 keytype=str,
48 itemtype=str,
49 doc="Mapping of input SciencePipelines columns to output DPDD "
50 "columns.",
51 default={"id": "id",
52 "parent": "parent",
53 "coord_ra": "coord_ra",
54 "coord_dec": "coord_dec"}
55 )
58class MapApDataTask(pipeBase.Task):
59 """Generic mapper class for copying values from a science pipelines catalog
60 into a product for use in ap_association or the APDB.
61 """
62 ConfigClass = MapApDataConfig
63 _DefaultName = "mapApDataTask"
65 def __init__(self, inputSchema, outputSchema, **kwargs):
66 pipeBase.Task.__init__(self, **kwargs)
67 self.inputSchema = inputSchema
68 self.outputSchema = outputSchema
70 self.mapper = afwTable.SchemaMapper(inputSchema, outputSchema)
72 for inputName, outputName in self.config.copyColumns.items():
73 self.mapper.addMapping(
74 self.inputSchema.find(inputName).key,
75 outputName,
76 True)
78 @pipeBase.timeMethod
79 def run(self, inputCatalog, exposure=None):
80 """Copy data from the inputCatalog into an output catalog with
81 requested columns.
83 Parameters
84 ----------
85 inputCatalog: `lsst.afw.table.SourceCatalog`
86 Input catalog with data to be copied into new output catalog.
88 Returns
89 -------
90 outputCatalog: `lsst.afw.table.SourceCatalog`
91 Output catalog with data copied from input and new column names.
92 """
93 outputCatalog = afwTable.SourceCatalog(self.outputSchema)
94 outputCatalog.extend(inputCatalog, self.mapper)
96 if not outputCatalog.isContiguous():
97 raise RuntimeError("Output catalogs must be contiguous.")
99 return outputCatalog
102class MapDiaSourceConfig(pexConfig.Config):
103 """Config for the DiaSourceMapperTask
104 """
105 copyColumns = pexConfig.DictField(
106 keytype=str,
107 itemtype=str,
108 doc="Mapping of input SciencePipelines columns to output DPDD "
109 "columns.",
110 default={"id": "id",
111 "parent": "parent",
112 "coord_ra": "coord_ra",
113 "coord_dec": "coord_dec",
114 "slot_Centroid_x": "x",
115 "slot_Centroid_xErr": "xErr",
116 "slot_Centroid_y": "y",
117 "slot_Centroid_yErr": "yErr",
118 "slot_ApFlux_instFlux": "apFlux",
119 "slot_ApFlux_instFluxErr": "apFluxErr",
120 "slot_PsfFlux_instFlux": "psFlux",
121 "slot_PsfFlux_instFluxErr": "psFluxErr",
122 "ip_diffim_DipoleFit_orientation": "dipAngle",
123 "ip_diffim_DipoleFit_chi2dof": "dipChi2",
124 "ip_diffim_forced_PsfFlux_instFlux": "totFlux",
125 "ip_diffim_forced_PsfFlux_instFluxErr": "totFluxErr",
126 "ip_diffim_DipoleFit_flag_classification": "isDipole",
127 "slot_Shape_xx": "ixx",
128 "slot_Shape_xxErr": "ixxErr",
129 "slot_Shape_yy": "iyy",
130 "slot_Shape_yyErr": "iyyErr",
131 "slot_Shape_xy": "ixy",
132 "slot_Shape_xyErr": "ixyErr",
133 "slot_PsfShape_xx": "ixxPSF",
134 "slot_PsfShape_yy": "iyyPSF",
135 "slot_PsfShape_xy": "ixyPSF"}
136 )
137 calibrateColumns = pexConfig.ListField(
138 dtype=str,
139 doc="Flux columns in the input catalog to calibrate.",
140 default=["slot_ApFlux", "slot_PsfFlux", "ip_diffim_forced_PsfFlux"]
141 )
142 flagMap = pexConfig.Field(
143 dtype=str,
144 doc="Yaml file specifying SciencePipelines flag fields to bit packs.",
145 default=os.path.join(getPackageDir("ap_association"),
146 "data",
147 "association-flag-map.yaml"),
148 )
149 dipFluxPrefix = pexConfig.Field(
150 dtype=str,
151 doc="Prefix of the Dipole measurement column containing negative and "
152 "positive flux lobes.",
153 default="ip_diffim_DipoleFit",
154 )
155 dipSepColumn = pexConfig.Field(
156 dtype=str,
157 doc="Column of the separation of the negative and positive poles of "
158 "the dipole.",
159 default="ip_diffim_DipoleFit_separation"
160 )
163class MapDiaSourceTask(MapApDataTask):
164 """Task specific for copying columns from science pipelines catalogs,
165 calibrating them, for use in ap_association and the APDB.
167 This task also copies information from the exposure such as the ExpsoureId
168 and the exposure date as specified in the DPDD.
169 """
171 ConfigClass = MapDiaSourceConfig
172 _DefaultName = "mapDiaSourceTask"
174 def __init__(self, inputSchema, **kwargs):
175 MapApDataTask.__init__(self,
176 inputSchema=inputSchema,
177 outputSchema=make_dia_source_schema(),
178 **kwargs)
179 self._create_bit_pack_mappings()
181 def _create_bit_pack_mappings(self):
182 """Setup all flag bit packings.
183 """
184 self.bit_pack_columns = []
185 with open(self.config.flagMap) as yaml_stream:
186 table_list = list(yaml.safe_load_all(yaml_stream))
187 for table in table_list:
188 if table['tableName'] == 'DiaSource':
189 self.bit_pack_columns = table['columns']
190 break
192 # Test that all flags requested are present in both the input and
193 # output schemas.
194 for outputFlag in self.bit_pack_columns:
195 try:
196 self.outputSchema.find(outputFlag['columnName'])
197 except KeyError:
198 raise KeyError(
199 "Requested column %s not found in MapDiaSourceTask output "
200 "schema. Please check that the requested output column "
201 "exists." % outputFlag['columnName'])
202 bitList = outputFlag['bitList']
203 for bit in bitList:
204 try:
205 self.inputSchema.find(bit['name'])
206 except KeyError:
207 raise KeyError(
208 "Requested column %s not found in MapDiaSourceTask input "
209 "schema. Please check that the requested input column "
210 "exists." % outputFlag['columnName'])
212 @pipeBase.timeMethod
213 def run(self, inputCatalog, exposure, return_pandas=False):
214 """Copy data from the inputCatalog into an output catalog with
215 requested columns.
217 Parameters
218 ----------
219 inputCatalog : `lsst.afw.table.SourceCatalog`
220 Input catalog with data to be copied into new output catalog.
221 exposure: `lsst.afw.image.Exposure`
222 Exposure with containing the PhotoCalib object relevant to this
223 catalog.
224 return_pandas : `bool`
225 Return `pandas.DataFrame` instead of `lsst.afw.table.SourceCatalog`
227 Returns
228 -------
229 outputCatalog: `lsst.afw.table.SourceCatalog` or `pandas.DataFrame`
230 Output catalog with data copied from input and new column names.
231 """
232 visit_info = exposure.getInfo().getVisitInfo()
233 ccdVisitId = visit_info.getExposureId()
234 midPointTaiMJD = visit_info.getDate().get(system=DateTime.MJD)
235 filterName = exposure.getFilterLabel().bandLabel
236 wcs = exposure.getWcs()
238 photoCalib = exposure.getPhotoCalib()
240 outputCatalog = afwTable.SourceCatalog(self.outputSchema)
241 outputCatalog.reserve(len(inputCatalog))
243 for inputRecord in inputCatalog:
244 outputRecord = outputCatalog.addNew()
245 outputRecord.assign(inputRecord, self.mapper)
246 self.calibrateFluxes(inputRecord, outputRecord, photoCalib)
247 self.computeDipoleFluxes(inputRecord, outputRecord, photoCalib)
248 self.computeDipoleSep(inputRecord, outputRecord, wcs)
249 self.bitPackFlags(inputRecord, outputRecord)
250 self.computeBBoxSize(inputRecord, outputRecord)
251 outputRecord.set("ccdVisitId", ccdVisitId)
252 outputRecord.set("midPointTai", midPointTaiMJD)
253 outputRecord.set("filterName", filterName)
255 if not outputCatalog.isContiguous():
256 raise RuntimeError("Output catalogs must be contiguous.")
258 if return_pandas:
259 return self._convert_to_pandas(outputCatalog)
260 return outputCatalog
262 def calibrateFluxes(self, inputRecord, outputRecord, photoCalib):
263 """Copy flux values into an output record and calibrate them.
265 Parameters
266 ----------
267 inputRecord : `lsst.afw.table.SourceRecord`
268 Record to copy flux values from.
269 outputRecord : `lsst.afw.table.SourceRecord`
270 Record to copy and calibrate values into.
271 photoCalib : `lsst.afw.image.PhotoCalib`
272 Calibration object from the difference exposure.
273 """
274 for col_name in self.config.calibrateColumns:
275 meas = photoCalib.instFluxToNanojansky(inputRecord, col_name)
276 outputRecord.set(self.config.copyColumns[col_name + "_instFlux"],
277 meas.value)
278 outputRecord.set(
279 self.config.copyColumns[col_name + "_instFluxErr"],
280 meas.error)
282 def computeDipoleFluxes(self, inputRecord, outputRecord, photoCalib):
283 """Calibrate and compute dipole mean flux and diff flux.
285 Parameters
286 ----------
287 inputRecord : `lsst.afw.table.SourceRecord`
288 Record to copy flux values from.
289 outputRecord : `lsst.afw.table.SourceRecord`
290 Record to copy and calibrate values into.
291 photoCalib `lsst.afw.image.PhotoCalib`
292 Calibration object from the difference exposure.
293 """
295 neg_meas = photoCalib.instFluxToNanojansky(
296 inputRecord, self.config.dipFluxPrefix + "_neg")
297 pos_meas = photoCalib.instFluxToNanojansky(
298 inputRecord, self.config.dipFluxPrefix + "_pos")
299 outputRecord.set(
300 "dipMeanFlux",
301 0.5 * (np.abs(neg_meas.value) + np.abs(pos_meas.value)))
302 outputRecord.set(
303 "dipMeanFluxErr",
304 0.5 * np.sqrt(neg_meas.error ** 2 + pos_meas.error ** 2))
305 outputRecord.set(
306 "dipFluxDiff",
307 np.abs(pos_meas.value) - np.abs(neg_meas.value))
308 outputRecord.set(
309 "dipFluxDiffErr",
310 np.sqrt(neg_meas.error ** 2 + pos_meas.error ** 2))
312 def computeDipoleSep(self, inputRecord, outputRecord, wcs):
313 """Convert the dipole separation from pixels to arcseconds.
315 Parameters
316 ----------
317 inputRecord : `lsst.afw.table.SourceRecord`
318 Record to copy flux values from.
319 outputRecord : `lsst.afw.table.SourceRecord`
320 Record to copy and calibrate values into.
321 wcs : `lsst.afw.geom.SkyWcs`
322 Wcs of image inputRecords was observed.
323 """
324 pixScale = wcs.getPixelScale(inputRecord.getCentroid())
325 dipSep = pixScale * inputRecord.get(self.config.dipSepColumn)
326 outputRecord.set("dipLength", dipSep.asArcseconds())
328 def bitPackFlags(self, inputRecord, outputRecord):
329 """Pack requested flag columns in inputRecord into single columns in
330 outputRecord.
332 Parameters
333 ----------
334 inputRecord : `lsst.afw.table.SourceRecord`
335 Record to copy flux values from.
336 outputRecord : `lsst.afw.table.SourceRecord`
337 Record to copy and calibrate values into.
338 """
339 for outputFlag in self.bit_pack_columns:
340 bitList = outputFlag['bitList']
341 value = 0
342 for bit in bitList:
343 value += inputRecord[bit['name']] * 2 ** bit['bit']
344 outputRecord.set(outputFlag['columnName'], value)
346 def computeBBoxSize(self, inputRecord, outputRecord):
347 """Compute the size of a square bbox that fully contains the detection
348 footprint.
350 Parameters
351 ----------
352 inputRecord : `lsst.afw.table.SourceRecord`
353 Record to copy flux values from.
354 outputRecord : `lsst.afw.table.SourceRecord`
355 Record to copy and calibrate values into.
356 """
357 footprintBBox = inputRecord.getFootprint().getBBox()
358 # Compute twice the size of the largest dimension of the footprint
359 # bounding box. This is the largest footprint we should need to cover
360 # the complete DiaSource assuming the centroid is withing the bounding
361 # box.
362 maxSize = 2 * np.max([footprintBBox.getWidth(),
363 footprintBBox.getHeight()])
364 recX = inputRecord.getCentroid().x
365 recY = inputRecord.getCentroid().y
366 bboxSize = int(
367 np.ceil(2 * np.max(np.fabs([footprintBBox.maxX - recX,
368 footprintBBox.minX - recX,
369 footprintBBox.maxY - recY,
370 footprintBBox.minY - recY]))))
371 if bboxSize > maxSize:
372 bboxSize = maxSize
373 outputRecord.set("bboxSize", bboxSize)
375 def _convert_to_pandas(self, inputCatalog):
376 """Convert input afw table to pandas.
378 Using afwTable.toAstropy().to_pandas() alone is not sufficient to
379 properly store data in the Apdb. We must also convert the RA/DEC values
380 from radians to degrees and rename several columns.
382 Parameters
383 ----------
384 inputCatalog : `lsst.afw.table.SourceCatalog`
385 Catalog to convert to panads and rename columns.
387 Returns
388 -------
389 catalog : `pandas.DataFrame`
390 """
391 catalog = inputCatalog.asAstropy().to_pandas()
392 catalog.rename(columns={"coord_ra": "ra",
393 "coord_dec": "decl",
394 "id": "diaSourceId",
395 "parent": "parentDiaSourceId"},
396 inplace=True)
397 catalog["ra"] = np.degrees(catalog["ra"])
398 catalog["decl"] = np.degrees(catalog["decl"])
400 return catalog
403class UnpackApdbFlags:
404 """Class for unpacking bits from integer flag fields stored in the Apdb.
406 Attributes
407 ----------
408 flag_map_file : `str`
409 Absolute or relative path to a yaml file specifiying mappings of flags
410 to integer bits.
411 table_name : `str`
412 Name of the Apdb table the integer bit data are coming from.
413 """
415 def __init__(self, flag_map_file, table_name):
416 self.bit_pack_columns = []
417 with open(flag_map_file) as yaml_stream:
418 table_list = list(yaml.safe_load_all(yaml_stream))
419 for table in table_list:
420 if table['tableName'] == table_name:
421 self.bit_pack_columns = table['columns']
422 break
424 self.output_flag_columns = {}
426 for column in self.bit_pack_columns:
427 names = []
428 for bit in column["bitList"]:
429 names.append((bit["name"], np.bool))
430 self.output_flag_columns[column["columnName"]] = names
432 def unpack(self, input_flag_values, flag_name):
433 """Determine individual boolean flags from an input array of unsigned
434 ints.
436 Parameters
437 ----------
438 input_flag_values : array-like of type uint
439 Input integer flags to unpack.
440 flag_name : `str`
441 Apdb column name of integer flags to unpack. Names of packed int
442 flags are given by the flag_map_file.
444 Returns
445 -------
446 output_flags : `numpy.ndarray`
447 Numpy named tuple of booleans.
448 """
449 bit_names_types = self.output_flag_columns[flag_name]
450 output_flags = np.zeros(len(input_flag_values), dtype=bit_names_types)
452 for bit_idx, (bit_name, dtypes) in enumerate(bit_names_types):
453 masked_bits = np.bitwise_and(input_flag_values, 2 ** bit_idx)
454 output_flags[bit_name] = masked_bits
456 return output_flags