Coverage for python/lsst/ap/verify/testPipeline.py: 44%
122 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-23 04:14 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-23 04:14 -0700
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
25# These classes exist only to be included in a mock pipeline, and don't need
26# to be public for that.
27__all__ = []
30import numpy as np
31import pandas
33import lsst.geom as geom
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.afw.table as afwTable
37from lsst.pipe.base import PipelineTask, Struct
38from lsst.ip.isr import IsrTaskConfig
39from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig
40from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
41from lsst.pipe.tasks.calibrate import CalibrateConfig
42from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig
45class MockIsrTask(PipelineTask):
46 """A do-nothing substitute for IsrTask.
47 """
48 ConfigClass = IsrTaskConfig
49 _DefaultName = "notIsr"
51 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
52 crosstalk=None, crosstalkSources=None,
53 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
54 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
55 sensorTransmission=None, atmosphereTransmission=None,
56 detectorNum=None, strayLightData=None, illumMaskedImage=None,
57 deferredCharge=None,
58 ):
59 """Accept ISR inputs, and produce ISR outputs with no processing.
61 Parameters
62 ----------
63 ccdExposure : `lsst.afw.image.Exposure`
64 The raw exposure that is to be run through ISR. The
65 exposure is modified by this method.
66 camera : `lsst.afw.cameraGeom.Camera`, optional
67 The camera geometry for this exposure. Required if
68 one or more of ``ccdExposure``, ``bias``, ``dark``, or
69 ``flat`` does not have an associated detector.
70 bias : `lsst.afw.image.Exposure`, optional
71 Bias calibration frame.
72 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
73 Functor for linearization.
74 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
75 Calibration for crosstalk.
76 crosstalkSources : `list`, optional
77 List of possible crosstalk sources.
78 dark : `lsst.afw.image.Exposure`, optional
79 Dark calibration frame.
80 flat : `lsst.afw.image.Exposure`, optional
81 Flat calibration frame.
82 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
83 Photon transfer curve dataset, with, e.g., gains
84 and read noise.
85 bfKernel : `numpy.ndarray`, optional
86 Brighter-fatter kernel.
87 bfGains : `dict` of `float`, optional
88 Gains used to override the detector's nominal gains for the
89 brighter-fatter correction. A dict keyed by amplifier name for
90 the detector in question.
91 defects : `lsst.ip.isr.Defects`, optional
92 List of defects.
93 fringes : `lsst.pipe.base.Struct`, optional
94 Struct containing the fringe correction data, with
95 elements:
96 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
97 - ``seed``: random seed derived from the ccdExposureId for random
98 number generator (`uint32`)
99 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
100 A ``TransmissionCurve`` that represents the throughput of the
101 optics, to be evaluated in focal-plane coordinates.
102 filterTransmission : `lsst.afw.image.TransmissionCurve`
103 A ``TransmissionCurve`` that represents the throughput of the
104 filter itself, to be evaluated in focal-plane coordinates.
105 sensorTransmission : `lsst.afw.image.TransmissionCurve`
106 A ``TransmissionCurve`` that represents the throughput of the
107 sensor itself, to be evaluated in post-assembly trimmed detector
108 coordinates.
109 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
110 A ``TransmissionCurve`` that represents the throughput of the
111 atmosphere, assumed to be spatially constant.
112 detectorNum : `int`, optional
113 The integer number for the detector to process.
114 isGen3 : bool, optional
115 Flag this call to run() as using the Gen3 butler environment.
116 strayLightData : `object`, optional
117 Opaque object containing calibration information for stray-light
118 correction. If `None`, no correction will be performed.
119 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
120 Illumination correction image.
122 Returns
123 -------
124 result : `lsst.pipe.base.Struct`
125 Result struct with components:
127 ``exposure``
128 The fully ISR corrected exposure (`afw.image.Exposure`).
129 ``outputExposure``
130 An alias for ``exposure`` (`afw.image.Exposure`).
131 ``ossThumb``
132 Thumbnail image of the exposure after overscan subtraction
133 (`numpy.ndarray`).
134 ``flattenedThumb``
135 Thumbnail image of the exposure after flat-field correction
136 (`numpy.ndarray`).
137 - ``outputStatistics`` : mapping [`str`]
138 Values of the additional statistics calculated.
139 """
140 return Struct(exposure=afwImage.ExposureF(),
141 outputExposure=afwImage.ExposureF(),
142 ossThumb=np.empty((1, 1)),
143 flattenedThumb=np.empty((1, 1)),
144 preInterpExposure=afwImage.ExposureF(),
145 outputOssThumbnail=np.empty((1, 1)),
146 outputFlattenedThumbnail=np.empty((1, 1)),
147 outputStatistics={},
148 )
151class MockCharacterizeImageTask(PipelineTask):
152 """A do-nothing substitute for CharacterizeImageTask.
153 """
154 ConfigClass = CharacterizeImageConfig
155 _DefaultName = "notCharacterizeImage"
157 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
158 super().__init__(**kwargs)
159 self.outputSchema = afwTable.SourceCatalog()
161 def runQuantum(self, butlerQC, inputRefs, outputRefs):
162 inputs = butlerQC.get(inputRefs)
163 if 'idGenerator' not in inputs.keys():
164 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
165 outputs = self.run(**inputs)
166 butlerQC.put(outputs, outputRefs)
168 def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None):
169 """Produce characterization outputs with no processing.
171 Parameters
172 ----------
173 exposure : `lsst.afw.image.Exposure`
174 Exposure to characterize.
175 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
176 ID info for exposure. Deprecated in favor of ``idGenerator``, and
177 ignored if that is provided.
178 background : `lsst.afw.math.BackgroundList`, optional
179 Initial model of background already subtracted from exposure.
180 idGenerator : `lsst.meas.base.IdGenerator`, optional
181 Object that generates source IDs and provides random number
182 generator seeds.
184 Returns
185 -------
186 result : `lsst.pipe.base.Struct`
187 Struct containing these fields:
189 ``characterized``
190 Characterized exposure (`lsst.afw.image.Exposure`).
191 ``sourceCat``
192 Detected sources (`lsst.afw.table.SourceCatalog`).
193 ``backgroundModel``
194 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`)
195 ``psfCellSet``
196 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`)
197 """
198 # Can't persist empty BackgroundList; DM-33714
199 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
200 afwImage.MaskedImageF(16, 16))
201 return Struct(characterized=exposure,
202 sourceCat=afwTable.SourceCatalog(),
203 backgroundModel=afwMath.BackgroundList(bg),
204 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10),
205 )
208class MockCalibrateTask(PipelineTask):
209 """A do-nothing substitute for CalibrateTask.
210 """
211 ConfigClass = CalibrateConfig
212 _DefaultName = "notCalibrate"
214 def __init__(self, butler=None, astromRefObjLoader=None,
215 photoRefObjLoader=None, icSourceSchema=None,
216 initInputs=None, **kwargs):
217 super().__init__(**kwargs)
218 self.outputSchema = afwTable.SourceCatalog()
220 def runQuantum(self, butlerQC, inputRefs, outputRefs):
221 inputs = butlerQC.get(inputRefs)
222 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
224 if self.config.doAstrometry:
225 inputs.pop('astromRefCat')
226 if self.config.doPhotoCal:
227 inputs.pop('photoRefCat')
229 outputs = self.run(**inputs)
231 if self.config.doWriteMatches and self.config.doAstrometry:
232 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
233 if self.config.doWriteMatchesDenormalized:
234 outputs.matchesDenormalized = outputs.astromMatches
235 outputs.matches = normalizedMatches
236 butlerQC.put(outputs, outputRefs)
238 def run(self, exposure, exposureIdInfo=None, background=None,
239 icSourceCat=None, idGenerator=None):
240 """Produce calibration outputs with no processing.
242 Parameters
243 ----------
244 exposure : `lsst.afw.image.Exposure`
245 Exposure to calibrate.
246 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
247 ID info for exposure. Deprecated in favor of ``idGenerator``, and
248 ignored if that is provided.
249 background : `lsst.afw.math.BackgroundList`, optional
250 Background model already subtracted from exposure.
251 icSourceCat : `lsst.afw.table.SourceCatalog`, optional
252 A SourceCatalog from CharacterizeImageTask from which we can copy some fields.
253 idGenerator : `lsst.meas.base.IdGenerator`, optional
254 Object that generates source IDs and provides random number
255 generator seeds.
257 Returns
258 -------
259 result : `lsst.pipe.base.Struct`
260 Struct containing these fields:
262 ``outputExposure``
263 Calibrated science exposure with refined WCS and PhotoCalib
264 (`lsst.afw.image.Exposure`).
265 ``outputBackground``
266 Model of background subtracted from exposure
267 (`lsst.afw.math.BackgroundList`).
268 ``outputCat``
269 Catalog of measured sources (`lsst.afw.table.SourceCatalog`).
270 ``astromMatches``
271 List of source/refObj matches from the astrometry solver
272 (`list` [`lsst.afw.table.ReferenceMatch`]).
273 """
274 # Can't persist empty BackgroundList; DM-33714
275 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
276 afwImage.MaskedImageF(16, 16))
277 return Struct(outputExposure=exposure,
278 outputBackground=afwMath.BackgroundList(bg),
279 outputCat=afwTable.SourceCatalog(),
280 astromMatches=[],
281 )
284class MockGetTemplateTask(PipelineTask):
285 """A do-nothing substitute for GetTemplateTask.
286 """
287 ConfigClass = GetTemplateConfig
288 _DefaultName = "notGetTemplate"
290 def runQuantum(self, butlerQC, inputRefs, outputRefs):
291 inputs = butlerQC.get(inputRefs)
292 # Mock GetTemplateTask.getOverlappingExposures
293 results = Struct(coaddExposures=[],
294 dataIds=[],
295 )
296 inputs["coaddExposures"] = results.coaddExposures
297 inputs["dataIds"] = results.dataIds
298 outputs = self.run(**inputs)
299 butlerQC.put(outputs, outputRefs)
301 def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs):
302 """Warp coadds from multiple tracts to form a template for image diff.
304 Where the tracts overlap, the resulting template image is averaged.
305 The PSF on the template is created by combining the CoaddPsf on each
306 template image into a meta-CoaddPsf.
308 Parameters
309 ----------
310 coaddExposures : `list` of `lsst.afw.image.Exposure`
311 Coadds to be mosaicked
312 bbox : `lsst.geom.Box2I`
313 Template Bounding box of the detector geometry onto which to
314 resample the coaddExposures
315 wcs : `lsst.afw.geom.SkyWcs`
316 Template WCS onto which to resample the coaddExposures
317 dataIds : `list` of `lsst.daf.butler.DataCoordinate`
318 Record of the tract and patch of each coaddExposure.
319 **kwargs
320 Any additional keyword parameters.
322 Returns
323 -------
324 result : `lsst.pipe.base.Struct` containing
325 - ``template`` : a template coadd exposure assembled out of patches
326 """
327 return Struct(template=afwImage.ExposureF(),
328 )
331class MockAlardLuptonSubtractTask(PipelineTask):
332 """A do-nothing substitute for AlardLuptonSubtractTask.
333 """
334 ConfigClass = AlardLuptonSubtractConfig
335 _DefaultName = "notAlardLuptonSubtract"
337 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None):
338 """PSF match, subtract, and decorrelate two images.
340 Parameters
341 ----------
342 template : `lsst.afw.image.ExposureF`
343 Template exposure, warped to match the science exposure.
344 science : `lsst.afw.image.ExposureF`
345 Science exposure to subtract from the template.
346 sources : `lsst.afw.table.SourceCatalog`
347 Identified sources on the science exposure. This catalog is used to
348 select sources in order to perform the AL PSF matching on stamp
349 images around them.
350 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
351 Exposure catalog with finalized psf models and aperture correction
352 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
353 the detector id for the catalog id, sorted on id for fast lookup.
355 Returns
356 -------
357 results : `lsst.pipe.base.Struct`
358 ``difference`` : `lsst.afw.image.ExposureF`
359 Result of subtracting template and science.
360 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
361 Warped and PSF-matched template exposure.
362 ``backgroundModel`` : `lsst.afw.math.Function2D`
363 Background model that was fit while solving for the PSF-matching kernel
364 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
365 Kernel used to PSF-match the convolved image.
366 """
367 return Struct(difference=afwImage.ExposureF(),
368 matchedTemplate=afwImage.ExposureF(),
369 backgroundModel=afwMath.NullFunction2D(),
370 psfMatchingKernel=afwMath.FixedKernel(),
371 )
374class MockDetectAndMeasureConfig(DetectAndMeasureConfig):
376 def setDefaults(self):
377 super().setDefaults()
378 # Avoid delegating to lsst.obs.base.Instrument specialization for the
379 # data ID packing algorithm to use, since test code often does not use a
380 # real Instrument in its data IDs.
381 self.idGenerator.packer.name = "observation"
384class MockDetectAndMeasureTask(PipelineTask):
385 """A do-nothing substitute for DetectAndMeasureTask.
386 """
387 ConfigClass = MockDetectAndMeasureConfig
388 _DefaultName = "notDetectAndMeasure"
390 def __init__(self, **kwargs):
391 super().__init__(**kwargs)
392 self.outputSchema = afwTable.SourceCatalog()
394 def runQuantum(self, butlerQC, inputRefs, outputRefs):
395 inputs = butlerQC.get(inputRefs)
396 idFactory = afwTable.IdFactory.makeSimple()
398 outputs = self.run(inputs['science'],
399 inputs['matchedTemplate'],
400 inputs['difference'],
401 idFactory=idFactory)
402 butlerQC.put(outputs, outputRefs)
404 def run(self, science, matchedTemplate, difference,
405 idFactory=None):
406 """Detect and measure sources on a difference image.
408 Parameters
409 ----------
410 science : `lsst.afw.image.ExposureF`
411 Science exposure that the template was subtracted from.
412 matchedTemplate : `lsst.afw.image.ExposureF`
413 Warped and PSF-matched template that was used produce the
414 difference image.
415 difference : `lsst.afw.image.ExposureF`
416 Result of subtracting template from the science image.
417 idFactory : `lsst.afw.table.IdFactory`, optional
418 Generator object to assign ids to detected sources in the difference image.
420 Returns
421 -------
422 results : `lsst.pipe.base.Struct`
423 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
424 Subtracted exposure with detection mask applied.
425 ``diaSources`` : `lsst.afw.table.SourceCatalog`
426 The catalog of detected sources.
427 """
428 return Struct(subtractedMeasuredExposure=difference,
429 diaSources=afwTable.SourceCatalog(),
430 )
433class MockTransformDiaSourceCatalogTask(PipelineTask):
434 """A do-nothing substitute for TransformDiaSourceCatalogTask.
435 """
436 ConfigClass = TransformDiaSourceCatalogConfig
437 _DefaultName = "notTransformDiaSourceCatalog"
439 def __init__(self, initInputs, **kwargs):
440 super().__init__(**kwargs)
442 def runQuantum(self, butlerQC, inputRefs, outputRefs):
443 inputs = butlerQC.get(inputRefs)
444 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
445 inputs["ccdVisitId"] = idGenerator.catalog_id
446 inputs["band"] = butlerQC.quantum.dataId["band"]
448 outputs = self.run(**inputs)
450 butlerQC.put(outputs, outputRefs)
452 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None):
453 """Produce transformation outputs with no processing.
455 Parameters
456 ----------
457 diaSourceCat : `lsst.afw.table.SourceCatalog`
458 The catalog to transform.
459 diffIm : `lsst.afw.image.Exposure`
460 An image, to provide supplementary information.
461 band : `str`
462 The band in which the sources were observed.
463 ccdVisitId : `int`
464 The exposure ID in which the sources were observed.
465 funcs, optional
466 Unused.
468 Returns
469 -------
470 results : `lsst.pipe.base.Struct`
471 Results struct with components:
473 ``diaSourceTable``
474 Catalog of DiaSources (`pandas.DataFrame`).
475 """
476 return Struct(diaSourceTable=pandas.DataFrame(),
477 )
480class MockDiaPipelineConfig(DiaPipelineConfig):
482 def setDefaults(self):
483 super().setDefaults()
484 # Avoid delegating to lsst.obs.base.Instrument specialization for the
485 # data ID packing algorithm to use, since test code often does not use a
486 # real Instrument in its data IDs.
487 self.idGenerator.packer.name = "observation"
490class MockDiaPipelineTask(PipelineTask):
491 """A do-nothing substitute for DiaPipelineTask.
492 """
493 ConfigClass = MockDiaPipelineConfig
494 _DefaultName = "notDiaPipe"
496 def runQuantum(self, butlerQC, inputRefs, outputRefs):
497 inputs = butlerQC.get(inputRefs)
498 inputs["idGenerator"] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
499 # Need to set ccdExposureIdBits (now deprecated) to None and pass it,
500 # since there are non-optional positional arguments after it.
501 inputs["ccdExposureIdBits"] = None
502 inputs["band"] = butlerQC.quantum.dataId["band"]
503 if not self.config.doSolarSystemAssociation:
504 inputs["solarSystemObjectTable"] = None
506 outputs = self.run(**inputs)
508 butlerQC.put(outputs, outputRefs)
510 def run(self,
511 diaSourceTable,
512 solarSystemObjectTable,
513 diffIm,
514 exposure,
515 template,
516 ccdExposureIdBits,
517 band,
518 idGenerator=None):
519 """Produce DiaSource and DiaObject outputs with no processing.
521 Parameters
522 ----------
523 diaSourceTable : `pandas.DataFrame`
524 Newly detected DiaSources.
525 solarSystemObjectTable : `pandas.DataFrame`
526 Expected solar system objects in the field of view.
527 diffIm : `lsst.afw.image.ExposureF`
528 Difference image exposure in which the sources in ``diaSourceCat``
529 were detected.
530 exposure : `lsst.afw.image.ExposureF`
531 Calibrated exposure differenced with a template to create
532 ``diffIm``.
533 template : `lsst.afw.image.ExposureF`
534 Template exposure used to create diffIm.
535 ccdExposureIdBits : `int`
536 Number of bits used for a unique ``ccdVisitId``. Deprecated in
537 favor of ``idGenerator``, and ignored if that is present. Pass
538 `None` explicitly to avoid a deprecation warning (a default is
539 impossible given that later positional arguments are not
540 defaulted).
541 band : `str`
542 The band in which the new DiaSources were detected.
543 idGenerator : `lsst.meas.base.IdGenerator`, optional
544 Object that generates source IDs and random number generator seeds.
545 Will be required after ``ccdExposureIdBits`` is removed.
547 Returns
548 -------
549 results : `lsst.pipe.base.Struct`
550 Results struct with components:
552 ``apdbMarker``
553 Marker dataset to store in the Butler indicating that this
554 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`).
555 ``associatedDiaSources``
556 Catalog of newly associated DiaSources (`pandas.DataFrame`).
557 """
558 return Struct(apdbMarker=self.config.apdb.value,
559 associatedDiaSources=pandas.DataFrame(),
560 diaForcedSources=pandas.DataFrame(),
561 diaObjects=pandas.DataFrame(),
562 )