Coverage for python/lsst/ap/verify/testPipeline.py: 44%
122 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-26 08:11 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-26 08:11 +0000
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
25# These classes exist only to be included in a mock pipeline, and don't need
26# to be public for that.
27__all__ = []
30import numpy as np
31import pandas
33import lsst.geom as geom
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.afw.table as afwTable
37from lsst.pipe.base import PipelineTask, Struct
38from lsst.ip.isr import IsrTaskConfig
39from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig
40from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
41from lsst.pipe.tasks.calibrate import CalibrateConfig
42from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig
45class MockIsrTask(PipelineTask):
46 """A do-nothing substitute for IsrTask.
47 """
48 ConfigClass = IsrTaskConfig
49 _DefaultName = "notIsr"
51 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
52 crosstalk=None, crosstalkSources=None,
53 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
54 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
55 sensorTransmission=None, atmosphereTransmission=None,
56 detectorNum=None, strayLightData=None, illumMaskedImage=None,
57 deferredCharge=None,
58 ):
59 """Accept ISR inputs, and produce ISR outputs with no processing.
61 Parameters
62 ----------
63 ccdExposure : `lsst.afw.image.Exposure`
64 The raw exposure that is to be run through ISR. The
65 exposure is modified by this method.
66 camera : `lsst.afw.cameraGeom.Camera`, optional
67 The camera geometry for this exposure. Required if
68 one or more of ``ccdExposure``, ``bias``, ``dark``, or
69 ``flat`` does not have an associated detector.
70 bias : `lsst.afw.image.Exposure`, optional
71 Bias calibration frame.
72 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
73 Functor for linearization.
74 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
75 Calibration for crosstalk.
76 crosstalkSources : `list`, optional
77 List of possible crosstalk sources.
78 dark : `lsst.afw.image.Exposure`, optional
79 Dark calibration frame.
80 flat : `lsst.afw.image.Exposure`, optional
81 Flat calibration frame.
82 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
83 Photon transfer curve dataset, with, e.g., gains
84 and read noise.
85 bfKernel : `numpy.ndarray`, optional
86 Brighter-fatter kernel.
87 bfGains : `dict` of `float`, optional
88 Gains used to override the detector's nominal gains for the
89 brighter-fatter correction. A dict keyed by amplifier name for
90 the detector in question.
91 defects : `lsst.ip.isr.Defects`, optional
92 List of defects.
93 fringes : `lsst.pipe.base.Struct`, optional
94 Struct containing the fringe correction data, with
95 elements:
96 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
97 - ``seed``: random seed derived from the ccdExposureId for random
98 number generator (`uint32`)
99 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
100 A ``TransmissionCurve`` that represents the throughput of the
101 optics, to be evaluated in focal-plane coordinates.
102 filterTransmission : `lsst.afw.image.TransmissionCurve`
103 A ``TransmissionCurve`` that represents the throughput of the
104 filter itself, to be evaluated in focal-plane coordinates.
105 sensorTransmission : `lsst.afw.image.TransmissionCurve`
106 A ``TransmissionCurve`` that represents the throughput of the
107 sensor itself, to be evaluated in post-assembly trimmed detector
108 coordinates.
109 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
110 A ``TransmissionCurve`` that represents the throughput of the
111 atmosphere, assumed to be spatially constant.
112 detectorNum : `int`, optional
113 The integer number for the detector to process.
114 isGen3 : bool, optional
115 Flag this call to run() as using the Gen3 butler environment.
116 strayLightData : `object`, optional
117 Opaque object containing calibration information for stray-light
118 correction. If `None`, no correction will be performed.
119 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
120 Illumination correction image.
122 Returns
123 -------
124 result : `lsst.pipe.base.Struct`
125 Result struct with components:
127 ``exposure``
128 The fully ISR corrected exposure (`afw.image.Exposure`).
129 ``outputExposure``
130 An alias for ``exposure`` (`afw.image.Exposure`).
131 ``ossThumb``
132 Thumbnail image of the exposure after overscan subtraction
133 (`numpy.ndarray`).
134 ``flattenedThumb``
135 Thumbnail image of the exposure after flat-field correction
136 (`numpy.ndarray`).
137 - ``outputStatistics`` : mapping [`str`]
138 Values of the additional statistics calculated.
139 """
140 return Struct(exposure=afwImage.ExposureF(),
141 outputExposure=afwImage.ExposureF(),
142 ossThumb=np.empty((1, 1)),
143 flattenedThumb=np.empty((1, 1)),
144 preInterpExposure=afwImage.ExposureF(),
145 outputOssThumbnail=np.empty((1, 1)),
146 outputFlattenedThumbnail=np.empty((1, 1)),
147 outputStatistics={},
148 )
151class MockCharacterizeImageTask(PipelineTask):
152 """A do-nothing substitute for CharacterizeImageTask.
153 """
154 ConfigClass = CharacterizeImageConfig
155 _DefaultName = "notCharacterizeImage"
157 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
158 super().__init__(**kwargs)
159 self.outputSchema = afwTable.SourceCatalog()
161 def runQuantum(self, butlerQC, inputRefs, outputRefs):
162 inputs = butlerQC.get(inputRefs)
163 if 'idGenerator' not in inputs.keys():
164 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
165 outputs = self.run(**inputs)
166 butlerQC.put(outputs, outputRefs)
168 def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None):
169 """Produce characterization outputs with no processing.
171 Parameters
172 ----------
173 exposure : `lsst.afw.image.Exposure`
174 Exposure to characterize.
175 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
176 ID info for exposure. Deprecated in favor of ``idGenerator``, and
177 ignored if that is provided.
178 background : `lsst.afw.math.BackgroundList`, optional
179 Initial model of background already subtracted from exposure.
180 idGenerator : `lsst.meas.base.IdGenerator`, optional
181 Object that generates source IDs and provides random number
182 generator seeds.
184 Returns
185 -------
186 result : `lsst.pipe.base.Struct`
187 Struct containing these fields:
189 ``characterized``
190 Characterized exposure (`lsst.afw.image.Exposure`).
191 ``sourceCat``
192 Detected sources (`lsst.afw.table.SourceCatalog`).
193 ``backgroundModel``
194 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`)
195 ``psfCellSet``
196 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`)
197 """
198 # Can't persist empty BackgroundList; DM-33714
199 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
200 afwImage.MaskedImageF(16, 16))
201 return Struct(characterized=exposure,
202 sourceCat=afwTable.SourceCatalog(),
203 backgroundModel=afwMath.BackgroundList(bg),
204 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10),
205 )
208class MockCalibrateTask(PipelineTask):
209 """A do-nothing substitute for CalibrateTask.
210 """
211 ConfigClass = CalibrateConfig
212 _DefaultName = "notCalibrate"
214 def __init__(self, butler=None, astromRefObjLoader=None,
215 photoRefObjLoader=None, icSourceSchema=None,
216 initInputs=None, **kwargs):
217 super().__init__(**kwargs)
218 self.outputSchema = afwTable.SourceCatalog()
220 def runQuantum(self, butlerQC, inputRefs, outputRefs):
221 inputs = butlerQC.get(inputRefs)
222 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
224 if self.config.doAstrometry:
225 inputs.pop('astromRefCat')
226 if self.config.doPhotoCal:
227 inputs.pop('photoRefCat')
229 outputs = self.run(**inputs)
231 if self.config.doWriteMatches and self.config.doAstrometry:
232 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
233 if self.config.doWriteMatchesDenormalized:
234 outputs.matchesDenormalized = outputs.astromMatches
235 outputs.matches = normalizedMatches
236 butlerQC.put(outputs, outputRefs)
238 def run(self, exposure, exposureIdInfo=None, background=None,
239 icSourceCat=None, idGenerator=None):
240 """Produce calibration outputs with no processing.
242 Parameters
243 ----------
244 exposure : `lsst.afw.image.Exposure`
245 Exposure to calibrate.
246 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
247 ID info for exposure. Deprecated in favor of ``idGenerator``, and
248 ignored if that is provided.
249 background : `lsst.afw.math.BackgroundList`, optional
250 Background model already subtracted from exposure.
251 icSourceCat : `lsst.afw.table.SourceCatalog`, optional
252 A SourceCatalog from CharacterizeImageTask from which we can copy some fields.
253 idGenerator : `lsst.meas.base.IdGenerator`, optional
254 Object that generates source IDs and provides random number
255 generator seeds.
257 Returns
258 -------
259 result : `lsst.pipe.base.Struct`
260 Struct containing these fields:
262 ``outputExposure``
263 Calibrated science exposure with refined WCS and PhotoCalib
264 (`lsst.afw.image.Exposure`).
265 ``outputBackground``
266 Model of background subtracted from exposure
267 (`lsst.afw.math.BackgroundList`).
268 ``outputCat``
269 Catalog of measured sources (`lsst.afw.table.SourceCatalog`).
270 ``astromMatches``
271 List of source/refObj matches from the astrometry solver
272 (`list` [`lsst.afw.table.ReferenceMatch`]).
273 """
274 # Can't persist empty BackgroundList; DM-33714
275 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
276 afwImage.MaskedImageF(16, 16))
277 return Struct(outputExposure=exposure,
278 outputBackground=afwMath.BackgroundList(bg),
279 outputCat=afwTable.SourceCatalog(),
280 astromMatches=[],
281 )
284class MockGetTemplateTask(PipelineTask):
285 """A do-nothing substitute for GetTemplateTask.
286 """
287 ConfigClass = GetTemplateConfig
288 _DefaultName = "notGetTemplate"
290 def runQuantum(self, butlerQC, inputRefs, outputRefs):
291 inputs = butlerQC.get(inputRefs)
292 # Mock GetTemplateTask.getOverlappingExposures
293 results = Struct(coaddExposures=[],
294 dataIds=[],
295 )
296 inputs["coaddExposures"] = results.coaddExposures
297 inputs["dataIds"] = results.dataIds
298 outputs = self.run(**inputs)
299 butlerQC.put(outputs, outputRefs)
301 def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs):
302 """Warp coadds from multiple tracts to form a template for image diff.
304 Where the tracts overlap, the resulting template image is averaged.
305 The PSF on the template is created by combining the CoaddPsf on each
306 template image into a meta-CoaddPsf.
308 Parameters
309 ----------
310 coaddExposures : `list` of `lsst.afw.image.Exposure`
311 Coadds to be mosaicked
312 bbox : `lsst.geom.Box2I`
313 Template Bounding box of the detector geometry onto which to
314 resample the coaddExposures
315 wcs : `lsst.afw.geom.SkyWcs`
316 Template WCS onto which to resample the coaddExposures
317 dataIds : `list` of `lsst.daf.butler.DataCoordinate`
318 Record of the tract and patch of each coaddExposure.
319 **kwargs
320 Any additional keyword parameters.
322 Returns
323 -------
324 result : `lsst.pipe.base.Struct` containing
325 - ``template`` : a template coadd exposure assembled out of patches
326 """
327 return Struct(template=afwImage.ExposureF(),
328 )
331class MockAlardLuptonSubtractTask(PipelineTask):
332 """A do-nothing substitute for AlardLuptonSubtractTask.
333 """
334 ConfigClass = AlardLuptonSubtractConfig
335 _DefaultName = "notAlardLuptonSubtract"
337 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None):
338 """PSF match, subtract, and decorrelate two images.
340 Parameters
341 ----------
342 template : `lsst.afw.image.ExposureF`
343 Template exposure, warped to match the science exposure.
344 science : `lsst.afw.image.ExposureF`
345 Science exposure to subtract from the template.
346 sources : `lsst.afw.table.SourceCatalog`
347 Identified sources on the science exposure. This catalog is used to
348 select sources in order to perform the AL PSF matching on stamp
349 images around them.
350 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
351 Exposure catalog with finalized psf models and aperture correction
352 maps to be applied if config.doApplyFinalizedPsf=True. Catalog
353 uses the detector id for the catalog id, sorted on id for fast
354 lookup. Deprecated in favor of ``visitSummary``, and will be
355 removed after v26.
356 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
357 Exposure catalog with external calibrations to be applied. Catalog
358 uses the detector id for the catalog id, sorted on id for fast
359 lookup. Ignored (for temporary backwards compatibility) if
360 ``finalizedPsfApCorrCatalog`` is provided.
362 Returns
363 -------
364 results : `lsst.pipe.base.Struct`
365 ``difference`` : `lsst.afw.image.ExposureF`
366 Result of subtracting template and science.
367 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
368 Warped and PSF-matched template exposure.
369 ``backgroundModel`` : `lsst.afw.math.Function2D`
370 Background model that was fit while solving for the
371 PSF-matching kernel
372 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
373 Kernel used to PSF-match the convolved image.
374 """
375 return Struct(difference=afwImage.ExposureF(),
376 matchedTemplate=afwImage.ExposureF(),
377 backgroundModel=afwMath.NullFunction2D(),
378 psfMatchingKernel=afwMath.FixedKernel(),
379 )
382class MockDetectAndMeasureConfig(DetectAndMeasureConfig):
384 def setDefaults(self):
385 super().setDefaults()
386 # Avoid delegating to lsst.obs.base.Instrument specialization for the
387 # data ID packing algorithm to use, since test code often does not use a
388 # real Instrument in its data IDs.
389 self.idGenerator.packer.name = "observation"
392class MockDetectAndMeasureTask(PipelineTask):
393 """A do-nothing substitute for DetectAndMeasureTask.
394 """
395 ConfigClass = MockDetectAndMeasureConfig
396 _DefaultName = "notDetectAndMeasure"
398 def __init__(self, **kwargs):
399 super().__init__(**kwargs)
400 self.outputSchema = afwTable.SourceCatalog()
402 def runQuantum(self, butlerQC, inputRefs, outputRefs):
403 inputs = butlerQC.get(inputRefs)
404 idFactory = afwTable.IdFactory.makeSimple()
406 outputs = self.run(inputs['science'],
407 inputs['matchedTemplate'],
408 inputs['difference'],
409 idFactory=idFactory)
410 butlerQC.put(outputs, outputRefs)
412 def run(self, science, matchedTemplate, difference,
413 idFactory=None):
414 """Detect and measure sources on a difference image.
416 Parameters
417 ----------
418 science : `lsst.afw.image.ExposureF`
419 Science exposure that the template was subtracted from.
420 matchedTemplate : `lsst.afw.image.ExposureF`
421 Warped and PSF-matched template that was used produce the
422 difference image.
423 difference : `lsst.afw.image.ExposureF`
424 Result of subtracting template from the science image.
425 idFactory : `lsst.afw.table.IdFactory`, optional
426 Generator object to assign ids to detected sources in the difference image.
428 Returns
429 -------
430 results : `lsst.pipe.base.Struct`
431 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
432 Subtracted exposure with detection mask applied.
433 ``diaSources`` : `lsst.afw.table.SourceCatalog`
434 The catalog of detected sources.
435 """
436 return Struct(subtractedMeasuredExposure=difference,
437 diaSources=afwTable.SourceCatalog(),
438 )
441class MockTransformDiaSourceCatalogTask(PipelineTask):
442 """A do-nothing substitute for TransformDiaSourceCatalogTask.
443 """
444 ConfigClass = TransformDiaSourceCatalogConfig
445 _DefaultName = "notTransformDiaSourceCatalog"
447 def __init__(self, initInputs, **kwargs):
448 super().__init__(**kwargs)
450 def runQuantum(self, butlerQC, inputRefs, outputRefs):
451 inputs = butlerQC.get(inputRefs)
452 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
453 inputs["ccdVisitId"] = idGenerator.catalog_id
454 inputs["band"] = butlerQC.quantum.dataId["band"]
456 outputs = self.run(**inputs)
458 butlerQC.put(outputs, outputRefs)
460 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None):
461 """Produce transformation outputs with no processing.
463 Parameters
464 ----------
465 diaSourceCat : `lsst.afw.table.SourceCatalog`
466 The catalog to transform.
467 diffIm : `lsst.afw.image.Exposure`
468 An image, to provide supplementary information.
469 band : `str`
470 The band in which the sources were observed.
471 ccdVisitId : `int`
472 The exposure ID in which the sources were observed.
473 funcs, optional
474 Unused.
476 Returns
477 -------
478 results : `lsst.pipe.base.Struct`
479 Results struct with components:
481 ``diaSourceTable``
482 Catalog of DiaSources (`pandas.DataFrame`).
483 """
484 return Struct(diaSourceTable=pandas.DataFrame(),
485 )
488class MockDiaPipelineConfig(DiaPipelineConfig):
490 def setDefaults(self):
491 super().setDefaults()
492 # Avoid delegating to lsst.obs.base.Instrument specialization for the
493 # data ID packing algorithm to use, since test code often does not use a
494 # real Instrument in its data IDs.
495 self.idGenerator.packer.name = "observation"
498class MockDiaPipelineTask(PipelineTask):
499 """A do-nothing substitute for DiaPipelineTask.
500 """
501 ConfigClass = MockDiaPipelineConfig
502 _DefaultName = "notDiaPipe"
504 def runQuantum(self, butlerQC, inputRefs, outputRefs):
505 inputs = butlerQC.get(inputRefs)
506 inputs["idGenerator"] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
507 # Need to set ccdExposureIdBits (now deprecated) to None and pass it,
508 # since there are non-optional positional arguments after it.
509 inputs["ccdExposureIdBits"] = None
510 inputs["band"] = butlerQC.quantum.dataId["band"]
511 if not self.config.doSolarSystemAssociation:
512 inputs["solarSystemObjectTable"] = None
514 outputs = self.run(**inputs)
516 butlerQC.put(outputs, outputRefs)
518 def run(self,
519 diaSourceTable,
520 solarSystemObjectTable,
521 diffIm,
522 exposure,
523 template,
524 ccdExposureIdBits,
525 band,
526 idGenerator=None):
527 """Produce DiaSource and DiaObject outputs with no processing.
529 Parameters
530 ----------
531 diaSourceTable : `pandas.DataFrame`
532 Newly detected DiaSources.
533 solarSystemObjectTable : `pandas.DataFrame`
534 Expected solar system objects in the field of view.
535 diffIm : `lsst.afw.image.ExposureF`
536 Difference image exposure in which the sources in ``diaSourceCat``
537 were detected.
538 exposure : `lsst.afw.image.ExposureF`
539 Calibrated exposure differenced with a template to create
540 ``diffIm``.
541 template : `lsst.afw.image.ExposureF`
542 Template exposure used to create diffIm.
543 ccdExposureIdBits : `int`
544 Number of bits used for a unique ``ccdVisitId``. Deprecated in
545 favor of ``idGenerator``, and ignored if that is present. Pass
546 `None` explicitly to avoid a deprecation warning (a default is
547 impossible given that later positional arguments are not
548 defaulted).
549 band : `str`
550 The band in which the new DiaSources were detected.
551 idGenerator : `lsst.meas.base.IdGenerator`, optional
552 Object that generates source IDs and random number generator seeds.
553 Will be required after ``ccdExposureIdBits`` is removed.
555 Returns
556 -------
557 results : `lsst.pipe.base.Struct`
558 Results struct with components:
560 ``apdbMarker``
561 Marker dataset to store in the Butler indicating that this
562 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`).
563 ``associatedDiaSources``
564 Catalog of newly associated DiaSources (`pandas.DataFrame`).
565 """
566 return Struct(apdbMarker=self.config.apdb.value,
567 associatedDiaSources=pandas.DataFrame(),
568 diaForcedSources=pandas.DataFrame(),
569 diaObjects=pandas.DataFrame(),
570 )