Coverage for python/lsst/ap/verify/testPipeline.py: 44%
122 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-18 12:21 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-18 12:21 +0000
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
25# These classes exist only to be included in a mock pipeline, and don't need
26# to be public for that.
27__all__ = []
30import numpy as np
31import pandas
33import lsst.geom as geom
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.afw.table as afwTable
37from lsst.pipe.base import PipelineTask, Struct
38from lsst.ip.isr import IsrTaskConfig
39from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig
40from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
41from lsst.pipe.tasks.calibrate import CalibrateConfig
42from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig
45class MockIsrTask(PipelineTask):
46 """A do-nothing substitute for IsrTask.
47 """
48 ConfigClass = IsrTaskConfig
49 _DefaultName = "notIsr"
51 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
52 crosstalk=None, crosstalkSources=None,
53 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
54 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
55 sensorTransmission=None, atmosphereTransmission=None,
56 detectorNum=None, strayLightData=None, illumMaskedImage=None,
57 deferredCharge=None,
58 ):
59 """Accept ISR inputs, and produce ISR outputs with no processing.
61 Parameters
62 ----------
63 ccdExposure : `lsst.afw.image.Exposure`
64 The raw exposure that is to be run through ISR. The
65 exposure is modified by this method.
66 camera : `lsst.afw.cameraGeom.Camera`, optional
67 The camera geometry for this exposure. Required if
68 one or more of ``ccdExposure``, ``bias``, ``dark``, or
69 ``flat`` does not have an associated detector.
70 bias : `lsst.afw.image.Exposure`, optional
71 Bias calibration frame.
72 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
73 Functor for linearization.
74 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
75 Calibration for crosstalk.
76 crosstalkSources : `list`, optional
77 List of possible crosstalk sources.
78 dark : `lsst.afw.image.Exposure`, optional
79 Dark calibration frame.
80 flat : `lsst.afw.image.Exposure`, optional
81 Flat calibration frame.
82 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
83 Photon transfer curve dataset, with, e.g., gains
84 and read noise.
85 bfKernel : `numpy.ndarray`, optional
86 Brighter-fatter kernel.
87 bfGains : `dict` of `float`, optional
88 Gains used to override the detector's nominal gains for the
89 brighter-fatter correction. A dict keyed by amplifier name for
90 the detector in question.
91 defects : `lsst.ip.isr.Defects`, optional
92 List of defects.
93 fringes : `lsst.pipe.base.Struct`, optional
94 Struct containing the fringe correction data, with
95 elements:
96 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
97 - ``seed``: random seed derived from the ccdExposureId for random
98 number generator (`uint32`)
99 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
100 A ``TransmissionCurve`` that represents the throughput of the
101 optics, to be evaluated in focal-plane coordinates.
102 filterTransmission : `lsst.afw.image.TransmissionCurve`
103 A ``TransmissionCurve`` that represents the throughput of the
104 filter itself, to be evaluated in focal-plane coordinates.
105 sensorTransmission : `lsst.afw.image.TransmissionCurve`
106 A ``TransmissionCurve`` that represents the throughput of the
107 sensor itself, to be evaluated in post-assembly trimmed detector
108 coordinates.
109 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
110 A ``TransmissionCurve`` that represents the throughput of the
111 atmosphere, assumed to be spatially constant.
112 detectorNum : `int`, optional
113 The integer number for the detector to process.
114 isGen3 : bool, optional
115 Flag this call to run() as using the Gen3 butler environment.
116 strayLightData : `object`, optional
117 Opaque object containing calibration information for stray-light
118 correction. If `None`, no correction will be performed.
119 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
120 Illumination correction image.
122 Returns
123 -------
124 result : `lsst.pipe.base.Struct`
125 Result struct with components:
127 ``exposure``
128 The fully ISR corrected exposure (`afw.image.Exposure`).
129 ``outputExposure``
130 An alias for ``exposure`` (`afw.image.Exposure`).
131 ``ossThumb``
132 Thumbnail image of the exposure after overscan subtraction
133 (`numpy.ndarray`).
134 ``flattenedThumb``
135 Thumbnail image of the exposure after flat-field correction
136 (`numpy.ndarray`).
137 - ``outputStatistics`` : mapping [`str`]
138 Values of the additional statistics calculated.
139 """
140 return Struct(exposure=afwImage.ExposureF(),
141 outputExposure=afwImage.ExposureF(),
142 ossThumb=np.empty((1, 1)),
143 flattenedThumb=np.empty((1, 1)),
144 preInterpExposure=afwImage.ExposureF(),
145 outputOssThumbnail=np.empty((1, 1)),
146 outputFlattenedThumbnail=np.empty((1, 1)),
147 outputStatistics={},
148 )
151class MockCharacterizeImageTask(PipelineTask):
152 """A do-nothing substitute for CharacterizeImageTask.
153 """
154 ConfigClass = CharacterizeImageConfig
155 _DefaultName = "notCharacterizeImage"
157 def __init__(self, refObjLoader=None, schema=None, **kwargs):
158 super().__init__(**kwargs)
159 self.outputSchema = afwTable.SourceCatalog()
161 def runQuantum(self, butlerQC, inputRefs, outputRefs):
162 inputs = butlerQC.get(inputRefs)
163 if 'idGenerator' not in inputs.keys():
164 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
165 outputs = self.run(**inputs)
166 butlerQC.put(outputs, outputRefs)
168 def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None):
169 """Produce characterization outputs with no processing.
171 Parameters
172 ----------
173 exposure : `lsst.afw.image.Exposure`
174 Exposure to characterize.
175 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
176 ID info for exposure. Deprecated in favor of ``idGenerator``, and
177 ignored if that is provided.
178 background : `lsst.afw.math.BackgroundList`, optional
179 Initial model of background already subtracted from exposure.
180 idGenerator : `lsst.meas.base.IdGenerator`, optional
181 Object that generates source IDs and provides random number
182 generator seeds.
184 Returns
185 -------
186 result : `lsst.pipe.base.Struct`
187 Struct containing these fields:
189 ``characterized``
190 Characterized exposure (`lsst.afw.image.Exposure`).
191 ``sourceCat``
192 Detected sources (`lsst.afw.table.SourceCatalog`).
193 ``backgroundModel``
194 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`)
195 ``psfCellSet``
196 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`)
197 """
198 # Can't persist empty BackgroundList; DM-33714
199 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
200 afwImage.MaskedImageF(16, 16))
201 return Struct(characterized=exposure,
202 sourceCat=afwTable.SourceCatalog(),
203 backgroundModel=afwMath.BackgroundList(bg),
204 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10),
205 )
208class MockCalibrateTask(PipelineTask):
209 """A do-nothing substitute for CalibrateTask.
210 """
211 ConfigClass = CalibrateConfig
212 _DefaultName = "notCalibrate"
214 def __init__(self, astromRefObjLoader=None,
215 photoRefObjLoader=None, icSourceSchema=None,
216 initInputs=None, **kwargs):
217 super().__init__(**kwargs)
218 self.outputSchema = afwTable.SourceCatalog()
220 def runQuantum(self, butlerQC, inputRefs, outputRefs):
221 inputs = butlerQC.get(inputRefs)
222 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
224 if self.config.doAstrometry:
225 inputs.pop('astromRefCat')
226 if self.config.doPhotoCal:
227 inputs.pop('photoRefCat')
229 outputs = self.run(**inputs)
231 if self.config.doWriteMatches and self.config.doAstrometry:
232 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
233 if self.config.doWriteMatchesDenormalized:
234 # Just need an empty BaseCatalog with a valid schema.
235 outputs.matchesDenormalized = afwTable.BaseCatalog(outputs.outputCat.schema)
236 outputs.matches = normalizedMatches
237 butlerQC.put(outputs, outputRefs)
239 def run(self, exposure, exposureIdInfo=None, background=None,
240 icSourceCat=None, idGenerator=None):
241 """Produce calibration outputs with no processing.
243 Parameters
244 ----------
245 exposure : `lsst.afw.image.Exposure`
246 Exposure to calibrate.
247 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
248 ID info for exposure. Deprecated in favor of ``idGenerator``, and
249 ignored if that is provided.
250 background : `lsst.afw.math.BackgroundList`, optional
251 Background model already subtracted from exposure.
252 icSourceCat : `lsst.afw.table.SourceCatalog`, optional
253 A SourceCatalog from CharacterizeImageTask from which we can copy some fields.
254 idGenerator : `lsst.meas.base.IdGenerator`, optional
255 Object that generates source IDs and provides random number
256 generator seeds.
258 Returns
259 -------
260 result : `lsst.pipe.base.Struct`
261 Struct containing these fields:
263 ``outputExposure``
264 Calibrated science exposure with refined WCS and PhotoCalib
265 (`lsst.afw.image.Exposure`).
266 ``outputBackground``
267 Model of background subtracted from exposure
268 (`lsst.afw.math.BackgroundList`).
269 ``outputCat``
270 Catalog of measured sources (`lsst.afw.table.SourceCatalog`).
271 ``astromMatches``
272 List of source/refObj matches from the astrometry solver
273 (`list` [`lsst.afw.table.ReferenceMatch`]).
274 """
275 # Can't persist empty BackgroundList; DM-33714
276 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
277 afwImage.MaskedImageF(16, 16))
278 return Struct(outputExposure=exposure,
279 outputBackground=afwMath.BackgroundList(bg),
280 outputCat=afwTable.SourceCatalog(),
281 astromMatches=[],
282 )
285class MockGetTemplateTask(PipelineTask):
286 """A do-nothing substitute for GetTemplateTask.
287 """
288 ConfigClass = GetTemplateConfig
289 _DefaultName = "notGetTemplate"
291 def runQuantum(self, butlerQC, inputRefs, outputRefs):
292 inputs = butlerQC.get(inputRefs)
293 # Mock GetTemplateTask.getOverlappingExposures
294 results = Struct(coaddExposures=[],
295 dataIds=[],
296 )
297 inputs["coaddExposures"] = results.coaddExposures
298 inputs["dataIds"] = results.dataIds
299 outputs = self.run(**inputs)
300 butlerQC.put(outputs, outputRefs)
302 def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs):
303 """Warp coadds from multiple tracts to form a template for image diff.
305 Where the tracts overlap, the resulting template image is averaged.
306 The PSF on the template is created by combining the CoaddPsf on each
307 template image into a meta-CoaddPsf.
309 Parameters
310 ----------
311 coaddExposures : `list` of `lsst.afw.image.Exposure`
312 Coadds to be mosaicked
313 bbox : `lsst.geom.Box2I`
314 Template Bounding box of the detector geometry onto which to
315 resample the coaddExposures
316 wcs : `lsst.afw.geom.SkyWcs`
317 Template WCS onto which to resample the coaddExposures
318 dataIds : `list` of `lsst.daf.butler.DataCoordinate`
319 Record of the tract and patch of each coaddExposure.
320 **kwargs
321 Any additional keyword parameters.
323 Returns
324 -------
325 result : `lsst.pipe.base.Struct` containing
326 - ``template`` : a template coadd exposure assembled out of patches
327 """
328 return Struct(template=afwImage.ExposureF(),
329 )
332class MockAlardLuptonSubtractTask(PipelineTask):
333 """A do-nothing substitute for AlardLuptonSubtractTask.
334 """
335 ConfigClass = AlardLuptonSubtractConfig
336 _DefaultName = "notAlardLuptonSubtract"
338 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None):
339 """PSF match, subtract, and decorrelate two images.
341 Parameters
342 ----------
343 template : `lsst.afw.image.ExposureF`
344 Template exposure, warped to match the science exposure.
345 science : `lsst.afw.image.ExposureF`
346 Science exposure to subtract from the template.
347 sources : `lsst.afw.table.SourceCatalog`
348 Identified sources on the science exposure. This catalog is used to
349 select sources in order to perform the AL PSF matching on stamp
350 images around them.
351 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
352 Exposure catalog with finalized psf models and aperture correction
353 maps to be applied if config.doApplyFinalizedPsf=True. Catalog
354 uses the detector id for the catalog id, sorted on id for fast
355 lookup. Deprecated in favor of ``visitSummary``, and will be
356 removed after v26.
357 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
358 Exposure catalog with external calibrations to be applied. Catalog
359 uses the detector id for the catalog id, sorted on id for fast
360 lookup. Ignored (for temporary backwards compatibility) if
361 ``finalizedPsfApCorrCatalog`` is provided.
363 Returns
364 -------
365 results : `lsst.pipe.base.Struct`
366 ``difference`` : `lsst.afw.image.ExposureF`
367 Result of subtracting template and science.
368 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
369 Warped and PSF-matched template exposure.
370 ``backgroundModel`` : `lsst.afw.math.Function2D`
371 Background model that was fit while solving for the
372 PSF-matching kernel
373 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
374 Kernel used to PSF-match the convolved image.
375 """
376 return Struct(difference=afwImage.ExposureF(),
377 matchedTemplate=afwImage.ExposureF(),
378 backgroundModel=afwMath.NullFunction2D(),
379 psfMatchingKernel=afwMath.FixedKernel(),
380 )
383class MockDetectAndMeasureConfig(DetectAndMeasureConfig):
385 def setDefaults(self):
386 super().setDefaults()
387 # Avoid delegating to lsst.obs.base.Instrument specialization for the
388 # data ID packing algorithm to use, since test code often does not use a
389 # real Instrument in its data IDs.
390 self.idGenerator.packer.name = "observation"
393class MockDetectAndMeasureTask(PipelineTask):
394 """A do-nothing substitute for DetectAndMeasureTask.
395 """
396 ConfigClass = MockDetectAndMeasureConfig
397 _DefaultName = "notDetectAndMeasure"
399 def __init__(self, **kwargs):
400 super().__init__(**kwargs)
401 self.outputSchema = afwTable.SourceCatalog()
403 def runQuantum(self, butlerQC, inputRefs, outputRefs):
404 inputs = butlerQC.get(inputRefs)
405 idFactory = afwTable.IdFactory.makeSimple()
407 outputs = self.run(inputs['science'],
408 inputs['matchedTemplate'],
409 inputs['difference'],
410 idFactory=idFactory)
411 butlerQC.put(outputs, outputRefs)
413 def run(self, science, matchedTemplate, difference,
414 idFactory=None):
415 """Detect and measure sources on a difference image.
417 Parameters
418 ----------
419 science : `lsst.afw.image.ExposureF`
420 Science exposure that the template was subtracted from.
421 matchedTemplate : `lsst.afw.image.ExposureF`
422 Warped and PSF-matched template that was used produce the
423 difference image.
424 difference : `lsst.afw.image.ExposureF`
425 Result of subtracting template from the science image.
426 idFactory : `lsst.afw.table.IdFactory`, optional
427 Generator object to assign ids to detected sources in the difference image.
429 Returns
430 -------
431 results : `lsst.pipe.base.Struct`
432 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
433 Subtracted exposure with detection mask applied.
434 ``diaSources`` : `lsst.afw.table.SourceCatalog`
435 The catalog of detected sources.
436 """
437 return Struct(subtractedMeasuredExposure=difference,
438 diaSources=afwTable.SourceCatalog(),
439 )
442class MockTransformDiaSourceCatalogTask(PipelineTask):
443 """A do-nothing substitute for TransformDiaSourceCatalogTask.
444 """
445 ConfigClass = TransformDiaSourceCatalogConfig
446 _DefaultName = "notTransformDiaSourceCatalog"
448 def __init__(self, initInputs, **kwargs):
449 super().__init__(**kwargs)
451 def runQuantum(self, butlerQC, inputRefs, outputRefs):
452 inputs = butlerQC.get(inputRefs)
453 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
454 inputs["ccdVisitId"] = idGenerator.catalog_id
455 inputs["band"] = butlerQC.quantum.dataId["band"]
457 outputs = self.run(**inputs)
459 butlerQC.put(outputs, outputRefs)
461 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None):
462 """Produce transformation outputs with no processing.
464 Parameters
465 ----------
466 diaSourceCat : `lsst.afw.table.SourceCatalog`
467 The catalog to transform.
468 diffIm : `lsst.afw.image.Exposure`
469 An image, to provide supplementary information.
470 band : `str`
471 The band in which the sources were observed.
472 ccdVisitId : `int`
473 The exposure ID in which the sources were observed.
474 funcs, optional
475 Unused.
477 Returns
478 -------
479 results : `lsst.pipe.base.Struct`
480 Results struct with components:
482 ``diaSourceTable``
483 Catalog of DiaSources (`pandas.DataFrame`).
484 """
485 return Struct(diaSourceTable=pandas.DataFrame(),
486 )
489class MockDiaPipelineConfig(DiaPipelineConfig):
491 def setDefaults(self):
492 super().setDefaults()
493 # Avoid delegating to lsst.obs.base.Instrument specialization for the
494 # data ID packing algorithm to use, since test code often does not use a
495 # real Instrument in its data IDs.
496 self.idGenerator.packer.name = "observation"
499class MockDiaPipelineTask(PipelineTask):
500 """A do-nothing substitute for DiaPipelineTask.
501 """
502 ConfigClass = MockDiaPipelineConfig
503 _DefaultName = "notDiaPipe"
505 def runQuantum(self, butlerQC, inputRefs, outputRefs):
506 inputs = butlerQC.get(inputRefs)
507 inputs["idGenerator"] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
508 # Need to set ccdExposureIdBits (now deprecated) to None and pass it,
509 # since there are non-optional positional arguments after it.
510 inputs["ccdExposureIdBits"] = None
511 inputs["band"] = butlerQC.quantum.dataId["band"]
512 if not self.config.doSolarSystemAssociation:
513 inputs["solarSystemObjectTable"] = None
515 outputs = self.run(**inputs)
517 butlerQC.put(outputs, outputRefs)
519 def run(self,
520 diaSourceTable,
521 solarSystemObjectTable,
522 diffIm,
523 exposure,
524 template,
525 ccdExposureIdBits,
526 band,
527 idGenerator=None):
528 """Produce DiaSource and DiaObject outputs with no processing.
530 Parameters
531 ----------
532 diaSourceTable : `pandas.DataFrame`
533 Newly detected DiaSources.
534 solarSystemObjectTable : `pandas.DataFrame`
535 Expected solar system objects in the field of view.
536 diffIm : `lsst.afw.image.ExposureF`
537 Difference image exposure in which the sources in ``diaSourceCat``
538 were detected.
539 exposure : `lsst.afw.image.ExposureF`
540 Calibrated exposure differenced with a template to create
541 ``diffIm``.
542 template : `lsst.afw.image.ExposureF`
543 Template exposure used to create diffIm.
544 ccdExposureIdBits : `int`
545 Number of bits used for a unique ``ccdVisitId``. Deprecated in
546 favor of ``idGenerator``, and ignored if that is present. Pass
547 `None` explicitly to avoid a deprecation warning (a default is
548 impossible given that later positional arguments are not
549 defaulted).
550 band : `str`
551 The band in which the new DiaSources were detected.
552 idGenerator : `lsst.meas.base.IdGenerator`, optional
553 Object that generates source IDs and random number generator seeds.
554 Will be required after ``ccdExposureIdBits`` is removed.
556 Returns
557 -------
558 results : `lsst.pipe.base.Struct`
559 Results struct with components:
561 ``apdbMarker``
562 Marker dataset to store in the Butler indicating that this
563 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`).
564 ``associatedDiaSources``
565 Catalog of newly associated DiaSources (`pandas.DataFrame`).
566 """
567 return Struct(apdbMarker=self.config.apdb.value,
568 associatedDiaSources=pandas.DataFrame(),
569 diaForcedSources=pandas.DataFrame(),
570 diaObjects=pandas.DataFrame(),
571 )