Coverage for python/lsst/ap/verify/testPipeline.py: 47%
133 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-07 14:42 +0000
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-07 14:42 +0000
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
25# These classes exist only to be included in a mock pipeline, and don't need
26# to be public for that.
27__all__ = []
30import numpy as np
31import pandas
33import lsst.geom as geom
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.afw.table as afwTable
37from lsst.ap.association import (TransformDiaSourceCatalogConfig,
38 DiaPipelineConfig, FilterDiaSourceCatalogConfig)
39from lsst.pipe.base import PipelineTask, Struct
40from lsst.ip.isr import IsrTaskConfig
41from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig
42from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
43from lsst.pipe.tasks.calibrate import CalibrateConfig
44from lsst.meas.transiNet import RBTransiNetConfig
47class MockIsrTask(PipelineTask):
48 """A do-nothing substitute for IsrTask.
49 """
50 ConfigClass = IsrTaskConfig
51 _DefaultName = "notIsr"
53 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
54 crosstalk=None, crosstalkSources=None,
55 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
56 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
57 sensorTransmission=None, atmosphereTransmission=None,
58 detectorNum=None, strayLightData=None, illumMaskedImage=None,
59 deferredCharge=None,
60 ):
61 """Accept ISR inputs, and produce ISR outputs with no processing.
63 Parameters
64 ----------
65 ccdExposure : `lsst.afw.image.Exposure`
66 The raw exposure that is to be run through ISR. The
67 exposure is modified by this method.
68 camera : `lsst.afw.cameraGeom.Camera`, optional
69 The camera geometry for this exposure. Required if
70 one or more of ``ccdExposure``, ``bias``, ``dark``, or
71 ``flat`` does not have an associated detector.
72 bias : `lsst.afw.image.Exposure`, optional
73 Bias calibration frame.
74 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
75 Functor for linearization.
76 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
77 Calibration for crosstalk.
78 crosstalkSources : `list`, optional
79 List of possible crosstalk sources.
80 dark : `lsst.afw.image.Exposure`, optional
81 Dark calibration frame.
82 flat : `lsst.afw.image.Exposure`, optional
83 Flat calibration frame.
84 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
85 Photon transfer curve dataset, with, e.g., gains
86 and read noise.
87 bfKernel : `numpy.ndarray`, optional
88 Brighter-fatter kernel.
89 bfGains : `dict` of `float`, optional
90 Gains used to override the detector's nominal gains for the
91 brighter-fatter correction. A dict keyed by amplifier name for
92 the detector in question.
93 defects : `lsst.ip.isr.Defects`, optional
94 List of defects.
95 fringes : `lsst.pipe.base.Struct`, optional
96 Struct containing the fringe correction data, with
97 elements:
98 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
99 - ``seed``: random seed derived from the ccdExposureId for random
100 number generator (`uint32`)
101 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
102 A ``TransmissionCurve`` that represents the throughput of the
103 optics, to be evaluated in focal-plane coordinates.
104 filterTransmission : `lsst.afw.image.TransmissionCurve`
105 A ``TransmissionCurve`` that represents the throughput of the
106 filter itself, to be evaluated in focal-plane coordinates.
107 sensorTransmission : `lsst.afw.image.TransmissionCurve`
108 A ``TransmissionCurve`` that represents the throughput of the
109 sensor itself, to be evaluated in post-assembly trimmed detector
110 coordinates.
111 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
112 A ``TransmissionCurve`` that represents the throughput of the
113 atmosphere, assumed to be spatially constant.
114 detectorNum : `int`, optional
115 The integer number for the detector to process.
116 isGen3 : bool, optional
117 Flag this call to run() as using the Gen3 butler environment.
118 strayLightData : `object`, optional
119 Opaque object containing calibration information for stray-light
120 correction. If `None`, no correction will be performed.
121 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
122 Illumination correction image.
124 Returns
125 -------
126 result : `lsst.pipe.base.Struct`
127 Result struct with components:
129 ``exposure``
130 The fully ISR corrected exposure (`afw.image.Exposure`).
131 ``outputExposure``
132 An alias for ``exposure`` (`afw.image.Exposure`).
133 ``ossThumb``
134 Thumbnail image of the exposure after overscan subtraction
135 (`numpy.ndarray`).
136 ``flattenedThumb``
137 Thumbnail image of the exposure after flat-field correction
138 (`numpy.ndarray`).
139 - ``outputStatistics`` : mapping [`str`]
140 Values of the additional statistics calculated.
141 """
142 return Struct(exposure=afwImage.ExposureF(),
143 outputExposure=afwImage.ExposureF(),
144 ossThumb=np.empty((1, 1)),
145 flattenedThumb=np.empty((1, 1)),
146 preInterpExposure=afwImage.ExposureF(),
147 outputOssThumbnail=np.empty((1, 1)),
148 outputFlattenedThumbnail=np.empty((1, 1)),
149 outputStatistics={},
150 )
153class MockCharacterizeImageTask(PipelineTask):
154 """A do-nothing substitute for CharacterizeImageTask.
155 """
156 ConfigClass = CharacterizeImageConfig
157 _DefaultName = "notCharacterizeImage"
159 def __init__(self, refObjLoader=None, schema=None, **kwargs):
160 super().__init__(**kwargs)
161 self.outputSchema = afwTable.SourceCatalog()
163 def runQuantum(self, butlerQC, inputRefs, outputRefs):
164 inputs = butlerQC.get(inputRefs)
165 if 'idGenerator' not in inputs.keys():
166 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
167 outputs = self.run(**inputs)
168 butlerQC.put(outputs, outputRefs)
170 def run(self, exposure, background=None, idGenerator=None):
171 """Produce characterization outputs with no processing.
173 Parameters
174 ----------
175 exposure : `lsst.afw.image.Exposure`
176 Exposure to characterize.
177 background : `lsst.afw.math.BackgroundList`, optional
178 Initial model of background already subtracted from exposure.
179 idGenerator : `lsst.meas.base.IdGenerator`, optional
180 Object that generates source IDs and provides random number
181 generator seeds.
183 Returns
184 -------
185 result : `lsst.pipe.base.Struct`
186 Struct containing these fields:
188 ``characterized``
189 Characterized exposure (`lsst.afw.image.Exposure`).
190 ``sourceCat``
191 Detected sources (`lsst.afw.table.SourceCatalog`).
192 ``backgroundModel``
193 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`)
194 ``psfCellSet``
195 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`)
196 """
197 # Can't persist empty BackgroundList; DM-33714
198 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
199 afwImage.MaskedImageF(16, 16))
200 return Struct(characterized=exposure,
201 sourceCat=afwTable.SourceCatalog(),
202 backgroundModel=afwMath.BackgroundList(bg),
203 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10),
204 )
207class MockCalibrateTask(PipelineTask):
208 """A do-nothing substitute for CalibrateTask.
209 """
210 ConfigClass = CalibrateConfig
211 _DefaultName = "notCalibrate"
213 def __init__(self, astromRefObjLoader=None,
214 photoRefObjLoader=None, icSourceSchema=None,
215 initInputs=None, **kwargs):
216 super().__init__(**kwargs)
217 self.outputSchema = afwTable.SourceCatalog()
219 def runQuantum(self, butlerQC, inputRefs, outputRefs):
220 inputs = butlerQC.get(inputRefs)
221 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
223 if self.config.doAstrometry:
224 inputs.pop('astromRefCat')
225 if self.config.doPhotoCal:
226 inputs.pop('photoRefCat')
228 outputs = self.run(**inputs)
230 if self.config.doWriteMatches and self.config.doAstrometry:
231 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
232 if self.config.doWriteMatchesDenormalized:
233 # Just need an empty BaseCatalog with a valid schema.
234 outputs.matchesDenormalized = afwTable.BaseCatalog(outputs.outputCat.schema)
235 outputs.matches = normalizedMatches
236 butlerQC.put(outputs, outputRefs)
238 def run(self, exposure, background=None,
239 icSourceCat=None, idGenerator=None):
240 """Produce calibration outputs with no processing.
242 Parameters
243 ----------
244 exposure : `lsst.afw.image.Exposure`
245 Exposure to calibrate.
246 background : `lsst.afw.math.BackgroundList`, optional
247 Background model already subtracted from exposure.
248 icSourceCat : `lsst.afw.table.SourceCatalog`, optional
249 A SourceCatalog from CharacterizeImageTask from which we can copy some fields.
250 idGenerator : `lsst.meas.base.IdGenerator`, optional
251 Object that generates source IDs and provides random number
252 generator seeds.
254 Returns
255 -------
256 result : `lsst.pipe.base.Struct`
257 Struct containing these fields:
259 ``outputExposure``
260 Calibrated science exposure with refined WCS and PhotoCalib
261 (`lsst.afw.image.Exposure`).
262 ``outputBackground``
263 Model of background subtracted from exposure
264 (`lsst.afw.math.BackgroundList`).
265 ``outputCat``
266 Catalog of measured sources (`lsst.afw.table.SourceCatalog`).
267 ``astromMatches``
268 List of source/refObj matches from the astrometry solver
269 (`list` [`lsst.afw.table.ReferenceMatch`]).
270 """
271 # Can't persist empty BackgroundList; DM-33714
272 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
273 afwImage.MaskedImageF(16, 16))
274 return Struct(outputExposure=exposure,
275 outputBackground=afwMath.BackgroundList(bg),
276 outputCat=afwTable.SourceCatalog(),
277 astromMatches=[],
278 )
281class MockGetTemplateTask(PipelineTask):
282 """A do-nothing substitute for GetTemplateTask.
283 """
284 ConfigClass = GetTemplateConfig
285 _DefaultName = "notGetTemplate"
287 def runQuantum(self, butlerQC, inputRefs, outputRefs):
288 inputs = butlerQC.get(inputRefs)
289 # Mock GetTemplateTask.getOverlappingExposures
290 results = Struct(coaddExposures=[],
291 dataIds=[],
292 )
293 inputs["coaddExposures"] = results.coaddExposures
294 inputs["dataIds"] = results.dataIds
295 outputs = self.run(**inputs)
296 butlerQC.put(outputs, outputRefs)
298 def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs):
299 """Warp coadds from multiple tracts to form a template for image diff.
301 Where the tracts overlap, the resulting template image is averaged.
302 The PSF on the template is created by combining the CoaddPsf on each
303 template image into a meta-CoaddPsf.
305 Parameters
306 ----------
307 coaddExposures : `list` of `lsst.afw.image.Exposure`
308 Coadds to be mosaicked
309 bbox : `lsst.geom.Box2I`
310 Template Bounding box of the detector geometry onto which to
311 resample the coaddExposures
312 wcs : `lsst.afw.geom.SkyWcs`
313 Template WCS onto which to resample the coaddExposures
314 dataIds : `list` of `lsst.daf.butler.DataCoordinate`
315 Record of the tract and patch of each coaddExposure.
316 **kwargs
317 Any additional keyword parameters.
319 Returns
320 -------
321 result : `lsst.pipe.base.Struct` containing
322 - ``template`` : a template coadd exposure assembled out of patches
323 """
324 return Struct(template=afwImage.ExposureF(),
325 )
328class MockAlardLuptonSubtractTask(PipelineTask):
329 """A do-nothing substitute for AlardLuptonSubtractTask.
330 """
331 ConfigClass = AlardLuptonSubtractConfig
332 _DefaultName = "notAlardLuptonSubtract"
334 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None):
335 """PSF match, subtract, and decorrelate two images.
337 Parameters
338 ----------
339 template : `lsst.afw.image.ExposureF`
340 Template exposure, warped to match the science exposure.
341 science : `lsst.afw.image.ExposureF`
342 Science exposure to subtract from the template.
343 sources : `lsst.afw.table.SourceCatalog`
344 Identified sources on the science exposure. This catalog is used to
345 select sources in order to perform the AL PSF matching on stamp
346 images around them.
347 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
348 Exposure catalog with finalized psf models and aperture correction
349 maps to be applied if config.doApplyFinalizedPsf=True. Catalog
350 uses the detector id for the catalog id, sorted on id for fast
351 lookup. Deprecated in favor of ``visitSummary``, and will be
352 removed after v26.
353 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
354 Exposure catalog with external calibrations to be applied. Catalog
355 uses the detector id for the catalog id, sorted on id for fast
356 lookup. Ignored (for temporary backwards compatibility) if
357 ``finalizedPsfApCorrCatalog`` is provided.
359 Returns
360 -------
361 results : `lsst.pipe.base.Struct`
362 ``difference`` : `lsst.afw.image.ExposureF`
363 Result of subtracting template and science.
364 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
365 Warped and PSF-matched template exposure.
366 ``backgroundModel`` : `lsst.afw.math.Function2D`
367 Background model that was fit while solving for the
368 PSF-matching kernel
369 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
370 Kernel used to PSF-match the convolved image.
371 """
372 return Struct(difference=afwImage.ExposureF(),
373 matchedTemplate=afwImage.ExposureF(),
374 backgroundModel=afwMath.NullFunction2D(),
375 psfMatchingKernel=afwMath.FixedKernel(),
376 )
379class MockDetectAndMeasureConfig(DetectAndMeasureConfig):
381 def setDefaults(self):
382 super().setDefaults()
383 # Avoid delegating to lsst.obs.base.Instrument specialization for the
384 # data ID packing algorithm to use, since test code often does not use a
385 # real Instrument in its data IDs.
386 self.idGenerator.packer.name = "observation"
389class MockDetectAndMeasureTask(PipelineTask):
390 """A do-nothing substitute for DetectAndMeasureTask.
391 """
392 ConfigClass = MockDetectAndMeasureConfig
393 _DefaultName = "notDetectAndMeasure"
395 def __init__(self, **kwargs):
396 super().__init__(**kwargs)
397 self.outputSchema = afwTable.SourceCatalog()
399 def runQuantum(self, butlerQC, inputRefs, outputRefs):
400 inputs = butlerQC.get(inputRefs)
401 idFactory = afwTable.IdFactory.makeSimple()
403 outputs = self.run(inputs['science'],
404 inputs['matchedTemplate'],
405 inputs['difference'],
406 idFactory=idFactory)
407 butlerQC.put(outputs, outputRefs)
409 def run(self, science, matchedTemplate, difference,
410 idFactory=None):
411 """Detect and measure sources on a difference image.
413 Parameters
414 ----------
415 science : `lsst.afw.image.ExposureF`
416 Science exposure that the template was subtracted from.
417 matchedTemplate : `lsst.afw.image.ExposureF`
418 Warped and PSF-matched template that was used produce the
419 difference image.
420 difference : `lsst.afw.image.ExposureF`
421 Result of subtracting template from the science image.
422 idFactory : `lsst.afw.table.IdFactory`, optional
423 Generator object to assign ids to detected sources in the difference image.
425 Returns
426 -------
427 results : `lsst.pipe.base.Struct`
428 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
429 Subtracted exposure with detection mask applied.
430 ``diaSources`` : `lsst.afw.table.SourceCatalog`
431 The catalog of detected sources.
432 """
433 return Struct(subtractedMeasuredExposure=difference,
434 diaSources=afwTable.SourceCatalog(),
435 )
438class MockFilterDiaSourceCatalogTask(PipelineTask):
439 """A do-nothing substitute for FilterDiaSourceCatalogTask.
440 """
441 ConfigClass = FilterDiaSourceCatalogConfig
442 _DefaultName = "notFilterDiaSourceCatalog"
444 def run(self, diaSourceCat):
445 """Produce filtering outputs with no processing.
447 Parameters
448 ----------
449 diaSourceCat : `lsst.afw.table.SourceCatalog`
450 Catalog of sources measured on the difference image.
452 Returns
453 -------
454 results : `lsst.pipe.base.Struct`
455 Results struct with components.
456 """
457 return Struct(filteredDiaSourceCat=afwTable.SourceCatalog(),
458 rejectedDiaSources=afwTable.SourceCatalog(),
459 )
462class MockRBTransiNetTask(PipelineTask):
463 """A do-nothing substitute for RBTransiNetTask.
464 """
465 _DefaultName = "notRbTransiNet"
466 ConfigClass = RBTransiNetConfig
468 def run(self, template, science, difference, diaSources, pretrainedModel=None):
469 return Struct(classifications=afwTable.BaseCatalog(afwTable.Schema()))
472class MockTransformDiaSourceCatalogTask(PipelineTask):
473 """A do-nothing substitute for TransformDiaSourceCatalogTask.
474 """
475 ConfigClass = TransformDiaSourceCatalogConfig
476 _DefaultName = "notTransformDiaSourceCatalog"
478 def __init__(self, initInputs, **kwargs):
479 super().__init__(**kwargs)
481 def runQuantum(self, butlerQC, inputRefs, outputRefs):
482 inputs = butlerQC.get(inputRefs)
483 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
484 inputs["ccdVisitId"] = idGenerator.catalog_id
485 inputs["band"] = butlerQC.quantum.dataId["band"]
487 outputs = self.run(**inputs)
489 butlerQC.put(outputs, outputRefs)
491 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None):
492 """Produce transformation outputs with no processing.
494 Parameters
495 ----------
496 diaSourceCat : `lsst.afw.table.SourceCatalog`
497 The catalog to transform.
498 diffIm : `lsst.afw.image.Exposure`
499 An image, to provide supplementary information.
500 band : `str`
501 The band in which the sources were observed.
502 ccdVisitId : `int`
503 The exposure ID in which the sources were observed.
504 funcs, optional
505 Unused.
507 Returns
508 -------
509 results : `lsst.pipe.base.Struct`
510 Results struct with components:
512 ``diaSourceTable``
513 Catalog of DiaSources (`pandas.DataFrame`).
514 """
515 return Struct(diaSourceTable=pandas.DataFrame(),
516 )
519class MockDiaPipelineConfig(DiaPipelineConfig):
521 def setDefaults(self):
522 super().setDefaults()
523 # Avoid delegating to lsst.obs.base.Instrument specialization for the
524 # data ID packing algorithm to use, since test code often does not use a
525 # real Instrument in its data IDs.
526 self.idGenerator.packer.name = "observation"
529class MockDiaPipelineTask(PipelineTask):
530 """A do-nothing substitute for DiaPipelineTask.
531 """
532 ConfigClass = MockDiaPipelineConfig
533 _DefaultName = "notDiaPipe"
535 def runQuantum(self, butlerQC, inputRefs, outputRefs):
536 inputs = butlerQC.get(inputRefs)
537 inputs["idGenerator"] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
538 # Need to set ccdExposureIdBits (now deprecated) to None and pass it,
539 # since there are non-optional positional arguments after it.
540 inputs["ccdExposureIdBits"] = None
541 inputs["band"] = butlerQC.quantum.dataId["band"]
542 if not self.config.doSolarSystemAssociation:
543 inputs["solarSystemObjectTable"] = None
545 outputs = self.run(**inputs)
547 butlerQC.put(outputs, outputRefs)
549 def run(self,
550 diaSourceTable,
551 solarSystemObjectTable,
552 diffIm,
553 exposure,
554 template,
555 ccdExposureIdBits,
556 band,
557 idGenerator=None):
558 """Produce DiaSource and DiaObject outputs with no processing.
560 Parameters
561 ----------
562 diaSourceTable : `pandas.DataFrame`
563 Newly detected DiaSources.
564 solarSystemObjectTable : `pandas.DataFrame`
565 Expected solar system objects in the field of view.
566 diffIm : `lsst.afw.image.ExposureF`
567 Difference image exposure in which the sources in ``diaSourceCat``
568 were detected.
569 exposure : `lsst.afw.image.ExposureF`
570 Calibrated exposure differenced with a template to create
571 ``diffIm``.
572 template : `lsst.afw.image.ExposureF`
573 Template exposure used to create diffIm.
574 ccdExposureIdBits : `int`
575 Number of bits used for a unique ``ccdVisitId``. Deprecated in
576 favor of ``idGenerator``, and ignored if that is present. Pass
577 `None` explicitly to avoid a deprecation warning (a default is
578 impossible given that later positional arguments are not
579 defaulted).
580 band : `str`
581 The band in which the new DiaSources were detected.
582 idGenerator : `lsst.meas.base.IdGenerator`, optional
583 Object that generates source IDs and random number generator seeds.
584 Will be required after ``ccdExposureIdBits`` is removed.
586 Returns
587 -------
588 results : `lsst.pipe.base.Struct`
589 Results struct with components:
591 ``apdbMarker``
592 Marker dataset to store in the Butler indicating that this
593 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`).
594 ``associatedDiaSources``
595 Catalog of newly associated DiaSources (`pandas.DataFrame`).
596 """
597 return Struct(apdbMarker=self.config.apdb.value,
598 associatedDiaSources=pandas.DataFrame(),
599 diaForcedSources=pandas.DataFrame(),
600 diaObjects=pandas.DataFrame(),
601 longTrailedSources=pandas.DataFrame(),
602 )