Coverage for python/lsst/ap/verify/testPipeline.py: 44%
115 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-30 10:15 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-30 10:15 +0000
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
25# These classes exist only to be included in a mock pipeline, and don't need
26# to be public for that.
27__all__ = []
30import numpy as np
31import pandas
33import lsst.geom as geom
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.afw.table as afwTable
37import lsst.obs.base as obsBase
38from lsst.pipe.base import PipelineTask, Struct
39from lsst.ip.isr import IsrTaskConfig
40from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig
41from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
42from lsst.pipe.tasks.calibrate import CalibrateConfig
43from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig
46class MockIsrTask(PipelineTask):
47 """A do-nothing substitute for IsrTask.
48 """
49 ConfigClass = IsrTaskConfig
50 _DefaultName = "notIsr"
52 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
53 crosstalk=None, crosstalkSources=None,
54 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
55 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
56 sensorTransmission=None, atmosphereTransmission=None,
57 detectorNum=None, strayLightData=None, illumMaskedImage=None,
58 deferredCharge=None,
59 ):
60 """Accept ISR inputs, and produce ISR outputs with no processing.
62 Parameters
63 ----------
64 ccdExposure : `lsst.afw.image.Exposure`
65 The raw exposure that is to be run through ISR. The
66 exposure is modified by this method.
67 camera : `lsst.afw.cameraGeom.Camera`, optional
68 The camera geometry for this exposure. Required if
69 one or more of ``ccdExposure``, ``bias``, ``dark``, or
70 ``flat`` does not have an associated detector.
71 bias : `lsst.afw.image.Exposure`, optional
72 Bias calibration frame.
73 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
74 Functor for linearization.
75 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
76 Calibration for crosstalk.
77 crosstalkSources : `list`, optional
78 List of possible crosstalk sources.
79 dark : `lsst.afw.image.Exposure`, optional
80 Dark calibration frame.
81 flat : `lsst.afw.image.Exposure`, optional
82 Flat calibration frame.
83 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
84 Photon transfer curve dataset, with, e.g., gains
85 and read noise.
86 bfKernel : `numpy.ndarray`, optional
87 Brighter-fatter kernel.
88 bfGains : `dict` of `float`, optional
89 Gains used to override the detector's nominal gains for the
90 brighter-fatter correction. A dict keyed by amplifier name for
91 the detector in question.
92 defects : `lsst.ip.isr.Defects`, optional
93 List of defects.
94 fringes : `lsst.pipe.base.Struct`, optional
95 Struct containing the fringe correction data, with
96 elements:
97 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
98 - ``seed``: random seed derived from the ccdExposureId for random
99 number generator (`uint32`)
100 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
101 A ``TransmissionCurve`` that represents the throughput of the
102 optics, to be evaluated in focal-plane coordinates.
103 filterTransmission : `lsst.afw.image.TransmissionCurve`
104 A ``TransmissionCurve`` that represents the throughput of the
105 filter itself, to be evaluated in focal-plane coordinates.
106 sensorTransmission : `lsst.afw.image.TransmissionCurve`
107 A ``TransmissionCurve`` that represents the throughput of the
108 sensor itself, to be evaluated in post-assembly trimmed detector
109 coordinates.
110 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
111 A ``TransmissionCurve`` that represents the throughput of the
112 atmosphere, assumed to be spatially constant.
113 detectorNum : `int`, optional
114 The integer number for the detector to process.
115 isGen3 : bool, optional
116 Flag this call to run() as using the Gen3 butler environment.
117 strayLightData : `object`, optional
118 Opaque object containing calibration information for stray-light
119 correction. If `None`, no correction will be performed.
120 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
121 Illumination correction image.
123 Returns
124 -------
125 result : `lsst.pipe.base.Struct`
126 Result struct with components:
128 ``exposure``
129 The fully ISR corrected exposure (`afw.image.Exposure`).
130 ``outputExposure``
131 An alias for ``exposure`` (`afw.image.Exposure`).
132 ``ossThumb``
133 Thumbnail image of the exposure after overscan subtraction
134 (`numpy.ndarray`).
135 ``flattenedThumb``
136 Thumbnail image of the exposure after flat-field correction
137 (`numpy.ndarray`).
138 - ``outputStatistics`` : mapping [`str`]
139 Values of the additional statistics calculated.
140 """
141 return Struct(exposure=afwImage.ExposureF(),
142 outputExposure=afwImage.ExposureF(),
143 ossThumb=np.empty((1, 1)),
144 flattenedThumb=np.empty((1, 1)),
145 preInterpExposure=afwImage.ExposureF(),
146 outputOssThumbnail=np.empty((1, 1)),
147 outputFlattenedThumbnail=np.empty((1, 1)),
148 outputStatistics={},
149 )
152class MockCharacterizeImageTask(PipelineTask):
153 """A do-nothing substitute for CharacterizeImageTask.
154 """
155 ConfigClass = CharacterizeImageConfig
156 _DefaultName = "notCharacterizeImage"
158 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
159 super().__init__(**kwargs)
160 self.outputSchema = afwTable.SourceCatalog()
162 def runQuantum(self, butlerQC, inputRefs, outputRefs):
163 inputs = butlerQC.get(inputRefs)
164 if 'exposureIdInfo' not in inputs.keys():
165 inputs['exposureIdInfo'] = obsBase.ExposureIdInfo.fromDataId(
166 butlerQC.quantum.dataId, "visit_detector")
167 outputs = self.run(**inputs)
168 butlerQC.put(outputs, outputRefs)
170 def run(self, exposure, exposureIdInfo=None, background=None):
171 """Produce characterization outputs with no processing.
173 Parameters
174 ----------
175 exposure : `lsst.afw.image.Exposure`
176 Exposure to characterize.
177 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
178 ID info for exposure.
179 background : `lsst.afw.math.BackgroundList`
180 Initial model of background already subtracted from exposure.
182 Returns
183 -------
184 result : `lsst.pipe.base.Struct`
185 Struct containing these fields:
187 ``characterized``
188 Characterized exposure (`lsst.afw.image.Exposure`).
189 ``sourceCat``
190 Detected sources (`lsst.afw.table.SourceCatalog`).
191 ``backgroundModel``
192 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`)
193 ``psfCellSet``
194 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`)
195 """
196 # Can't persist empty BackgroundList; DM-33714
197 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
198 afwImage.MaskedImageF(16, 16))
199 return Struct(characterized=exposure,
200 sourceCat=afwTable.SourceCatalog(),
201 backgroundModel=afwMath.BackgroundList(bg),
202 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10),
203 )
206class MockCalibrateTask(PipelineTask):
207 """A do-nothing substitute for CalibrateTask.
208 """
209 ConfigClass = CalibrateConfig
210 _DefaultName = "notCalibrate"
212 def __init__(self, butler=None, astromRefObjLoader=None,
213 photoRefObjLoader=None, icSourceSchema=None,
214 initInputs=None, **kwargs):
215 super().__init__(**kwargs)
216 self.outputSchema = afwTable.SourceCatalog()
218 def runQuantum(self, butlerQC, inputRefs, outputRefs):
219 inputs = butlerQC.get(inputRefs)
220 inputs['exposureIdInfo'] = obsBase.ExposureIdInfo.fromDataId(
221 butlerQC.quantum.dataId, "visit_detector")
223 if self.config.doAstrometry:
224 inputs.pop('astromRefCat')
225 if self.config.doPhotoCal:
226 inputs.pop('photoRefCat')
228 outputs = self.run(**inputs)
230 if self.config.doWriteMatches and self.config.doAstrometry:
231 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
232 if self.config.doWriteMatchesDenormalized:
233 outputs.matchesDenormalized = outputs.astromMatches
234 outputs.matches = normalizedMatches
235 butlerQC.put(outputs, outputRefs)
237 def run(self, exposure, exposureIdInfo=None, background=None,
238 icSourceCat=None):
239 """Produce calibration outputs with no processing.
241 Parameters
242 ----------
243 exposure : `lsst.afw.image.Exposure`
244 Exposure to calibrate.
245 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
246 ID info for exposure.
247 background : `lsst.afw.math.BackgroundList`
248 Background model already subtracted from exposure.
249 icSourceCat : `lsst.afw.table.SourceCatalog`
250 A SourceCatalog from CharacterizeImageTask from which we can copy some fields.
252 Returns
253 -------
254 result : `lsst.pipe.base.Struct`
255 Struct containing these fields:
257 ``outputExposure``
258 Calibrated science exposure with refined WCS and PhotoCalib
259 (`lsst.afw.image.Exposure`).
260 ``outputBackground``
261 Model of background subtracted from exposure
262 (`lsst.afw.math.BackgroundList`).
263 ``outputCat``
264 Catalog of measured sources (`lsst.afw.table.SourceCatalog`).
265 ``astromMatches``
266 List of source/refObj matches from the astrometry solver
267 (`list` [`lsst.afw.table.ReferenceMatch`]).
268 """
269 # Can't persist empty BackgroundList; DM-33714
270 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
271 afwImage.MaskedImageF(16, 16))
272 return Struct(outputExposure=exposure,
273 outputBackground=afwMath.BackgroundList(bg),
274 outputCat=afwTable.SourceCatalog(),
275 astromMatches=[],
276 )
279class MockGetTemplateTask(PipelineTask):
280 """A do-nothing substitute for GetTemplateTask.
281 """
282 ConfigClass = GetTemplateConfig
283 _DefaultName = "notGetTemplate"
285 def runQuantum(self, butlerQC, inputRefs, outputRefs):
286 inputs = butlerQC.get(inputRefs)
287 # Mock GetTemplateTask.getOverlappingExposures
288 results = Struct(coaddExposures=[],
289 dataIds=[],
290 )
291 inputs["coaddExposures"] = results.coaddExposures
292 inputs["dataIds"] = results.dataIds
293 outputs = self.run(**inputs)
294 butlerQC.put(outputs, outputRefs)
296 def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs):
297 """Warp coadds from multiple tracts to form a template for image diff.
299 Where the tracts overlap, the resulting template image is averaged.
300 The PSF on the template is created by combining the CoaddPsf on each
301 template image into a meta-CoaddPsf.
303 Parameters
304 ----------
305 coaddExposures : `list` of `lsst.afw.image.Exposure`
306 Coadds to be mosaicked
307 bbox : `lsst.geom.Box2I`
308 Template Bounding box of the detector geometry onto which to
309 resample the coaddExposures
310 wcs : `lsst.afw.geom.SkyWcs`
311 Template WCS onto which to resample the coaddExposures
312 dataIds : `list` of `lsst.daf.butler.DataCoordinate`
313 Record of the tract and patch of each coaddExposure.
314 **kwargs
315 Any additional keyword parameters.
317 Returns
318 -------
319 result : `lsst.pipe.base.Struct` containing
320 - ``template`` : a template coadd exposure assembled out of patches
321 """
322 return Struct(template=afwImage.ExposureF(),
323 )
326class MockAlardLuptonSubtractTask(PipelineTask):
327 """A do-nothing substitute for AlardLuptonSubtractTask.
328 """
329 ConfigClass = AlardLuptonSubtractConfig
330 _DefaultName = "notAlardLuptonSubtract"
332 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None):
333 """PSF match, subtract, and decorrelate two images.
335 Parameters
336 ----------
337 template : `lsst.afw.image.ExposureF`
338 Template exposure, warped to match the science exposure.
339 science : `lsst.afw.image.ExposureF`
340 Science exposure to subtract from the template.
341 sources : `lsst.afw.table.SourceCatalog`
342 Identified sources on the science exposure. This catalog is used to
343 select sources in order to perform the AL PSF matching on stamp
344 images around them.
345 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
346 Exposure catalog with finalized psf models and aperture correction
347 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
348 the detector id for the catalog id, sorted on id for fast lookup.
350 Returns
351 -------
352 results : `lsst.pipe.base.Struct`
353 ``difference`` : `lsst.afw.image.ExposureF`
354 Result of subtracting template and science.
355 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
356 Warped and PSF-matched template exposure.
357 ``backgroundModel`` : `lsst.afw.math.Function2D`
358 Background model that was fit while solving for the PSF-matching kernel
359 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
360 Kernel used to PSF-match the convolved image.
361 """
362 return Struct(difference=afwImage.ExposureF(),
363 matchedTemplate=afwImage.ExposureF(),
364 backgroundModel=afwMath.NullFunction2D(),
365 psfMatchingKernel=afwMath.FixedKernel(),
366 )
369class MockDetectAndMeasureTask(PipelineTask):
370 """A do-nothing substitute for DetectAndMeasureTask.
371 """
372 ConfigClass = DetectAndMeasureConfig
373 _DefaultName = "notDetectAndMeasure"
375 def __init__(self, **kwargs):
376 super().__init__(**kwargs)
377 self.outputSchema = afwTable.SourceCatalog()
379 def runQuantum(self, butlerQC, inputRefs, outputRefs):
380 inputs = butlerQC.get(inputRefs)
381 idFactory = obsBase.ExposureIdInfo(8, 4).makeSourceIdFactory()
383 outputs = self.run(inputs['science'],
384 inputs['matchedTemplate'],
385 inputs['difference'],
386 idFactory=idFactory)
387 butlerQC.put(outputs, outputRefs)
389 def run(self, science, matchedTemplate, difference,
390 idFactory=None):
391 """Detect and measure sources on a difference image.
393 Parameters
394 ----------
395 science : `lsst.afw.image.ExposureF`
396 Science exposure that the template was subtracted from.
397 matchedTemplate : `lsst.afw.image.ExposureF`
398 Warped and PSF-matched template that was used produce the
399 difference image.
400 difference : `lsst.afw.image.ExposureF`
401 Result of subtracting template from the science image.
402 idFactory : `lsst.afw.table.IdFactory`, optional
403 Generator object to assign ids to detected sources in the difference image.
405 Returns
406 -------
407 results : `lsst.pipe.base.Struct`
408 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
409 Subtracted exposure with detection mask applied.
410 ``diaSources`` : `lsst.afw.table.SourceCatalog`
411 The catalog of detected sources.
412 """
413 return Struct(subtractedMeasuredExposure=difference,
414 diaSources=afwTable.SourceCatalog(),
415 )
418class MockTransformDiaSourceCatalogTask(PipelineTask):
419 """A do-nothing substitute for TransformDiaSourceCatalogTask.
420 """
421 ConfigClass = TransformDiaSourceCatalogConfig
422 _DefaultName = "notTransformDiaSourceCatalog"
424 def __init__(self, initInputs, **kwargs):
425 super().__init__(**kwargs)
427 def runQuantum(self, butlerQC, inputRefs, outputRefs):
428 inputs = butlerQC.get(inputRefs)
429 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
430 returnMaxBits=True)
431 inputs["ccdVisitId"] = expId
432 inputs["band"] = butlerQC.quantum.dataId["band"]
434 outputs = self.run(**inputs)
436 butlerQC.put(outputs, outputRefs)
438 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None):
439 """Produce transformation outputs with no processing.
441 Parameters
442 ----------
443 diaSourceCat : `lsst.afw.table.SourceCatalog`
444 The catalog to transform.
445 diffIm : `lsst.afw.image.Exposure`
446 An image, to provide supplementary information.
447 band : `str`
448 The band in which the sources were observed.
449 ccdVisitId : `int`
450 The exposure ID in which the sources were observed.
451 funcs
452 Unused.
454 Returns
455 -------
456 results : `lsst.pipe.base.Struct`
457 Results struct with components:
459 ``diaSourceTable``
460 Catalog of DiaSources (`pandas.DataFrame`).
461 """
462 return Struct(diaSourceTable=pandas.DataFrame(),
463 )
466class MockDiaPipelineTask(PipelineTask):
467 """A do-nothing substitute for DiaPipelineTask.
468 """
469 ConfigClass = DiaPipelineConfig
470 _DefaultName = "notDiaPipe"
472 def runQuantum(self, butlerQC, inputRefs, outputRefs):
473 inputs = butlerQC.get(inputRefs)
474 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
475 returnMaxBits=True)
476 inputs["ccdExposureIdBits"] = expBits
477 inputs["band"] = butlerQC.quantum.dataId["band"]
478 if not self.config.doSolarSystemAssociation:
479 inputs["solarSystemObjectTable"] = None
481 outputs = self.run(**inputs)
483 butlerQC.put(outputs, outputRefs)
485 def run(self,
486 diaSourceTable,
487 solarSystemObjectTable,
488 diffIm,
489 exposure,
490 template,
491 ccdExposureIdBits,
492 band):
493 """Produce DiaSource and DiaObject outputs with no processing.
495 Parameters
496 ----------
497 diaSourceTable : `pandas.DataFrame`
498 Newly detected DiaSources.
499 solarSystemObjectTable : `pandas.DataFrame`
500 Expected solar system objects in the field of view.
501 diffIm : `lsst.afw.image.ExposureF`
502 Difference image exposure in which the sources in ``diaSourceCat``
503 were detected.
504 exposure : `lsst.afw.image.ExposureF`
505 Calibrated exposure differenced with a template to create
506 ``diffIm``.
507 template : `lsst.afw.image.ExposureF`
508 Template exposure used to create diffIm.
509 ccdExposureIdBits : `int`
510 Number of bits used for a unique ``ccdVisitId``.
511 band : `str`
512 The band in which the new DiaSources were detected.
514 Returns
515 -------
516 results : `lsst.pipe.base.Struct`
517 Results struct with components:
519 ``apdbMarker``
520 Marker dataset to store in the Butler indicating that this
521 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`).
522 ``associatedDiaSources``
523 Catalog of newly associated DiaSources (`pandas.DataFrame`).
524 """
525 return Struct(apdbMarker=self.config.apdb.value,
526 associatedDiaSources=pandas.DataFrame(),
527 diaForcedSources=pandas.DataFrame(),
528 diaObjects=pandas.DataFrame(),
529 )