Coverage for python/lsst/ap/verify/testPipeline.py: 49%
97 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-10 01:58 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-10 01:58 -0700
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
25# These classes exist only to be included in a mock pipeline, and don't need
26# to be public for that.
27__all__ = []
30import numpy as np
31import pandas
33import lsst.geom as geom
34import lsst.afw.image as afwImage
35import lsst.afw.math as afwMath
36import lsst.afw.table as afwTable
37import lsst.obs.base as obsBase
38from lsst.pipe.base import PipelineTask, Struct
39from lsst.ip.isr import IsrTaskConfig
40from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
41from lsst.pipe.tasks.calibrate import CalibrateConfig
42from lsst.pipe.tasks.imageDifference import ImageDifferenceConfig
43from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig
46class MockIsrTask(PipelineTask):
47 """A do-nothing substitute for IsrTask.
48 """
49 ConfigClass = IsrTaskConfig
50 _DefaultName = "notIsr"
52 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
53 crosstalk=None, crosstalkSources=None,
54 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
55 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
56 sensorTransmission=None, atmosphereTransmission=None,
57 detectorNum=None, strayLightData=None, illumMaskedImage=None,
58 isGen3=False,
59 ):
60 """Accept ISR inputs, and produce ISR outputs with no processing.
62 Parameters
63 ----------
64 ccdExposure : `lsst.afw.image.Exposure`
65 The raw exposure that is to be run through ISR. The
66 exposure is modified by this method.
67 camera : `lsst.afw.cameraGeom.Camera`, optional
68 The camera geometry for this exposure. Required if
69 one or more of ``ccdExposure``, ``bias``, ``dark``, or
70 ``flat`` does not have an associated detector.
71 bias : `lsst.afw.image.Exposure`, optional
72 Bias calibration frame.
73 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
74 Functor for linearization.
75 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
76 Calibration for crosstalk.
77 crosstalkSources : `list`, optional
78 List of possible crosstalk sources.
79 dark : `lsst.afw.image.Exposure`, optional
80 Dark calibration frame.
81 flat : `lsst.afw.image.Exposure`, optional
82 Flat calibration frame.
83 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
84 Photon transfer curve dataset, with, e.g., gains
85 and read noise.
86 bfKernel : `numpy.ndarray`, optional
87 Brighter-fatter kernel.
88 bfGains : `dict` of `float`, optional
89 Gains used to override the detector's nominal gains for the
90 brighter-fatter correction. A dict keyed by amplifier name for
91 the detector in question.
92 defects : `lsst.ip.isr.Defects`, optional
93 List of defects.
94 fringes : `lsst.pipe.base.Struct`, optional
95 Struct containing the fringe correction data, with
96 elements:
97 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
98 - ``seed``: random seed derived from the ccdExposureId for random
99 number generator (`uint32`)
100 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
101 A ``TransmissionCurve`` that represents the throughput of the
102 optics, to be evaluated in focal-plane coordinates.
103 filterTransmission : `lsst.afw.image.TransmissionCurve`
104 A ``TransmissionCurve`` that represents the throughput of the
105 filter itself, to be evaluated in focal-plane coordinates.
106 sensorTransmission : `lsst.afw.image.TransmissionCurve`
107 A ``TransmissionCurve`` that represents the throughput of the
108 sensor itself, to be evaluated in post-assembly trimmed detector
109 coordinates.
110 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
111 A ``TransmissionCurve`` that represents the throughput of the
112 atmosphere, assumed to be spatially constant.
113 detectorNum : `int`, optional
114 The integer number for the detector to process.
115 isGen3 : bool, optional
116 Flag this call to run() as using the Gen3 butler environment.
117 strayLightData : `object`, optional
118 Opaque object containing calibration information for stray-light
119 correction. If `None`, no correction will be performed.
120 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
121 Illumination correction image.
123 Returns
124 -------
125 result : `lsst.pipe.base.Struct`
126 Result struct with components:
128 ``exposure``
129 The fully ISR corrected exposure (`afw.image.Exposure`).
130 ``outputExposure``
131 An alias for ``exposure`` (`afw.image.Exposure`).
132 ``ossThumb``
133 Thumbnail image of the exposure after overscan subtraction
134 (`numpy.ndarray`).
135 ``flattenedThumb``
136 Thumbnail image of the exposure after flat-field correction
137 (`numpy.ndarray`).
138 """
139 return Struct(exposure=afwImage.ExposureF(),
140 outputExposure=afwImage.ExposureF(),
141 ossThumb=np.empty((1, 1)),
142 flattenedThumb=np.empty((1, 1)),
143 )
146class MockCharacterizeImageTask(PipelineTask):
147 """A do-nothing substitute for CharacterizeImageTask.
148 """
149 ConfigClass = CharacterizeImageConfig
150 _DefaultName = "notCharacterizeImage"
152 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
153 super().__init__(**kwargs)
154 self.outputSchema = afwTable.SourceCatalog()
156 def runQuantum(self, butlerQC, inputRefs, outputRefs):
157 inputs = butlerQC.get(inputRefs)
158 if 'exposureIdInfo' not in inputs.keys():
159 inputs['exposureIdInfo'] = obsBase.ExposureIdInfo.fromDataId(
160 butlerQC.quantum.dataId, "visit_detector")
161 outputs = self.run(**inputs)
162 butlerQC.put(outputs, outputRefs)
164 def run(self, exposure, exposureIdInfo=None, background=None):
165 """Produce characterization outputs with no processing.
167 Parameters
168 ----------
169 exposure : `lsst.afw.image.Exposure`
170 Exposure to characterize.
171 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
172 ID info for exposure.
173 background : `lsst.afw.math.BackgroundList`
174 Initial model of background already subtracted from exposure.
176 Returns
177 -------
178 result : `lsst.pipe.base.Struct`
179 Struct containing these fields:
181 ``characterized``
182 Characterized exposure (`lsst.afw.image.Exposure`).
183 ``sourceCat``
184 Detected sources (`lsst.afw.table.SourceCatalog`).
185 ``backgroundModel``
186 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`)
187 ``psfCellSet``
188 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`)
189 """
190 # Can't persist empty BackgroundList; DM-33714
191 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
192 afwImage.MaskedImageF(16, 16))
193 return Struct(characterized=exposure,
194 sourceCat=afwTable.SourceCatalog(),
195 backgroundModel=afwMath.BackgroundList(bg),
196 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10),
197 )
200class MockCalibrateTask(PipelineTask):
201 """A do-nothing substitute for CalibrateTask.
202 """
203 ConfigClass = CalibrateConfig
204 _DefaultName = "notCalibrate"
206 def __init__(self, butler=None, astromRefObjLoader=None,
207 photoRefObjLoader=None, icSourceSchema=None,
208 initInputs=None, **kwargs):
209 super().__init__(**kwargs)
210 self.outputSchema = afwTable.SourceCatalog()
212 def runQuantum(self, butlerQC, inputRefs, outputRefs):
213 inputs = butlerQC.get(inputRefs)
214 inputs['exposureIdInfo'] = obsBase.ExposureIdInfo.fromDataId(
215 butlerQC.quantum.dataId, "visit_detector")
217 if self.config.doAstrometry:
218 inputs.pop('astromRefCat')
219 if self.config.doPhotoCal:
220 inputs.pop('photoRefCat')
222 outputs = self.run(**inputs)
224 if self.config.doWriteMatches and self.config.doAstrometry:
225 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
226 if self.config.doWriteMatchesDenormalized:
227 outputs.matchesDenormalized = outputs.astromMatches
228 outputs.matches = normalizedMatches
229 butlerQC.put(outputs, outputRefs)
231 def run(self, exposure, exposureIdInfo=None, background=None,
232 icSourceCat=None):
233 """Produce calibration outputs with no processing.
235 Parameters
236 ----------
237 exposure : `lsst.afw.image.Exposure`
238 Exposure to calibrate.
239 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
240 ID info for exposure.
241 background : `lsst.afw.math.BackgroundList`
242 Background model already subtracted from exposure.
243 icSourceCat : `lsst.afw.table.SourceCatalog`
244 A SourceCatalog from CharacterizeImageTask from which we can copy some fields.
246 Returns
247 -------
248 result : `lsst.pipe.base.Struct`
249 Struct containing these fields:
251 ``outputExposure``
252 Calibrated science exposure with refined WCS and PhotoCalib
253 (`lsst.afw.image.Exposure`).
254 ``outputBackground``
255 Model of background subtracted from exposure
256 (`lsst.afw.math.BackgroundList`).
257 ``outputCat``
258 Catalog of measured sources (`lsst.afw.table.SourceCatalog`).
259 ``astromMatches``
260 List of source/refObj matches from the astrometry solver
261 (`list` [`lsst.afw.table.ReferenceMatch`]).
262 """
263 # Can't persist empty BackgroundList; DM-33714
264 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
265 afwImage.MaskedImageF(16, 16))
266 return Struct(outputExposure=exposure,
267 outputBackground=afwMath.BackgroundList(bg),
268 outputCat=afwTable.SourceCatalog(),
269 astromMatches=[],
270 )
273class MockImageDifferenceTask(PipelineTask):
274 """A do-nothing substitute for ImageDifferenceTask.
275 """
276 ConfigClass = ImageDifferenceConfig
277 _DefaultName = "notImageDifference"
279 def __init__(self, butler=None, **kwargs):
280 super().__init__(**kwargs)
281 self.outputSchema = afwTable.SourceCatalog()
283 def runQuantum(self, butlerQC, inputRefs, outputRefs):
284 inputs = butlerQC.get(inputRefs)
285 outputs = self.run(exposure=inputs['exposure'],
286 templateExposure=afwImage.ExposureF(),
287 idFactory=obsBase.ExposureIdInfo(8, 4).makeSourceIdFactory())
288 butlerQC.put(outputs, outputRefs)
290 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
291 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
292 """Produce differencing outputs with no processing.
294 Parameters
295 ----------
296 exposure : `lsst.afw.image.ExposureF`, optional
297 The science exposure, the minuend in the image subtraction.
298 Can be None only if ``config.doSubtract==False``.
299 selectSources : `lsst.afw.table.SourceCatalog`, optional
300 Identified sources on the science exposure. This catalog is used to
301 select sources in order to perform the AL PSF matching on stamp images
302 around them. The selection steps depend on config options and whether
303 ``templateSources`` and ``matchingSources`` specified.
304 templateExposure : `lsst.afw.image.ExposureF`, optional
305 The template to be subtracted from ``exposure`` in the image subtraction.
306 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
307 The template exposure should cover the same sky area as the science exposure.
308 It is either a stich of patches of a coadd skymap image or a calexp
309 of the same pointing as the science exposure. Can be None only
310 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
311 templateSources : `lsst.afw.table.SourceCatalog`, optional
312 Identified sources on the template exposure.
313 idFactory : `lsst.afw.table.IdFactory`
314 Generator object to assign ids to detected sources in the difference image.
315 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
316 Background exposure to be added back to the science exposure
317 if ``config.doAddCalexpBackground==True``
318 subtractedExposure : `lsst.afw.image.ExposureF`, optional
319 If ``config.doSubtract==False`` and ``config.doDetection==True``,
320 performs the post subtraction source detection only on this exposure.
321 Otherwise should be None.
323 Returns
324 -------
325 results : `lsst.pipe.base.Struct`
327 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
328 Difference image.
329 ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
330 The zogy score exposure, if calculated.
331 ``matchedExposure`` : `lsst.afw.image.ExposureF`
332 The matched PSF exposure.
333 ``warpedExposure`` : `lsst.afw.image.ExposureF`
334 The warped PSF exposure.
335 ``subtractRes`` : `lsst.pipe.base.Struct`
336 The returned result structure of the ImagePsfMatchTask subtask.
337 ``diaSources`` : `lsst.afw.table.SourceCatalog`
338 The catalog of detected sources.
339 ``selectSources`` : `lsst.afw.table.SourceCatalog`
340 The input source catalog with optionally added Qa information.
341 """
342 return Struct(
343 subtractedExposure=afwImage.ExposureF(),
344 scoreExposure=afwImage.ExposureF(),
345 warpedExposure=afwImage.ExposureF(),
346 matchedExposure=afwImage.ExposureF(),
347 subtractRes=Struct(),
348 diaSources=afwTable.SourceCatalog(),
349 selectSources=afwTable.SourceCatalog(),
350 )
353class MockTransformDiaSourceCatalogTask(PipelineTask):
354 """A do-nothing substitute for TransformDiaSourceCatalogTask.
355 """
356 ConfigClass = TransformDiaSourceCatalogConfig
357 _DefaultName = "notTransformDiaSourceCatalog"
359 def __init__(self, initInputs, **kwargs):
360 super().__init__(**kwargs)
362 def runQuantum(self, butlerQC, inputRefs, outputRefs):
363 inputs = butlerQC.get(inputRefs)
364 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
365 returnMaxBits=True)
366 inputs["ccdVisitId"] = expId
367 inputs["band"] = butlerQC.quantum.dataId["band"]
369 outputs = self.run(**inputs)
371 butlerQC.put(outputs, outputRefs)
373 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None):
374 """Produce transformation outputs with no processing.
376 Parameters
377 ----------
378 diaSourceCat : `lsst.afw.table.SourceCatalog`
379 The catalog to transform.
380 diffIm : `lsst.afw.image.Exposure`
381 An image, to provide supplementary information.
382 band : `str`
383 The band in which the sources were observed.
384 ccdVisitId : `int`
385 The exposure ID in which the sources were observed.
386 funcs
387 Unused.
389 Returns
390 -------
391 results : `lsst.pipe.base.Struct`
392 Results struct with components:
394 ``diaSourceTable``
395 Catalog of DiaSources (`pandas.DataFrame`).
396 """
397 return Struct(diaSourceTable=pandas.DataFrame(),
398 )
401class MockDiaPipelineTask(PipelineTask):
402 """A do-nothing substitute for DiaPipelineTask.
403 """
404 ConfigClass = DiaPipelineConfig
405 _DefaultName = "notDiaPipe"
407 def runQuantum(self, butlerQC, inputRefs, outputRefs):
408 inputs = butlerQC.get(inputRefs)
409 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
410 returnMaxBits=True)
411 inputs["ccdExposureIdBits"] = expBits
412 inputs["band"] = butlerQC.quantum.dataId["band"]
413 if not self.config.doSolarSystemAssociation:
414 inputs["solarSystemObjectTable"] = None
416 outputs = self.run(**inputs)
418 butlerQC.put(outputs, outputRefs)
420 def run(self,
421 diaSourceTable,
422 solarSystemObjectTable,
423 diffIm,
424 exposure,
425 warpedExposure,
426 ccdExposureIdBits,
427 band):
428 """Produce DiaSource and DiaObject outputs with no processing.
430 Parameters
431 ----------
432 diaSourceTable : `pandas.DataFrame`
433 Newly detected DiaSources.
434 solarSystemObjectTable : `pandas.DataFrame`
435 Expected solar system objects in the field of view.
436 diffIm : `lsst.afw.image.ExposureF`
437 Difference image exposure in which the sources in ``diaSourceCat``
438 were detected.
439 exposure : `lsst.afw.image.ExposureF`
440 Calibrated exposure differenced with a template to create
441 ``diffIm``.
442 warpedExposure : `lsst.afw.image.ExposureF`
443 Template exposure used to create diffIm.
444 ccdExposureIdBits : `int`
445 Number of bits used for a unique ``ccdVisitId``.
446 band : `str`
447 The band in which the new DiaSources were detected.
449 Returns
450 -------
451 results : `lsst.pipe.base.Struct`
452 Results struct with components:
454 ``apdbMarker``
455 Marker dataset to store in the Butler indicating that this
456 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`).
457 ``associatedDiaSources``
458 Catalog of newly associated DiaSources (`pandas.DataFrame`).
459 """
460 return Struct(apdbMarker=self.config.apdb.value,
461 associatedDiaSources=pandas.DataFrame(),
462 )