Coverage for tests/assembleCoaddTestUtils.py: 24%
178 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-27 02:45 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-27 02:45 -0800
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""Set up simulated test data and simplified APIs for AssembleCoaddTask
23and its derived classes.
25This is not intended to test accessing data with the Butler and instead uses
26mock Butler data references to pass in the simulated data.
27"""
28from astropy.time import Time
29from astropy import units as u
30from astropy.coordinates import SkyCoord, EarthLocation, Angle
31import numpy as np
33from lsst.afw.cameraGeom.testUtils import DetectorWrapper
34import lsst.afw.geom as afwGeom
35import lsst.afw.image as afwImage
36import lsst.daf.butler
37import lsst.geom as geom
38from lsst.geom import arcseconds, degrees
39from lsst.meas.algorithms.testUtils import plantSources
40from lsst.obs.base import MakeRawVisitInfoViaObsInfo
41import lsst.pipe.base as pipeBase
42from lsst.pipe.tasks.coaddInputRecorder import CoaddInputRecorderTask, CoaddInputRecorderConfig
44from astro_metadata_translator import makeObservationInfo
46__all__ = ["MockWarpReference", "makeMockSkyInfo", "MockCoaddTestData"]
49class MockWarpReference(lsst.daf.butler.DeferredDatasetHandle):
50 """Very simple object that looks like a Gen 3 data reference to a warped
51 exposure.
53 Parameters
54 ----------
55 exposure : `lsst.afw.image.Exposure`
56 The exposure to be retrieved by the data reference.
57 coaddName : `str`
58 The type of coadd being produced. Typically 'deep'.
59 patch : `int`
60 Unique identifier for a subdivision of a tract.
61 tract : `int`
62 Unique identifier for a tract of a skyMap
63 visit : `int`
64 Unique identifier for an observation,
65 potentially consisting of multiple ccds.
66 """
67 def __init__(self, exposure, coaddName='deep', patch=42, tract=0, visit=100):
68 self.coaddName = coaddName
69 self.exposure = exposure
70 self.tract = tract
71 self.patch = patch
72 self.visit = visit
74 def get(self, bbox=None, component=None, parameters=None, **kwargs):
75 """Retrieve the specified dataset using the API of the Gen 3 Butler.
77 Parameters
78 ----------
79 bbox : `lsst.geom.box.Box2I`, optional
80 If supplied, retrieve only a subregion of the exposure.
81 component : `str`, optional
82 If supplied, return the named metadata of the exposure.
83 parameters : `dict`, optional
84 If supplied, use the parameters to modify the exposure,
85 typically by taking a subset.
86 **kwargs
87 Additional keyword arguments such as `immediate=True` that would
88 control internal butler behavior.
90 Returns
91 -------
92 `lsst.afw.image.Exposure` or `lsst.afw.image.VisitInfo`
93 or `lsst.meas.algorithms.SingleGaussianPsf`
94 Either the exposure or its metadata, depending on the datasetType.
95 """
96 if component == 'psf':
97 return self.exposure.getPsf()
98 elif component == 'visitInfo':
99 return self.exposure.getInfo().getVisitInfo()
100 if parameters is not None:
101 if "bbox" in parameters:
102 bbox = parameters["bbox"]
103 exp = self.exposure.clone()
104 if bbox is not None:
105 return exp[bbox]
106 else:
107 return exp
109 @property
110 def dataId(self):
111 """Generate a valid data identifier.
113 Returns
114 -------
115 dataId : `lsst.daf.butler.DataCoordinate`
116 Data identifier dict for the patch.
117 """
118 return lsst.daf.butler.DataCoordinate.standardize(
119 tract=self.tract,
120 patch=self.patch,
121 visit=self.visit,
122 instrument="DummyCam",
123 skymap="Skymap",
124 universe=lsst.daf.butler.DimensionUniverse(),
125 )
128def makeMockSkyInfo(bbox, wcs, patch):
129 """Construct a `Struct` containing the geometry of the patch to be coadded.
131 Parameters
132 ----------
133 bbox : `lsst.geom.Box`
134 Bounding box of the patch to be coadded.
135 wcs : `lsst.afw.geom.SkyWcs`
136 Coordinate system definition (wcs) for the exposure.
138 Returns
139 -------
140 skyInfo : `lsst.pipe.base.Struct`
141 Patch geometry information.
142 """
143 def getIndex():
144 return patch
145 patchInfo = pipeBase.Struct(getIndex=getIndex)
146 skyInfo = pipeBase.Struct(bbox=bbox, wcs=wcs, patchInfo=patchInfo)
147 return skyInfo
150class MockCoaddTestData:
151 """Generate repeatable simulated exposures with consistent metadata that
152 are realistic enough to test the image coaddition algorithms.
154 Notes
155 -----
156 The simple GaussianPsf used by lsst.meas.algorithms.testUtils.plantSources
157 will always return an average position of (0, 0).
158 The bounding box of the exposures MUST include (0, 0), or else the PSF will
159 not be valid and `AssembleCoaddTask` will fail with the error
160 'Could not find a valid average position for CoaddPsf'.
162 Parameters
163 ----------
164 shape : `lsst.geom.Extent2I`, optional
165 Size of the bounding box of the exposures to be simulated, in pixels.
166 offset : `lsst.geom.Point2I`, optional
167 Pixel coordinate of the lower left corner of the bounding box.
168 backgroundLevel : `float`, optional
169 Background value added to all pixels in the simulated images.
170 seed : `int`, optional
171 Seed value to initialize the random number generator.
172 nSrc : `int`, optional
173 Number of sources to simulate.
174 fluxRange : `float`, optional
175 Range in flux amplitude of the simulated sources.
176 noiseLevel : `float`, optional
177 Standard deviation of the noise to add to each pixel.
178 sourceSigma : `float`, optional
179 Average amplitude of the simulated sources,
180 relative to ``noiseLevel``
181 minPsfSize : `float`, optional
182 The smallest PSF width (sigma) to use, in pixels.
183 maxPsfSize : `float`, optional
184 The largest PSF width (sigma) to use, in pixels.
185 pixelScale : `lsst.geom.Angle`, optional
186 The plate scale of the simulated images.
187 ra : `lsst.geom.Angle`, optional
188 Right Ascension of the boresight of the camera for the observation.
189 dec : `lsst.geom.Angle`, optional
190 Declination of the boresight of the camera for the observation.
191 ccd : `int`, optional
192 CCD number to put in the metadata of the exposure.
193 patch : `int`, optional
194 Unique identifier for a subdivision of a tract.
195 tract : `int`, optional
196 Unique identifier for a tract of a skyMap.
198 Raises
199 ------
200 ValueError
201 If the bounding box does not contain the pixel coordinate (0, 0).
202 This is due to `GaussianPsf` that is used by `lsst.meas.algorithms.testUtils.plantSources`
203 lacking the option to specify the pixel origin.
204 """
205 rotAngle = 0.*degrees
206 "Rotation of the pixel grid on the sky, East from North (`lsst.geom.Angle`)."
207 filterLabel = None
208 """The filter definition, usually set in the current instruments' obs package.
209 For these tests, a simple filter is defined without using an obs package (`lsst.afw.image.FilterLabel`).
210 """
211 rngData = None
212 """Pre-initialized random number generator for constructing the test images
213 repeatably (`numpy.random.Generator`).
214 """
215 rngMods = None
216 """Pre-initialized random number generator for applying modifications to
217 the test images for only some test cases (`numpy.random.Generator`).
218 """
219 kernelSize = None
220 "Width of the kernel used for simulating sources, in pixels."
221 exposures = {}
222 "The simulated test data, with variable PSF sizes (`dict` of `lsst.afw.image.Exposure`)"
223 matchedExposures = {}
224 """The simulated exposures, all with PSF width set to `maxPsfSize`
225 (`dict` of `lsst.afw.image.Exposure`).
226 """
227 photoCalib = afwImage.makePhotoCalibFromCalibZeroPoint(27, 10)
228 "The photometric zero point to use for converting counts to flux units (`lsst.afw.image.PhotoCalib`)."
229 badMaskPlanes = ["NO_DATA", "BAD"]
230 "Mask planes that, if set, the associated pixel should not be included in the coaddTempExp."
231 detector = None
232 "Properties of the CCD for the exposure (`lsst.afw.cameraGeom.Detector`)."
234 def __init__(self, shape=geom.Extent2I(201, 301), offset=geom.Point2I(-123, -45),
235 backgroundLevel=314.592, seed=42, nSrc=37,
236 fluxRange=2., noiseLevel=5, sourceSigma=200.,
237 minPsfSize=1.5, maxPsfSize=3.,
238 pixelScale=0.2*arcseconds, ra=209.*degrees, dec=-20.25*degrees,
239 ccd=37, patch=42, tract=0):
240 self.ra = ra
241 self.dec = dec
242 self.pixelScale = pixelScale
243 self.patch = patch
244 self.tract = tract
245 self.filterLabel = afwImage.FilterLabel(band="gTest", physical="gTest")
246 self.rngData = np.random.default_rng(seed)
247 self.rngMods = np.random.default_rng(seed + 1)
248 self.bbox = geom.Box2I(offset, shape)
249 if not self.bbox.contains(0, 0):
250 raise ValueError(f"The bounding box must contain the coordinate (0, 0). {repr(self.bbox)}")
251 self.wcs = self.makeDummyWcs()
253 # Set up properties of the simulations
254 nSigmaForKernel = 5
255 self.kernelSize = (int(maxPsfSize*nSigmaForKernel + 0.5)//2)*2 + 1 # make sure it is odd
257 bufferSize = self.kernelSize//2
258 x0, y0 = self.bbox.getBegin()
259 xSize, ySize = self.bbox.getDimensions()
260 # Set the pixel coordinates and fluxes of the simulated sources.
261 self.xLoc = self.rngData.random(nSrc)*(xSize - 2*bufferSize) + bufferSize + x0
262 self.yLoc = self.rngData.random(nSrc)*(ySize - 2*bufferSize) + bufferSize + y0
263 self.flux = (self.rngData.random(nSrc)*(fluxRange - 1.) + 1.)*sourceSigma*noiseLevel
265 self.backgroundLevel = backgroundLevel
266 self.noiseLevel = noiseLevel
267 self.minPsfSize = minPsfSize
268 self.maxPsfSize = maxPsfSize
269 self.detector = DetectorWrapper(name=f"detector {ccd}", id=ccd).detector
271 def setDummyCoaddInputs(self, exposure, expId):
272 """Generate an `ExposureCatalog` as though the exposures had been
273 processed using `warpAndPsfMatch`.
275 Parameters
276 ----------
277 exposure : `lsst.afw.image.Exposure`
278 The exposure to construct a `CoaddInputs` `ExposureCatalog` for.
279 expId : `int`
280 A unique identifier for the visit.
281 """
282 badPixelMask = afwImage.Mask.getPlaneBitMask(self.badMaskPlanes)
283 nGoodPix = np.sum(exposure.getMask().getArray() & badPixelMask == 0)
285 config = CoaddInputRecorderConfig()
286 inputRecorder = CoaddInputRecorderTask(config=config, name="inputRecorder")
287 tempExpInputRecorder = inputRecorder.makeCoaddTempExpRecorder(expId, num=1)
288 tempExpInputRecorder.addCalExp(exposure, expId, nGoodPix)
289 tempExpInputRecorder.finish(exposure, nGoodPix=nGoodPix)
291 def makeCoaddTempExp(self, rawExposure, visitInfo, expId):
292 """Add the metadata required by `AssembleCoaddTask` to an exposure.
294 Parameters
295 ----------
296 rawExposure : `lsst.afw.image.Exposure`
297 The simulated exposure.
298 visitInfo : `lsst.afw.image.VisitInfo`
299 VisitInfo containing metadata for the exposure.
300 expId : `int`
301 A unique identifier for the visit.
303 Returns
304 -------
305 tempExp : `lsst.afw.image.Exposure`
306 The exposure, with all of the metadata needed for coaddition.
307 """
308 tempExp = rawExposure.clone()
309 tempExp.setWcs(self.wcs)
311 tempExp.setFilter(self.filterLabel)
312 tempExp.setPhotoCalib(self.photoCalib)
313 tempExp.getInfo().setVisitInfo(visitInfo)
314 tempExp.getInfo().setDetector(self.detector)
315 self.setDummyCoaddInputs(tempExp, expId)
316 return tempExp
318 def makeDummyWcs(self, rotAngle=None, pixelScale=None, crval=None, flipX=True):
319 """Make a World Coordinate System object for testing.
321 Parameters
322 ----------
323 rotAngle : `lsst.geom.Angle`
324 Rotation of the CD matrix, East from North
325 pixelScale : `lsst.geom.Angle`
326 Pixel scale of the projection.
327 crval : `lsst.afw.geom.SpherePoint`
328 Coordinates of the reference pixel of the wcs.
329 flipX : `bool`, optional
330 Flip the direction of increasing Right Ascension.
332 Returns
333 -------
334 wcs : `lsst.afw.geom.skyWcs.SkyWcs`
335 A wcs that matches the inputs.
336 """
337 if rotAngle is None:
338 rotAngle = self.rotAngle
339 if pixelScale is None:
340 pixelScale = self.pixelScale
341 if crval is None:
342 crval = geom.SpherePoint(self.ra, self.dec)
343 crpix = geom.Box2D(self.bbox).getCenter()
344 cdMatrix = afwGeom.makeCdMatrix(scale=pixelScale, orientation=rotAngle, flipX=flipX)
345 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
346 return wcs
348 def makeDummyVisitInfo(self, exposureId, randomizeTime=False):
349 """Make a self-consistent visitInfo object for testing.
351 Parameters
352 ----------
353 exposureId : `int`, optional
354 Unique integer identifier for this observation.
355 randomizeTime : `bool`, optional
356 Add a random offset within a 6 hour window to the observation time.
358 Returns
359 -------
360 visitInfo : `lsst.afw.image.VisitInfo`
361 VisitInfo for the exposure.
362 """
363 lsstLat = -30.244639*u.degree
364 lsstLon = -70.749417*u.degree
365 lsstAlt = 2663.*u.m
366 lsstTemperature = 20.*u.Celsius
367 lsstHumidity = 40. # in percent
368 lsstPressure = 73892.*u.pascal
369 loc = EarthLocation(lat=lsstLat,
370 lon=lsstLon,
371 height=lsstAlt)
373 time = Time(2000.0, format="jyear", scale="tt")
374 if randomizeTime:
375 # Pick a random time within a 6 hour window
376 time += 6*u.hour*(self.rngMods.random() - 0.5)
377 radec = SkyCoord(dec=self.dec.asDegrees(), ra=self.ra.asDegrees(),
378 unit='deg', obstime=time, frame='icrs', location=loc)
379 airmass = float(1.0/np.sin(radec.altaz.alt))
380 obsInfo = makeObservationInfo(location=loc,
381 detector_exposure_id=exposureId,
382 datetime_begin=time,
383 datetime_end=time,
384 boresight_airmass=airmass,
385 boresight_rotation_angle=Angle(0.*u.degree),
386 boresight_rotation_coord='sky',
387 temperature=lsstTemperature,
388 pressure=lsstPressure,
389 relative_humidity=lsstHumidity,
390 tracking_radec=radec,
391 altaz_begin=radec.altaz,
392 observation_type='science',
393 )
394 visitInfo = MakeRawVisitInfoViaObsInfo.observationInfo2visitInfo(obsInfo)
395 return visitInfo
397 def makeTestImage(self, expId, noiseLevel=None, psfSize=None, backgroundLevel=None,
398 detectionSigma=5., badRegionBox=None):
399 """Make a reproduceable PSF-convolved masked image for testing.
401 Parameters
402 ----------
403 expId : `int`
404 A unique identifier to use to refer to the visit.
405 noiseLevel : `float`, optional
406 Standard deviation of the noise to add to each pixel.
407 psfSize : `float`, optional
408 Width of the PSF of the simulated sources, in pixels.
409 backgroundLevel : `float`, optional
410 Background value added to all pixels in the simulated images.
411 detectionSigma : `float`, optional
412 Threshold amplitude of the image to set the "DETECTED" mask.
413 badRegionBox : `lsst.geom.Box2I`, optional
414 Add a bad region bounding box (set to "BAD").
415 """
416 if backgroundLevel is None:
417 backgroundLevel = self.backgroundLevel
418 if noiseLevel is None:
419 noiseLevel = 5.
420 visitInfo = self.makeDummyVisitInfo(expId, randomizeTime=True)
422 if psfSize is None:
423 psfSize = self.rngMods.random()*(self.maxPsfSize - self.minPsfSize) + self.minPsfSize
424 nSrc = len(self.flux)
425 sigmas = [psfSize for src in range(nSrc)]
426 sigmasPsfMatched = [self.maxPsfSize for src in range(nSrc)]
427 coordList = list(zip(self.xLoc, self.yLoc, self.flux, sigmas))
428 coordListPsfMatched = list(zip(self.xLoc, self.yLoc, self.flux, sigmasPsfMatched))
429 xSize, ySize = self.bbox.getDimensions()
430 model = plantSources(self.bbox, self.kernelSize, self.backgroundLevel,
431 coordList, addPoissonNoise=False)
432 modelPsfMatched = plantSources(self.bbox, self.kernelSize, self.backgroundLevel,
433 coordListPsfMatched, addPoissonNoise=False)
434 model.variance.array = np.abs(model.image.array) + noiseLevel
435 modelPsfMatched.variance.array = np.abs(modelPsfMatched.image.array) + noiseLevel
436 noise = self.rngData.random((ySize, xSize))*noiseLevel
437 noise -= np.median(noise)
438 model.image.array += noise
439 modelPsfMatched.image.array += noise
440 detectedMask = afwImage.Mask.getPlaneBitMask("DETECTED")
441 detectionThreshold = self.backgroundLevel + detectionSigma*noiseLevel
442 model.mask.array[model.image.array > detectionThreshold] += detectedMask
444 if badRegionBox is not None:
445 model.mask[badRegionBox] = afwImage.Mask.getPlaneBitMask("BAD")
447 exposure = self.makeCoaddTempExp(model, visitInfo, expId)
448 matchedExposure = self.makeCoaddTempExp(modelPsfMatched, visitInfo, expId)
449 return exposure, matchedExposure
451 @staticmethod
452 def makeDataRefList(exposures, matchedExposures, warpType, tract=0, patch=42, coaddName="deep"):
453 """Make data references from the simulated exposures that can be
454 retrieved using the Gen 3 Butler API.
456 Parameters
457 ----------
458 warpType : `str`
459 Either 'direct' or 'psfMatched'.
460 tract : `int`, optional
461 Unique identifier for a tract of a skyMap.
462 patch : `int`, optional
463 Unique identifier for a subdivision of a tract.
464 coaddName : `str`, optional
465 The type of coadd being produced. Typically 'deep'.
467 Returns
468 -------
469 dataRefList : `list` of `MockWarpReference`
470 The data references.
472 Raises
473 ------
474 ValueError
475 If an unknown `warpType` is supplied.
476 """
477 dataRefList = []
478 for expId in exposures:
479 if warpType == 'direct':
480 exposure = exposures[expId]
481 elif warpType == 'psfMatched':
482 exposure = matchedExposures[expId]
483 else:
484 raise ValueError("warpType must be one of 'direct' or 'psfMatched'")
485 dataRef = MockWarpReference(exposure, coaddName=coaddName,
486 tract=tract, patch=patch, visit=expId)
487 dataRefList.append(dataRef)
488 return dataRefList