lsst.pipe.tasks  15.0-6-g9a9df217+7
mockCoadd.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 from builtins import range
25 import lsst.afw.image
26 import lsst.afw.geom
27 import lsst.pex.config
28 import lsst.afw.table
29 import lsst.pipe.base
30 from lsst.pipe.tasks.makeSkyMap import MakeSkyMapTask
31 from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask
32 from lsst.pipe.tasks.assembleCoadd import (AssembleCoaddTask, SafeClipAssembleCoaddTask,
33  CompareWarpAssembleCoaddTask)
34 from .mockObject import MockObjectTask
35 from .mockObservation import MockObservationTask
36 from .mockSelect import MockSelectImagesTask
37 
38 
39 class MockCoaddConfig(lsst.pex.config.Config):
40  makeSkyMap = lsst.pex.config.ConfigurableField(
41  doc="SkyMap builder subtask",
42  target=MakeSkyMapTask
43  )
44  mockObject = lsst.pex.config.ConfigurableField(
45  doc="Subtask that generates and draws the objects/sources in the mock images",
46  target=MockObjectTask
47  )
48  mockObservation = lsst.pex.config.ConfigurableField(
49  doc="Subtask that generates the Wcs, Psf, Calib, etc. of mock images",
50  target=MockObservationTask
51  )
52  coaddName = lsst.pex.config.Field(
53  doc="Coadd name used as a prefix for other datasets",
54  dtype=str,
55  optional=False,
56  default="deep"
57  )
58  nObservations = lsst.pex.config.Field(
59  doc="Number of mock observations to generate.",
60  dtype=int,
61  optional=False,
62  default=12
63  )
64  edgeBuffer = lsst.pex.config.Field(
65  doc=("Number of pixels by which to grow object bounding boxes when determining whether they land "
66  " completely on a generated image"),
67  dtype=int,
68  optional=False,
69  default=5
70  )
71 
72  def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2*lsst.afw.geom.arcseconds):
73  """
74  Set the nested [discrete] skymap config parameters such that the full tract
75  has nPatches x nPatches patches of the given size and pixel scale.
76  """
77  self.makeSkyMap.skyMap['discrete'].patchInnerDimensions = [patchSize, patchSize]
78  self.makeSkyMap.skyMap['discrete'].pixelScale = pixelScale.asArcseconds()
79  # multiply by 0.5 because we want a half-width; subtract 0.49 to ensure that we get the right
80  # number after skyMap.TractInfo rounds up.
81  radius = (0.5 * nPatches - 0.49) * patchSize * pixelScale.asDegrees()
82  self.makeSkyMap.skyMap['discrete'].radiusList = [radius]
83 
84  def setDefaults(self):
85  self.makeSkyMap.skyMap.name = 'discrete'
86  self.makeSkyMap.skyMap['discrete'].raList = [90.0]
87  self.makeSkyMap.skyMap['discrete'].decList = [0.0]
88  self.makeSkyMap.skyMap['discrete'].patchBorder = 10
89  self.makeSkyMap.skyMap['discrete'].projection = "TAN"
90  self.makeSkyMap.skyMap['discrete'].tractOverlap = 0.0
91  self.setupSkyMapPatches()
92 
93 
94 class MockCoaddTask(lsst.pipe.base.CmdLineTask):
95  """MockCoaddTask is a driver task for creating mock coadds. As opposed to more realistic
96  simulations, MockCoadd generates and uses extremely simple "toy" data that can be used to more
97  rigorously test the behavior of high-level task code because the expected results are
98  more easily predicted. In particular, calexps are generated directly from the truth catalog,
99  and contain only zero-noise stars that are created using the same Psf, Calib, and Wcs that will
100  be attached to the mock calexp.
101 
102  In addition to creating the mock calexps and truth catalogs, MockCoadd also contains driver
103  code to run the MakeSkyMap, MakeCoaddTempExp, and AssembleCoadd tasks on the mock calexps,
104  and code to directly create a mock coadd image using CoaddPsf, which can be compared to the
105  output of the regular coadd tasks to check that the coadd code and CoaddPsf are consistent.
106 
107  Note that aside from MakeSkyMapTask, the coadd tasks are *not* subtasks of MockCoaddTasks,
108  and their configs are not part of MockCoaddConfig; these are created locally within
109  MockCoaddTask methods when needed, as not all coadd task config options are appropriate
110  for the mock data generated by MockCoadd.
111  """
112 
113  ConfigClass = MockCoaddConfig
114 
115  _DefaultName = "MockCoadd"
116 
117  def __init__(self, **kwds):
118  """Construct a MockCoaddTask and the subtasks used for generating skymaps, objects,
119  and observations (i.e. calexp parameters).
120  """
121  lsst.pipe.base.CmdLineTask.__init__(self, **kwds)
122  self.makeSubtask("makeSkyMap")
123  self.makeSubtask("mockObject")
124  self.makeSubtask("mockObservation")
126  self.objectIdKey = self.schema.addField("objectId", type="L", doc="foreign key to truth catalog")
127  self.exposureIdKey = self.schema.addField("exposureId", type="L",
128  doc="foreign key to observation catalog")
129  self.centroidInBBoxKey = self.schema.addField(
130  "centroidInBBox", type="Flag",
131  doc="set if this source's center position is inside the generated image's bbox"
132  )
133  self.partialOverlapKey = self.schema.addField(
134  "partialOverlap", type="Flag",
135  doc="set if this source was not completely inside the generated image"
136  )
137 
138  def buildSkyMap(self, butler):
139  """Build the skymap for the mock dataset."""
140  return self.makeSkyMap.run(butler.dataRef(self.config.coaddName + "Coadd_skyMap")).skyMap
141 
142  def buildTruthCatalog(self, butler=None, skyMap=None, tract=0):
143  """Create and save (if butler is not None) a truth catalog containing all the mock objects.
144 
145  Must be run after buildSkyMap.
146 
147  Most of the work is delegated to the mockObject subtask.
148  """
149  if skyMap is None:
150  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
151  catalog = self.mockObject.run(tractInfo=skyMap[tract])
152  if butler is not None:
153  butler.put(catalog, "truth", tract=tract)
154  return catalog
155 
156  def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None):
157  """Create and save (if butler is not None) an ExposureCatalog of simulated observations,
158  containing the Psfs, Wcss, Calibs, etc. of the calexps to be simulated.
159 
160  Must be run after buildSkyMap.
161 
162  Most of the work is delegated to the mockObservation subtask.
163  """
164  if skyMap is None:
165  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
166  if camera is None:
167  camera = butler.get("camera")
168  catalog = self.mockObservation.run(butler=butler,
169  n=self.config.nObservations, camera=camera,
170  tractInfo=skyMap[tract])
171  if butler is not None:
172  butler.put(catalog, "observations", tract=tract)
173  return catalog
174 
175  def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0):
176  """Use the truth catalog and observation catalog to create and save (if butler is not None)
177  mock calexps and an ExposureCatalog ('simsrc') that contains information about which objects
178  appear partially or fully in each exposure.
179 
180  Must be run after buildTruthCatalog and buildObservationCatalog.
181  """
182  if obsCatalog is None:
183  obsCatalog = butler.get("observations", tract=tract)
184  if truthCatalog is None:
185  truthCatalog = butler.get("truth", tract=tract)
186  ccdKey = obsCatalog.getSchema().find("ccd").key
187  visitKey = obsCatalog.getSchema().find("visit").key
188  simSrcCatalog = lsst.afw.table.SimpleCatalog(self.schema)
189  for obsRecord in obsCatalog:
190  ccd = obsRecord.getI(ccdKey)
191  visit = obsRecord.getI(visitKey)
192  self.log.info("Generating image for visit={visit}, ccd={ccd}".format(ccd=ccd, visit=visit))
193  exposure = lsst.afw.image.ExposureF(obsRecord.getBBox())
194  exposure.setCalib(obsRecord.getCalib())
195  exposure.setWcs(obsRecord.getWcs())
196  exposure.setPsf(obsRecord.getPsf())
197  exposure.getInfo().setApCorrMap(obsRecord.getApCorrMap())
198  exposure.getInfo().setTransmissionCurve(obsRecord.getTransmissionCurve())
199  for truthRecord in truthCatalog:
200  status = self.mockObject.drawSource(truthRecord, exposure, buffer=self.config.edgeBuffer)
201  if status:
202  simSrcRecord = simSrcCatalog.addNew()
203  simSrcRecord.setCoord(truthRecord.getCoord())
204  simSrcRecord.setL(self.objectIdKey, truthRecord.getId())
205  simSrcRecord.setL(self.exposureIdKey, obsRecord.getId())
206  simSrcRecord.setFlag(self.centroidInBBoxKey, obsRecord.contains(truthRecord.getCoord()))
207  simSrcRecord.setFlag(self.partialOverlapKey, status == 1)
208  self.log.info(" added object {id}".format(id=truthRecord.getId()))
209  exposure.getMaskedImage().getVariance().set(1.0)
210  if butler is not None:
211  butler.put(exposure, "calexp", ccd=ccd, visit=visit)
212  if butler is not None:
213  butler.put(simSrcCatalog, "simsrc", tract=tract)
214  return simSrcCatalog
215 
216  def buildAllInputs(self, butler):
217  """Convenience function that calls buildSkyMap, buildObservationCatalog, buildTruthCatalog,
218  and buildInputImages.
219  """
220  skyMap = self.buildSkyMap(butler)
221  observations = self.buildObservationCatalog(butler, skyMap=skyMap)
222  truth = self.buildTruthCatalog(butler, skyMap=skyMap)
223  simSrcCatalog = self.buildInputImages(butler, obsCatalog=observations, truthCatalog=truth)
224 
225  def makeCoaddTask(self, cls, assemblePsfMatched=False):
226  """Helper function to create a Coadd task with configuration appropriate for the simulations.
227 
228  MockCoaddTask does not include MakeCoaddTempExpTask or AssembleCoaddTask as subtasks, because
229  we want explicit control over their configs, rather than leaving this up to the user.
230  However, we have to install our own SelectImages task for both of these, so it made sense
231  to have a single method that would create one of these two tasks, set the config values we
232  want, and install the custom SelectImagesTask.
233  """
234  config = cls.ConfigClass()
235  config.coaddName = self.config.coaddName
236  config.select.retarget(MockSelectImagesTask)
237  if cls == MakeCoaddTempExpTask:
238  config.bgSubtracted = True
239  config.makeDirect = True
240  config.makePsfMatched = True
241  config.modelPsf.defaultFwhm = 9
242  config.modelPsf.addWing = False
243  config.warpAndPsfMatch.psfMatch.kernel['AL'].scaleByFwhm = False
244  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize = 25
245  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64
246  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64
247 
248  elif cls in [AssembleCoaddTask, SafeClipAssembleCoaddTask, CompareWarpAssembleCoaddTask]:
249  if assemblePsfMatched:
250  config.warpType = 'psfMatched'
251  if cls != AssembleCoaddTask:
252  config.doWrite = False
253  if cls == CompareWarpAssembleCoaddTask:
254  config.assembleStaticSkyModel.select.retarget(MockSelectImagesTask)
255  config.doAttachTransmissionCurve = True
256  return cls(config=config)
257 
258  def iterPatchRefs(self, butler, tractInfo):
259  """Generator that iterates over the patches in a tract, yielding dataRefs.
260  """
261  nPatchX, nPatchY = tractInfo.getNumPatches()
262  for iPatchX in range(nPatchX):
263  for iPatchY in range(nPatchY):
264  patchRef = butler.dataRef(self.config.coaddName + "Coadd",
265  tract=tractInfo.getId(), patch="%d,%d" % (iPatchX, iPatchY),
266  filter='r')
267  yield patchRef
268 
269  def buildCoadd(self, butler, skyMap=None, tract=0):
270  """Run the coadd tasks (MakeCoaddTempExp and AssembleCoadd) on the mock data.
271 
272  Must be run after buildInputImages.
273  Makes both direct and PSF-matched coadds
274  """
275  if skyMap is None:
276  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
277  tractInfo = skyMap[tract]
278  makeCoaddTempExpTask = self.makeCoaddTask(MakeCoaddTempExpTask)
279  directCoaddTaskList = []
280  for coaddTask in [SafeClipAssembleCoaddTask, CompareWarpAssembleCoaddTask, AssembleCoaddTask]:
281  directCoaddTaskList.append(self.makeCoaddTask(coaddTask))
282  assemblePsfMatchedCoaddTask = self.makeCoaddTask(AssembleCoaddTask, assemblePsfMatched=True)
283  for patchRef in self.iterPatchRefs(butler, tractInfo):
284  makeCoaddTempExpTask.run(patchRef)
285  for patchRef in self.iterPatchRefs(butler, tractInfo):
286  for directCoaddTask in directCoaddTaskList:
287  directCoaddTask.run(patchRef)
288  assemblePsfMatchedCoaddTask.run(patchRef)
289 
290  def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0):
291  """Directly create a simulation of the coadd, using the CoaddPsf (and ModelPsf)
292  of the direct (and psfMatched) coadd exposure and the truth catalog.
293 
294  Must be run after buildCoadd.
295  """
296  if truthCatalog is None:
297  truthCatalog = butler.get("truth", tract=tract)
298  if skyMap is None:
299  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
300  tractInfo = skyMap[tract]
301  tractWcs = tractInfo.getWcs()
302  for patchRef in self.iterPatchRefs(butler, tractInfo):
303  # TODO: pybind11 remove `immediate=True` once DM-9112 is resolved
304  for dataProduct in ["Coadd", "CoaddPsfMatched"]:
305  exposure = patchRef.get(self.config.coaddName + dataProduct, immediate=True)
306  exposure.getMaskedImage().getImage().set(0.0)
308  exposure.getInfo().getCoaddInputs().ccds, exposure.getWcs()
309  )
310  exposure.setPsf(coaddPsf)
311  for truthRecord in truthCatalog:
312  self.mockObject.drawSource(truthRecord, exposure, buffer=0)
313  patchRef.put(exposure, self.config.coaddName + dataProduct + "_mock")
314 
315 
316 def run(root):
317  """Convenience function to create and run MockCoaddTask with default settings.
318  """
319  from .simpleMapper import makeDataRepo
320  butler = makeDataRepo(root=root)
321  task = MockCoaddTask()
322  task.buildAllInputs(butler)
323  task.buildCoadd(butler)
324  task.buildMockCoadd(butler)
static Schema makeMinimalSchema()
def buildTruthCatalog(self, butler=None, skyMap=None, tract=0)
Definition: mockCoadd.py:142
def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None)
Definition: mockCoadd.py:156
def makeCoaddTask(self, cls, assemblePsfMatched=False)
Definition: mockCoadd.py:225
def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2 *lsst.afw.geom.arcseconds)
Definition: mockCoadd.py:72
def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0)
Definition: mockCoadd.py:175
def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0)
Definition: mockCoadd.py:290
def iterPatchRefs(self, butler, tractInfo)
Definition: mockCoadd.py:258
def buildCoadd(self, butler, skyMap=None, tract=0)
Definition: mockCoadd.py:269