lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
mockCoadd.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 from builtins import range
25 import lsst.afw.image
26 import lsst.afw.geom
27 import lsst.pex.config
28 import lsst.afw.table
29 import lsst.pipe.base
30 from lsst.pipe.tasks.makeSkyMap import MakeSkyMapTask
31 from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask
32 from lsst.pipe.tasks.assembleCoadd import (AssembleCoaddTask, SafeClipAssembleCoaddTask,
33  CompareWarpAssembleCoaddTask)
34 from .mockObject import MockObjectTask
35 from .mockObservation import MockObservationTask
36 from .mockSelect import MockSelectImagesTask
37 
38 
39 class MockCoaddConfig(lsst.pex.config.Config):
40  makeSkyMap = lsst.pex.config.ConfigurableField(
41  doc="SkyMap builder subtask",
42  target=MakeSkyMapTask
43  )
44  mockObject = lsst.pex.config.ConfigurableField(
45  doc="Subtask that generates and draws the objects/sources in the mock images",
46  target=MockObjectTask
47  )
48  mockObservation = lsst.pex.config.ConfigurableField(
49  doc="Subtask that generates the Wcs, Psf, Calib, etc. of mock images",
50  target=MockObservationTask
51  )
52  coaddName = lsst.pex.config.Field(
53  doc="Coadd name used as a prefix for other datasets",
54  dtype=str,
55  optional=False,
56  default="deep"
57  )
58  nObservations = lsst.pex.config.Field(
59  doc="Number of mock observations to generate.",
60  dtype=int,
61  optional=False,
62  default=12
63  )
64  edgeBuffer = lsst.pex.config.Field(
65  doc=("Number of pixels by which to grow object bounding boxes when determining whether they land "
66  " completely on a generated image"),
67  dtype=int,
68  optional=False,
69  default=5
70  )
71 
72  def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2*lsst.afw.geom.arcseconds):
73  """
74  Set the nested [discrete] skymap config parameters such that the full tract
75  has nPatches x nPatches patches of the given size and pixel scale.
76  """
77  self.makeSkyMap.skyMap['discrete'].patchInnerDimensions = [patchSize, patchSize]
78  self.makeSkyMap.skyMap['discrete'].pixelScale = pixelScale.asArcseconds()
79  # multiply by 0.5 because we want a half-width; subtract 0.49 to ensure that we get the right
80  # number after skyMap.TractInfo rounds up.
81  radius = (0.5 * nPatches - 0.49) * patchSize * pixelScale.asDegrees()
82  self.makeSkyMap.skyMap['discrete'].radiusList = [radius]
83 
84  def setDefaults(self):
85  self.makeSkyMap.skyMap.name = 'discrete'
86  self.makeSkyMap.skyMap['discrete'].raList = [90.0]
87  self.makeSkyMap.skyMap['discrete'].decList = [0.0]
88  self.makeSkyMap.skyMap['discrete'].patchBorder = 10
89  self.makeSkyMap.skyMap['discrete'].projection = "TAN"
90  self.makeSkyMap.skyMap['discrete'].tractOverlap = 0.0
91  self.setupSkyMapPatches()
92 
93 
94 class MockCoaddTask(lsst.pipe.base.CmdLineTask):
95  """MockCoaddTask is a driver task for creating mock coadds. As opposed to more realistic
96  simulations, MockCoadd generates and uses extremely simple "toy" data that can be used to more
97  rigorously test the behavior of high-level task code because the expected results are
98  more easily predicted. In particular, calexps are generated directly from the truth catalog,
99  and contain only zero-noise stars that are created using the same Psf, Calib, and Wcs that will
100  be attached to the mock calexp.
101 
102  In addition to creating the mock calexps and truth catalogs, MockCoadd also contains driver
103  code to run the MakeSkyMap, MakeCoaddTempExp, and AssembleCoadd tasks on the mock calexps,
104  and code to directly create a mock coadd image using CoaddPsf, which can be compared to the
105  output of the regular coadd tasks to check that the coadd code and CoaddPsf are consistent.
106 
107  Note that aside from MakeSkyMapTask, the coadd tasks are *not* subtasks of MockCoaddTasks,
108  and their configs are not part of MockCoaddConfig; these are created locally within
109  MockCoaddTask methods when needed, as not all coadd task config options are appropriate
110  for the mock data generated by MockCoadd.
111  """
112 
113  ConfigClass = MockCoaddConfig
114 
115  _DefaultName = "MockCoadd"
116 
117  def __init__(self, **kwds):
118  """Construct a MockCoaddTask and the subtasks used for generating skymaps, objects,
119  and observations (i.e. calexp parameters).
120  """
121  lsst.pipe.base.CmdLineTask.__init__(self, **kwds)
122  self.makeSubtask("makeSkyMap")
123  self.makeSubtask("mockObject")
124  self.makeSubtask("mockObservation")
125  self.schema = lsst.afw.table.SimpleTable.makeMinimalSchema()
126  self.objectIdKey = self.schema.addField("objectId", type="L", doc="foreign key to truth catalog")
127  self.exposureIdKey = self.schema.addField("exposureId", type="L",
128  doc="foreign key to observation catalog")
129  self.centroidInBBoxKey = self.schema.addField(
130  "centroidInBBox", type="Flag",
131  doc="set if this source's center position is inside the generated image's bbox"
132  )
133  self.partialOverlapKey = self.schema.addField(
134  "partialOverlap", type="Flag",
135  doc="set if this source was not completely inside the generated image"
136  )
137 
138  def buildSkyMap(self, butler):
139  """Build the skymap for the mock dataset."""
140  return self.makeSkyMap.run(butler.dataRef(self.config.coaddName + "Coadd_skyMap")).skyMap
141 
142  def buildTruthCatalog(self, butler=None, skyMap=None, tract=0):
143  """Create and save (if butler is not None) a truth catalog containing all the mock objects.
144 
145  Must be run after buildSkyMap.
146 
147  Most of the work is delegated to the mockObject subtask.
148  """
149  if skyMap is None:
150  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
151  catalog = self.mockObject.run(tractInfo=skyMap[tract])
152  if butler is not None:
153  butler.put(catalog, "truth", tract=tract)
154  return catalog
155 
156  def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None):
157  """Create and save (if butler is not None) an ExposureCatalog of simulated observations,
158  containing the Psfs, Wcss, Calibs, etc. of the calexps to be simulated.
159 
160  Must be run after buildSkyMap.
161 
162  Most of the work is delegated to the mockObservation subtask.
163  """
164  if skyMap is None:
165  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
166  if camera is None:
167  camera = butler.get("camera")
168  catalog = self.mockObservation.run(butler=butler,
169  n=self.config.nObservations, camera=camera,
170  tractInfo=skyMap[tract])
171  if butler is not None:
172  butler.put(catalog, "observations", tract=tract)
173  return catalog
174 
175  def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0):
176  """Use the truth catalog and observation catalog to create and save (if butler is not None)
177  mock calexps and an ExposureCatalog ('simsrc') that contains information about which objects
178  appear partially or fully in each exposure.
179 
180  Must be run after buildTruthCatalog and buildObservationCatalog.
181  """
182  if obsCatalog is None:
183  obsCatalog = butler.get("observations", tract=tract)
184  if truthCatalog is None:
185  truthCatalog = butler.get("truth", tract=tract)
186  ccdKey = obsCatalog.getSchema().find("ccd").key
187  visitKey = obsCatalog.getSchema().find("visit").key
188  simSrcCatalog = lsst.afw.table.SimpleCatalog(self.schema)
189  for obsRecord in obsCatalog:
190  ccd = obsRecord.getI(ccdKey)
191  visit = obsRecord.getI(visitKey)
192  self.log.info("Generating image for visit={visit}, ccd={ccd}".format(ccd=ccd, visit=visit))
193  exposure = lsst.afw.image.ExposureF(obsRecord.getBBox())
194  exposure.setCalib(obsRecord.getCalib())
195  exposure.setWcs(obsRecord.getWcs())
196  exposure.setPsf(obsRecord.getPsf())
197  exposure.getInfo().setApCorrMap(obsRecord.getApCorrMap())
198  for truthRecord in truthCatalog:
199  status = self.mockObject.drawSource(truthRecord, exposure, buffer=self.config.edgeBuffer)
200  if status:
201  simSrcRecord = simSrcCatalog.addNew()
202  simSrcRecord.setCoord(truthRecord.getCoord())
203  simSrcRecord.setL(self.objectIdKey, truthRecord.getId())
204  simSrcRecord.setL(self.exposureIdKey, obsRecord.getId())
205  simSrcRecord.setFlag(self.centroidInBBoxKey, obsRecord.contains(truthRecord.getCoord()))
206  simSrcRecord.setFlag(self.partialOverlapKey, status == 1)
207  self.log.info(" added object {id}".format(id=truthRecord.getId()))
208  exposure.getMaskedImage().getVariance().set(1.0)
209  if butler is not None:
210  butler.put(exposure, "calexp", ccd=ccd, visit=visit)
211  if butler is not None:
212  butler.put(simSrcCatalog, "simsrc", tract=tract)
213  return simSrcCatalog
214 
215  def buildAllInputs(self, butler):
216  """Convenience function that calls buildSkyMap, buildObservationCatalog, buildTruthCatalog,
217  and buildInputImages.
218  """
219  skyMap = self.buildSkyMap(butler)
220  observations = self.buildObservationCatalog(butler, skyMap=skyMap)
221  truth = self.buildTruthCatalog(butler, skyMap=skyMap)
222  simSrcCatalog = self.buildInputImages(butler, obsCatalog=observations, truthCatalog=truth)
223 
224  def makeCoaddTask(self, cls, assemblePsfMatched=False):
225  """Helper function to create a Coadd task with configuration appropriate for the simulations.
226 
227  MockCoaddTask does not include MakeCoaddTempExpTask or AssembleCoaddTask as subtasks, because
228  we want explicit control over their configs, rather than leaving this up to the user.
229  However, we have to install our own SelectImages task for both of these, so it made sense
230  to have a single method that would create one of these two tasks, set the config values we
231  want, and install the custom SelectImagesTask.
232  """
233  config = cls.ConfigClass()
234  config.coaddName = self.config.coaddName
235  config.select.retarget(MockSelectImagesTask)
236  if cls == MakeCoaddTempExpTask:
237  config.bgSubtracted = True
238  config.makeDirect = True
239  config.makePsfMatched = True
240  config.modelPsf.defaultFwhm = 9
241  config.modelPsf.addWing = False
242  config.warpAndPsfMatch.psfMatch.kernel['AL'].scaleByFwhm = False
243  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize = 25
244  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64
245  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64
246 
247  elif cls in [AssembleCoaddTask, SafeClipAssembleCoaddTask, CompareWarpAssembleCoaddTask]:
248  if assemblePsfMatched:
249  config.warpType = 'psfMatched'
250  if cls != AssembleCoaddTask:
251  config.doWrite = False
252  if cls == CompareWarpAssembleCoaddTask:
253  config.assembleStaticSkyModel.select.retarget(MockSelectImagesTask)
254  return cls(config)
255 
256  def iterPatchRefs(self, butler, tractInfo):
257  """Generator that iterates over the patches in a tract, yielding dataRefs.
258  """
259  nPatchX, nPatchY = tractInfo.getNumPatches()
260  for iPatchX in range(nPatchX):
261  for iPatchY in range(nPatchY):
262  patchRef = butler.dataRef(self.config.coaddName + "Coadd",
263  tract=tractInfo.getId(), patch="%d,%d" % (iPatchX, iPatchY),
264  filter='r')
265  yield patchRef
266 
267  def buildCoadd(self, butler, skyMap=None, tract=0):
268  """Run the coadd tasks (MakeCoaddTempExp and AssembleCoadd) on the mock data.
269 
270  Must be run after buildInputImages.
271  Makes both direct and PSF-matched coadds
272  """
273  if skyMap is None:
274  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
275  tractInfo = skyMap[tract]
276  makeCoaddTempExpTask = self.makeCoaddTask(MakeCoaddTempExpTask)
277  directCoaddTaskList = []
278  for coaddTask in [SafeClipAssembleCoaddTask, CompareWarpAssembleCoaddTask, AssembleCoaddTask]:
279  directCoaddTaskList.append(self.makeCoaddTask(coaddTask))
280  assemblePsfMatchedCoaddTask = self.makeCoaddTask(AssembleCoaddTask, assemblePsfMatched=True)
281  for patchRef in self.iterPatchRefs(butler, tractInfo):
282  makeCoaddTempExpTask.run(patchRef)
283  for patchRef in self.iterPatchRefs(butler, tractInfo):
284  for directCoaddTask in directCoaddTaskList:
285  directCoaddTask.run(patchRef)
286  assemblePsfMatchedCoaddTask.run(patchRef)
287 
288  def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0):
289  """Directly create a simulation of the coadd, using the CoaddPsf (and ModelPsf)
290  of the direct (and psfMatched) coadd exposure and the truth catalog.
291 
292  Must be run after buildCoadd.
293  """
294  if truthCatalog is None:
295  truthCatalog = butler.get("truth", tract=tract)
296  if skyMap is None:
297  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
298  tractInfo = skyMap[tract]
299  tractWcs = tractInfo.getWcs()
300  for patchRef in self.iterPatchRefs(butler, tractInfo):
301  # TODO: pybind11 remove `immediate=True` once DM-9112 is resolved
302  for dataProduct in ["Coadd", "CoaddPsfMatched"]:
303  exposure = patchRef.get(self.config.coaddName + dataProduct, immediate=True)
304  exposure.getMaskedImage().getImage().set(0.0)
305  coaddPsf = lsst.meas.algorithms.CoaddPsf(
306  exposure.getInfo().getCoaddInputs().ccds, exposure.getWcs()
307  )
308  exposure.setPsf(coaddPsf)
309  for truthRecord in truthCatalog:
310  self.mockObject.drawSource(truthRecord, exposure, buffer=0)
311  patchRef.put(exposure, self.config.coaddName + dataProduct + "_mock")
312 
313 def run(root):
314  """Convenience function to create and run MockCoaddTask with default settings.
315  """
316  from .simpleMapper import makeDataRepo
317  butler = makeDataRepo(root=root)
318  task = MockCoaddTask()
319  task.buildAllInputs(butler)
320  task.buildCoadd(butler)
321  task.buildMockCoadd(butler)
def buildTruthCatalog(self, butler=None, skyMap=None, tract=0)
Definition: mockCoadd.py:142
def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None)
Definition: mockCoadd.py:156
def makeCoaddTask(self, cls, assemblePsfMatched=False)
Definition: mockCoadd.py:224
def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2 *lsst.afw.geom.arcseconds)
Definition: mockCoadd.py:72
def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0)
Definition: mockCoadd.py:175
def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0)
Definition: mockCoadd.py:288
def iterPatchRefs(self, butler, tractInfo)
Definition: mockCoadd.py:256
def buildCoadd(self, butler, skyMap=None, tract=0)
Definition: mockCoadd.py:267