lsst.pipe.tasks  13.0-64-gd461ca2f+2
mockCoadd.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 from builtins import range
25 import lsst.afw.image
26 import lsst.afw.geom
27 import lsst.pex.config
28 import lsst.afw.table
29 import lsst.pipe.base
30 from lsst.pipe.tasks.makeSkyMap import MakeSkyMapTask
31 from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask
32 from lsst.pipe.tasks.assembleCoadd import AssembleCoaddTask
33 from .mockObject import MockObjectTask
34 from .mockObservation import MockObservationTask
35 from .mockSelect import MockSelectImagesTask
36 
37 
38 class MockCoaddConfig(lsst.pex.config.Config):
39  makeSkyMap = lsst.pex.config.ConfigurableField(
40  doc="SkyMap builder subtask",
41  target=MakeSkyMapTask
42  )
43  mockObject = lsst.pex.config.ConfigurableField(
44  doc="Subtask that generates and draws the objects/sources in the mock images",
45  target=MockObjectTask
46  )
47  mockObservation = lsst.pex.config.ConfigurableField(
48  doc="Subtask that generates the Wcs, Psf, Calib, etc. of mock images",
49  target=MockObservationTask
50  )
51  coaddName = lsst.pex.config.Field(
52  doc="Coadd name used as a prefix for other datasets",
53  dtype=str,
54  optional=False,
55  default="deep"
56  )
57  nObservations = lsst.pex.config.Field(
58  doc="Number of mock observations to generate.",
59  dtype=int,
60  optional=False,
61  default=12
62  )
63  edgeBuffer = lsst.pex.config.Field(
64  doc=("Number of pixels by which to grow object bounding boxes when determining whether they land "
65  " completely on a generated image"),
66  dtype=int,
67  optional=False,
68  default=5
69  )
70 
71  def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2*lsst.afw.geom.arcseconds):
72  """
73  Set the nested [discrete] skymap config parameters such that the full tract
74  has nPatches x nPatches patches of the given size and pixel scale.
75  """
76  self.makeSkyMap.skyMap['discrete'].patchInnerDimensions = [patchSize, patchSize]
77  self.makeSkyMap.skyMap['discrete'].pixelScale = pixelScale.asArcseconds()
78  # multiply by 0.5 because we want a half-width; subtract 0.49 to ensure that we get the right
79  # number after skyMap.TractInfo rounds up.
80  radius = (0.5 * nPatches - 0.49) * patchSize * pixelScale.asDegrees()
81  self.makeSkyMap.skyMap['discrete'].radiusList = [radius]
82 
83  def setDefaults(self):
84  self.makeSkyMap.skyMap.name = 'discrete'
85  self.makeSkyMap.skyMap['discrete'].raList = [90.0]
86  self.makeSkyMap.skyMap['discrete'].decList = [0.0]
87  self.makeSkyMap.skyMap['discrete'].patchBorder = 10
88  self.makeSkyMap.skyMap['discrete'].projection = "TAN"
89  self.makeSkyMap.skyMap['discrete'].tractOverlap = 0.0
90  self.setupSkyMapPatches()
91 
92 
93 class MockCoaddTask(lsst.pipe.base.CmdLineTask):
94  """MockCoaddTask is a driver task for creating mock coadds. As opposed to more realistic
95  simulations, MockCoadd generates and uses extremely simple "toy" data that can be used to more
96  rigorously test the behavior of high-level task code because the expected results are
97  more easily predicted. In particular, calexps are generated directly from the truth catalog,
98  and contain only zero-noise stars that are created using the same Psf, Calib, and Wcs that will
99  be attached to the mock calexp.
100 
101  In addition to creating the mock calexps and truth catalogs, MockCoadd also contains driver
102  code to run the MakeSkyMap, MakeCoaddTempExp, and AssembleCoadd tasks on the mock calexps,
103  and code to directly create a mock coadd image using CoaddPsf, which can be compared to the
104  output of the regular coadd tasks to check that the coadd code and CoaddPsf are consistent.
105 
106  Note that aside from MakeSkyMapTask, the coadd tasks are *not* subtasks of MockCoaddTasks,
107  and their configs are not part of MockCoaddConfig; these are created locally within
108  MockCoaddTask methods when needed, as not all coadd task config options are appropriate
109  for the mock data generated by MockCoadd.
110  """
111 
112  ConfigClass = MockCoaddConfig
113 
114  _DefaultName = "MockCoadd"
115 
116  def __init__(self, **kwds):
117  """Construct a MockCoaddTask and the subtasks used for generating skymaps, objects,
118  and observations (i.e. calexp parameters).
119  """
120  lsst.pipe.base.CmdLineTask.__init__(self, **kwds)
121  self.makeSubtask("makeSkyMap")
122  self.makeSubtask("mockObject")
123  self.makeSubtask("mockObservation")
124  self.schema = lsst.afw.table.SimpleTable.makeMinimalSchema()
125  self.objectIdKey = self.schema.addField("objectId", type="L", doc="foreign key to truth catalog")
126  self.exposureIdKey = self.schema.addField("exposureId", type="L",
127  doc="foreign key to observation catalog")
128  self.centroidInBBoxKey = self.schema.addField(
129  "centroidInBBox", type="Flag",
130  doc="set if this source's center position is inside the generated image's bbox"
131  )
132  self.partialOverlapKey = self.schema.addField(
133  "partialOverlap", type="Flag",
134  doc="set if this source was not completely inside the generated image"
135  )
136 
137  def buildSkyMap(self, butler):
138  """Build the skymap for the mock dataset."""
139  return self.makeSkyMap.run(butler.dataRef(self.config.coaddName + "Coadd_skyMap")).skyMap
140 
141  def buildTruthCatalog(self, butler=None, skyMap=None, tract=0):
142  """Create and save (if butler is not None) a truth catalog containing all the mock objects.
143 
144  Must be run after buildSkyMap.
145 
146  Most of the work is delegated to the mockObject subtask.
147  """
148  if skyMap is None:
149  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
150  catalog = self.mockObject.run(tractInfo=skyMap[tract])
151  if butler is not None:
152  butler.put(catalog, "truth", tract=tract)
153  return catalog
154 
155  def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None):
156  """Create and save (if butler is not None) an ExposureCatalog of simulated observations,
157  containing the Psfs, Wcss, Calibs, etc. of the calexps to be simulated.
158 
159  Must be run after buildSkyMap.
160 
161  Most of the work is delegated to the mockObservation subtask.
162  """
163  if skyMap is None:
164  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
165  if camera is None:
166  camera = butler.get("camera")
167  catalog = self.mockObservation.run(butler=butler,
168  n=self.config.nObservations, camera=camera,
169  tractInfo=skyMap[tract])
170  if butler is not None:
171  butler.put(catalog, "observations", tract=tract)
172  return catalog
173 
174  def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0):
175  """Use the truth catalog and observation catalog to create and save (if butler is not None)
176  mock calexps and an ExposureCatalog ('simsrc') that contains information about which objects
177  appear partially or fully in each exposure.
178 
179  Must be run after buildTruthCatalog and buildObservationCatalog.
180  """
181  if obsCatalog is None:
182  obsCatalog = butler.get("observations", tract=tract)
183  if truthCatalog is None:
184  truthCatalog = butler.get("truth", tract=tract)
185  ccdKey = obsCatalog.getSchema().find("ccd").key
186  visitKey = obsCatalog.getSchema().find("visit").key
187  simSrcCatalog = lsst.afw.table.SimpleCatalog(self.schema)
188  for obsRecord in obsCatalog:
189  ccd = obsRecord.getI(ccdKey)
190  visit = obsRecord.getI(visitKey)
191  self.log.info("Generating image for visit={visit}, ccd={ccd}".format(ccd=ccd, visit=visit))
192  exposure = lsst.afw.image.ExposureF(obsRecord.getBBox())
193  exposure.setCalib(obsRecord.getCalib())
194  exposure.setWcs(obsRecord.getWcs())
195  exposure.setPsf(obsRecord.getPsf())
196  exposure.getInfo().setApCorrMap(obsRecord.getApCorrMap())
197  for truthRecord in truthCatalog:
198  status = self.mockObject.drawSource(truthRecord, exposure, buffer=self.config.edgeBuffer)
199  if status:
200  simSrcRecord = simSrcCatalog.addNew()
201  simSrcRecord.setCoord(truthRecord.getCoord())
202  simSrcRecord.setL(self.objectIdKey, truthRecord.getId())
203  simSrcRecord.setL(self.exposureIdKey, obsRecord.getId())
204  simSrcRecord.setFlag(self.centroidInBBoxKey, obsRecord.contains(truthRecord.getCoord()))
205  simSrcRecord.setFlag(self.partialOverlapKey, status == 1)
206  self.log.info(" added object {id}".format(id=truthRecord.getId()))
207  exposure.getMaskedImage().getVariance().set(1.0)
208  if butler is not None:
209  butler.put(exposure, "calexp", ccd=ccd, visit=visit)
210  if butler is not None:
211  butler.put(simSrcCatalog, "simsrc", tract=tract)
212  return simSrcCatalog
213 
214  def buildAllInputs(self, butler):
215  """Convenience function that calls buildSkyMap, buildObservationCatalog, buildTruthCatalog,
216  and buildInputImages.
217  """
218  skyMap = self.buildSkyMap(butler)
219  observations = self.buildObservationCatalog(butler, skyMap=skyMap)
220  truth = self.buildTruthCatalog(butler, skyMap=skyMap)
221  simSrcCatalog = self.buildInputImages(butler, obsCatalog=observations, truthCatalog=truth)
222 
223  def makeCoaddTask(self, cls, assemblePsfMatched=False):
224  """Helper function to create a Coadd task with configuration appropriate for the simulations.
225 
226  MockCoaddTask does not include MakeCoaddTempExpTask or AssembleCoaddTask as subtasks, because
227  we want explicit control over their configs, rather than leaving this up to the user.
228  However, we have to install our own SelectImages task for both of these, so it made sense
229  to have a single method that would create one of these two tasks, set the config values we
230  want, and install the custom SelectImagesTask.
231  """
232  config = cls.ConfigClass()
233  config.coaddName = self.config.coaddName
234  config.select.retarget(MockSelectImagesTask)
235  if cls == MakeCoaddTempExpTask:
236  config.bgSubtracted = True
237  config.makeDirect = True
238  config.makePsfMatched = True
239  config.modelPsf.defaultFwhm = 9
240  config.modelPsf.addWing = False
241  config.warpAndPsfMatch.psfMatch.kernel['AL'].scaleByFwhm = False
242  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize = 25
243  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64
244  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64
245 
246  elif cls == AssembleCoaddTask:
247  config.doMatchBackgrounds = False
248  if assemblePsfMatched:
249  config.warpType = 'psfMatched'
250  return cls(config)
251 
252  def iterPatchRefs(self, butler, tractInfo):
253  """Generator that iterates over the patches in a tract, yielding dataRefs.
254  """
255  nPatchX, nPatchY = tractInfo.getNumPatches()
256  for iPatchX in range(nPatchX):
257  for iPatchY in range(nPatchY):
258  patchRef = butler.dataRef(self.config.coaddName + "Coadd",
259  tract=tractInfo.getId(), patch="%d,%d" % (iPatchX, iPatchY),
260  filter='r')
261  yield patchRef
262 
263  def buildCoadd(self, butler, skyMap=None, tract=0):
264  """Run the coadd tasks (MakeCoaddTempExp and AssembleCoadd) on the mock data.
265 
266  Must be run after buildInputImages.
267  Makes both direct and PSF-matched coadds
268  """
269  if skyMap is None:
270  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
271  tractInfo = skyMap[tract]
272  makeCoaddTempExpTask = self.makeCoaddTask(MakeCoaddTempExpTask)
273  assembleCoaddTask = self.makeCoaddTask(AssembleCoaddTask)
274  assemblePsfMatchedCoaddTask = self.makeCoaddTask(AssembleCoaddTask, assemblePsfMatched=True)
275  for patchRef in self.iterPatchRefs(butler, tractInfo):
276  makeCoaddTempExpTask.run(patchRef)
277  for patchRef in self.iterPatchRefs(butler, tractInfo):
278  assembleCoaddTask.run(patchRef)
279  assemblePsfMatchedCoaddTask.run(patchRef)
280 
281  def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0):
282  """Directly create a simulation of the coadd, using the CoaddPsf (and ModelPsf)
283  of the direct (and psfMatched) coadd exposure and the truth catalog.
284 
285  Must be run after buildCoadd.
286  """
287  if truthCatalog is None:
288  truthCatalog = butler.get("truth", tract=tract)
289  if skyMap is None:
290  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
291  tractInfo = skyMap[tract]
292  tractWcs = tractInfo.getWcs()
293  for patchRef in self.iterPatchRefs(butler, tractInfo):
294  # TODO: pybind11 remove `immediate=True` once DM-9112 is resolved
295  for dataProduct in ["Coadd", "CoaddPsfMatched"]:
296  exposure = patchRef.get(self.config.coaddName + dataProduct, immediate=True)
297  exposure.getMaskedImage().getImage().set(0.0)
298  coaddPsf = lsst.meas.algorithms.CoaddPsf(
299  exposure.getInfo().getCoaddInputs().ccds, exposure.getWcs()
300  )
301  exposure.setPsf(coaddPsf)
302  for truthRecord in truthCatalog:
303  self.mockObject.drawSource(truthRecord, exposure, buffer=0)
304  patchRef.put(exposure, self.config.coaddName + dataProduct + "_mock")
305 
306 def run(root):
307  """Convenience function to create and run MockCoaddTask with default settings.
308  """
309  from .simpleMapper import makeDataRepo
310  butler = makeDataRepo(root=root)
311  task = MockCoaddTask()
312  task.buildAllInputs(butler)
313  task.buildCoadd(butler)
314  task.buildMockCoadd(butler)
def buildTruthCatalog(self, butler=None, skyMap=None, tract=0)
Definition: mockCoadd.py:141
def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None)
Definition: mockCoadd.py:155
def makeCoaddTask(self, cls, assemblePsfMatched=False)
Definition: mockCoadd.py:223
def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2 *lsst.afw.geom.arcseconds)
Definition: mockCoadd.py:71
def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0)
Definition: mockCoadd.py:174
def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0)
Definition: mockCoadd.py:281
def iterPatchRefs(self, butler, tractInfo)
Definition: mockCoadd.py:252
def buildCoadd(self, butler, skyMap=None, tract=0)
Definition: mockCoadd.py:263