lsst.pipe.tasks  16.0-32-g044e2860
mockCoadd.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import lsst.afw.image
23 import lsst.afw.geom
24 import lsst.pex.config
25 import lsst.afw.table
26 import lsst.pipe.base
27 from lsst.pipe.tasks.makeSkyMap import MakeSkyMapTask
28 from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask
29 from lsst.pipe.tasks.assembleCoadd import (AssembleCoaddTask, SafeClipAssembleCoaddTask,
30  CompareWarpAssembleCoaddTask)
31 from .mockObject import MockObjectTask
32 from .mockObservation import MockObservationTask
33 from .mockSelect import MockSelectImagesTask
34 
35 
36 class MockCoaddConfig(lsst.pex.config.Config):
37  makeSkyMap = lsst.pex.config.ConfigurableField(
38  doc="SkyMap builder subtask",
39  target=MakeSkyMapTask
40  )
41  mockObject = lsst.pex.config.ConfigurableField(
42  doc="Subtask that generates and draws the objects/sources in the mock images",
43  target=MockObjectTask
44  )
45  mockObservation = lsst.pex.config.ConfigurableField(
46  doc="Subtask that generates the Wcs, Psf, Calib, etc. of mock images",
47  target=MockObservationTask
48  )
49  coaddName = lsst.pex.config.Field(
50  doc="Coadd name used as a prefix for other datasets",
51  dtype=str,
52  optional=False,
53  default="deep"
54  )
55  nObservations = lsst.pex.config.Field(
56  doc="Number of mock observations to generate.",
57  dtype=int,
58  optional=False,
59  default=12
60  )
61  edgeBuffer = lsst.pex.config.Field(
62  doc=("Number of pixels by which to grow object bounding boxes when determining whether they land "
63  " completely on a generated image"),
64  dtype=int,
65  optional=False,
66  default=5
67  )
68 
69  def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2*lsst.afw.geom.arcseconds):
70  """
71  Set the nested [discrete] skymap config parameters such that the full tract
72  has nPatches x nPatches patches of the given size and pixel scale.
73  """
74  self.makeSkyMap.skyMap['discrete'].patchInnerDimensions = [patchSize, patchSize]
75  self.makeSkyMap.skyMap['discrete'].pixelScale = pixelScale.asArcseconds()
76  # multiply by 0.5 because we want a half-width; subtract 0.49 to ensure that we get the right
77  # number after skyMap.TractInfo rounds up.
78  radius = (0.5 * nPatches - 0.49) * patchSize * pixelScale.asDegrees()
79  self.makeSkyMap.skyMap['discrete'].radiusList = [radius]
80 
81  def setDefaults(self):
82  self.makeSkyMap.skyMap.name = 'discrete'
83  self.makeSkyMap.skyMap['discrete'].raList = [90.0]
84  self.makeSkyMap.skyMap['discrete'].decList = [0.0]
85  self.makeSkyMap.skyMap['discrete'].patchBorder = 10
86  self.makeSkyMap.skyMap['discrete'].projection = "TAN"
87  self.makeSkyMap.skyMap['discrete'].tractOverlap = 0.0
88  self.setupSkyMapPatches()
89 
90 
91 class MockCoaddTask(lsst.pipe.base.CmdLineTask):
92  """MockCoaddTask is a driver task for creating mock coadds. As opposed to more realistic
93  simulations, MockCoadd generates and uses extremely simple "toy" data that can be used to more
94  rigorously test the behavior of high-level task code because the expected results are
95  more easily predicted. In particular, calexps are generated directly from the truth catalog,
96  and contain only zero-noise stars that are created using the same Psf, Calib, and Wcs that will
97  be attached to the mock calexp.
98 
99  In addition to creating the mock calexps and truth catalogs, MockCoadd also contains driver
100  code to run the MakeSkyMap, MakeCoaddTempExp, and AssembleCoadd tasks on the mock calexps,
101  and code to directly create a mock coadd image using CoaddPsf, which can be compared to the
102  output of the regular coadd tasks to check that the coadd code and CoaddPsf are consistent.
103 
104  Note that aside from MakeSkyMapTask, the coadd tasks are *not* subtasks of MockCoaddTasks,
105  and their configs are not part of MockCoaddConfig; these are created locally within
106  MockCoaddTask methods when needed, as not all coadd task config options are appropriate
107  for the mock data generated by MockCoadd.
108  """
109 
110  ConfigClass = MockCoaddConfig
111 
112  _DefaultName = "MockCoadd"
113 
114  def __init__(self, **kwds):
115  """Construct a MockCoaddTask and the subtasks used for generating skymaps, objects,
116  and observations (i.e. calexp parameters).
117  """
118  lsst.pipe.base.CmdLineTask.__init__(self, **kwds)
119  self.makeSubtask("makeSkyMap")
120  self.makeSubtask("mockObject")
121  self.makeSubtask("mockObservation")
123  self.objectIdKey = self.schema.addField("objectId", type="L", doc="foreign key to truth catalog")
124  self.exposureIdKey = self.schema.addField("exposureId", type="L",
125  doc="foreign key to observation catalog")
126  self.centroidInBBoxKey = self.schema.addField(
127  "centroidInBBox", type="Flag",
128  doc="set if this source's center position is inside the generated image's bbox"
129  )
130  self.partialOverlapKey = self.schema.addField(
131  "partialOverlap", type="Flag",
132  doc="set if this source was not completely inside the generated image"
133  )
134 
135  def buildSkyMap(self, butler):
136  """Build the skymap for the mock dataset."""
137  return self.makeSkyMap.runDataRef(butler.dataRef(self.config.coaddName + "Coadd_skyMap")).skyMap
138 
139  def buildTruthCatalog(self, butler=None, skyMap=None, tract=0):
140  """Create and save (if butler is not None) a truth catalog containing all the mock objects.
141 
142  Must be run after buildSkyMap.
143 
144  Most of the work is delegated to the mockObject subtask.
145  """
146  if skyMap is None:
147  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
148  catalog = self.mockObject.run(tractInfo=skyMap[tract])
149  if butler is not None:
150  butler.put(catalog, "truth", tract=tract)
151  return catalog
152 
153  def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None):
154  """Create and save (if butler is not None) an ExposureCatalog of simulated observations,
155  containing the Psfs, Wcss, Calibs, etc. of the calexps to be simulated.
156 
157  Must be run after buildSkyMap.
158 
159  Most of the work is delegated to the mockObservation subtask.
160  """
161  if skyMap is None:
162  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
163  if camera is None:
164  camera = butler.get("camera")
165  catalog = self.mockObservation.run(butler=butler,
166  n=self.config.nObservations, camera=camera,
167  tractInfo=skyMap[tract])
168  catalog.sort()
169  if butler is not None:
170  butler.put(catalog, "observations", tract=tract)
171  return catalog
172 
173  def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0):
174  """Use the truth catalog and observation catalog to create and save (if butler is not None)
175  mock calexps and an ExposureCatalog ('simsrc') that contains information about which objects
176  appear partially or fully in each exposure.
177 
178  Must be run after buildTruthCatalog and buildObservationCatalog.
179  """
180  if obsCatalog is None:
181  obsCatalog = butler.get("observations", tract=tract)
182  if truthCatalog is None:
183  truthCatalog = butler.get("truth", tract=tract)
184  ccdKey = obsCatalog.getSchema().find("ccd").key
185  visitKey = obsCatalog.getSchema().find("visit").key
186  simSrcCatalog = lsst.afw.table.SimpleCatalog(self.schema)
187  for obsRecord in obsCatalog:
188  ccd = obsRecord.getI(ccdKey)
189  visit = obsRecord.getI(visitKey)
190  self.log.info("Generating image for visit={visit}, ccd={ccd}".format(ccd=ccd, visit=visit))
191  exposure = lsst.afw.image.ExposureF(obsRecord.getBBox())
192  exposure.setCalib(obsRecord.getCalib())
193  exposure.setWcs(obsRecord.getWcs())
194  exposure.setPsf(obsRecord.getPsf())
195  exposure.getInfo().setApCorrMap(obsRecord.getApCorrMap())
196  exposure.getInfo().setTransmissionCurve(obsRecord.getTransmissionCurve())
197  for truthRecord in truthCatalog:
198  status = self.mockObject.drawSource(truthRecord, exposure, buffer=self.config.edgeBuffer)
199  if status:
200  simSrcRecord = simSrcCatalog.addNew()
201  simSrcRecord.setCoord(truthRecord.getCoord())
202  simSrcRecord.setL(self.objectIdKey, truthRecord.getId())
203  simSrcRecord.setL(self.exposureIdKey, obsRecord.getId())
204  simSrcRecord.setFlag(self.centroidInBBoxKey, obsRecord.contains(truthRecord.getCoord()))
205  simSrcRecord.setFlag(self.partialOverlapKey, status == 1)
206  self.log.info(" added object {id}".format(id=truthRecord.getId()))
207  exposure.getMaskedImage().getVariance().set(1.0)
208  if butler is not None:
209  butler.put(exposure, "calexp", ccd=ccd, visit=visit)
210  if butler is not None:
211  butler.put(simSrcCatalog, "simsrc", tract=tract)
212  return simSrcCatalog
213 
214  def buildAllInputs(self, butler):
215  """Convenience function that calls buildSkyMap, buildObservationCatalog, buildTruthCatalog,
216  and buildInputImages.
217  """
218  skyMap = self.buildSkyMap(butler)
219  observations = self.buildObservationCatalog(butler, skyMap=skyMap)
220  truth = self.buildTruthCatalog(butler, skyMap=skyMap)
221  self.buildInputImages(butler, obsCatalog=observations, truthCatalog=truth)
222 
223  def makeCoaddTask(self, cls, assemblePsfMatched=False):
224  """Helper function to create a Coadd task with configuration appropriate for the simulations.
225 
226  MockCoaddTask does not include MakeCoaddTempExpTask or AssembleCoaddTask as subtasks, because
227  we want explicit control over their configs, rather than leaving this up to the user.
228  However, we have to install our own SelectImages task for both of these, so it made sense
229  to have a single method that would create one of these two tasks, set the config values we
230  want, and install the custom SelectImagesTask.
231  """
232  config = cls.ConfigClass()
233  config.coaddName = self.config.coaddName
234  config.select.retarget(MockSelectImagesTask)
235  if cls == MakeCoaddTempExpTask:
236  config.bgSubtracted = True
237  config.makeDirect = True
238  config.makePsfMatched = True
239  config.modelPsf.defaultFwhm = 9
240  config.modelPsf.addWing = False
241  config.warpAndPsfMatch.psfMatch.kernel['AL'].scaleByFwhm = False
242  config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize = 25
243  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64
244  config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64
245 
246  elif cls in [AssembleCoaddTask, SafeClipAssembleCoaddTask, CompareWarpAssembleCoaddTask]:
247  if assemblePsfMatched:
248  config.warpType = 'psfMatched'
249  if cls != AssembleCoaddTask:
250  config.doWrite = False
251  if cls == CompareWarpAssembleCoaddTask:
252  config.assembleStaticSkyModel.select.retarget(MockSelectImagesTask)
253  config.doAttachTransmissionCurve = True
254  return cls(config=config)
255 
256  def iterPatchRefs(self, butler, tractInfo):
257  """Generator that iterates over the patches in a tract, yielding dataRefs.
258  """
259  nPatchX, nPatchY = tractInfo.getNumPatches()
260  for iPatchX in range(nPatchX):
261  for iPatchY in range(nPatchY):
262  patchRef = butler.dataRef(self.config.coaddName + "Coadd",
263  tract=tractInfo.getId(), patch="%d,%d" % (iPatchX, iPatchY),
264  filter='r')
265  yield patchRef
266 
267  def buildCoadd(self, butler, skyMap=None, tract=0):
268  """Run the coadd tasks (MakeCoaddTempExp and AssembleCoadd) on the mock data.
269 
270  Must be run after buildInputImages.
271  Makes both direct and PSF-matched coadds
272  """
273  if skyMap is None:
274  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
275  tractInfo = skyMap[tract]
276  makeCoaddTempExpTask = self.makeCoaddTask(MakeCoaddTempExpTask)
277  directCoaddTaskList = []
278  for coaddTask in [SafeClipAssembleCoaddTask, CompareWarpAssembleCoaddTask, AssembleCoaddTask]:
279  directCoaddTaskList.append(self.makeCoaddTask(coaddTask))
280  assemblePsfMatchedCoaddTask = self.makeCoaddTask(AssembleCoaddTask, assemblePsfMatched=True)
281  for patchRef in self.iterPatchRefs(butler, tractInfo):
282  makeCoaddTempExpTask.runDataRef(patchRef)
283  for patchRef in self.iterPatchRefs(butler, tractInfo):
284  for directCoaddTask in directCoaddTaskList:
285  directCoaddTask.runDataRef(patchRef)
286  assemblePsfMatchedCoaddTask.runDataRef(patchRef)
287 
288  def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0):
289  """Directly create a simulation of the coadd, using the CoaddPsf (and ModelPsf)
290  of the direct (and psfMatched) coadd exposure and the truth catalog.
291 
292  Must be run after buildCoadd.
293  """
294  if truthCatalog is None:
295  truthCatalog = butler.get("truth", tract=tract)
296  if skyMap is None:
297  skyMap = butler.get(self.config.coaddName + "Coadd_skyMap")
298  tractInfo = skyMap[tract]
299  for patchRef in self.iterPatchRefs(butler, tractInfo):
300  for dataProduct in ["Coadd", "CoaddPsfMatched"]:
301  exposure = patchRef.get(self.config.coaddName + dataProduct)
302  exposure.getMaskedImage().getImage().set(0.0)
304  exposure.getInfo().getCoaddInputs().ccds, exposure.getWcs()
305  )
306  exposure.setPsf(coaddPsf)
307  for truthRecord in truthCatalog:
308  self.mockObject.drawSource(truthRecord, exposure, buffer=0)
309  patchRef.put(exposure, self.config.coaddName + dataProduct + "_mock")
310 
311 
312 def run(root):
313  """Convenience function to create and run MockCoaddTask with default settings.
314  """
315  from .simpleMapper import makeDataRepo
316  butler = makeDataRepo(root=root)
317  task = MockCoaddTask()
318  task.buildAllInputs(butler)
319  task.buildCoadd(butler)
320  task.buildMockCoadd(butler)
static Schema makeMinimalSchema()
def buildTruthCatalog(self, butler=None, skyMap=None, tract=0)
Definition: mockCoadd.py:139
def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None)
Definition: mockCoadd.py:153
def makeCoaddTask(self, cls, assemblePsfMatched=False)
Definition: mockCoadd.py:223
def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2 *lsst.afw.geom.arcseconds)
Definition: mockCoadd.py:69
def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0)
Definition: mockCoadd.py:173
def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0)
Definition: mockCoadd.py:288
def iterPatchRefs(self, butler, tractInfo)
Definition: mockCoadd.py:256
def buildCoadd(self, butler, skyMap=None, tract=0)
Definition: mockCoadd.py:267