1 from __future__
import absolute_import, division, print_function
3 from builtins
import zip
4 from builtins
import map
6 import lsst.afw.image
as afwImage
7 import lsst.afw.geom
as afwGeom
8 from lsst.afw.fits.fitsLib
import FitsError
9 from lsst.ctrl.pool.parallel
import BatchPoolTask
10 from lsst.ctrl.pool.pool
import Pool, abortOnError, NODE
11 from lsst.geom
import convexHull
12 from lsst.pex.config
import Config, Field, ConfigurableField
13 from lsst.pipe.base
import Struct, ArgumentParser
14 from lsst.pipe.tasks.coaddBase
import CoaddTaskRunner
15 from lsst.pipe.tasks.makeCoaddTempExp
import MakeCoaddTempExpTask
16 from lsst.pipe.tasks.multiBand
import DetectCoaddSourcesTask
17 from lsst.pipe.tasks.selectImages
import WcsSelectImagesTask
18 from lsst.pipe.tasks.assembleCoadd
import SafeClipAssembleCoaddTask
23 coaddName = Field(dtype=str, default=
"deep", doc=
"Name for coadd")
24 select = ConfigurableField(
25 target=WcsSelectImagesTask, doc=
"Select images to process")
26 makeCoaddTempExp = ConfigurableField(
27 target=MakeCoaddTempExpTask, doc=
"Warp images to sky")
28 doBackgroundReference = Field(
29 dtype=bool, default=
False, doc=
"Build background reference?")
30 backgroundReference = ConfigurableField(
31 target=NullSelectImagesTask, doc=
"Build background reference")
32 assembleCoadd = ConfigurableField(
33 target=SafeClipAssembleCoaddTask, doc=
"Assemble warps into coadd")
34 detectCoaddSources = ConfigurableField(
35 target=DetectCoaddSourcesTask, doc=
"Detect sources on coadd")
36 doOverwriteCoadd = Field(dtype=bool, default=
False, doc=
"Overwrite coadd?")
39 self.makeCoaddTempExp.select.retarget(NullSelectImagesTask)
40 self.assembleCoadd.select.retarget(NullSelectImagesTask)
41 self.makeCoaddTempExp.doOverwrite =
False
42 self.assembleCoadd.doWrite =
False
43 self.assembleCoadd.doMatchBackgrounds =
False
44 self.makeCoaddTempExp.bgSubtracted =
True
45 self.assembleCoadd.badMaskPlanes = [
46 'BAD',
'EDGE',
'SAT',
'INTRP',
'NO_DATA']
49 if self.makeCoaddTempExp.coaddName != self.
coaddName:
51 "makeCoaddTempExp.coaddName and coaddName don't match")
52 if self.assembleCoadd.coaddName != self.
coaddName:
54 "assembleCoadd.coaddName and coaddName don't match")
61 """!Get bare butler into Task
63 @param parsedCmd results of parsing command input
65 kwargs[
"butler"] = parsedCmd.butler
66 kwargs[
"selectIdList"] = [
67 ref.dataId
for ref
in parsedCmd.selectId.refList]
68 return [(parsedCmd.id.refList, kwargs), ]
72 ConfigClass = CoaddDriverConfig
73 _DefaultName =
"coaddDriver"
74 RunnerClass = CoaddDriverTaskRunner
77 BatchPoolTask.__init__(self, *args, **kwargs)
78 self.makeSubtask(
"select")
79 self.makeSubtask(
"makeCoaddTempExp")
80 self.makeSubtask(
"backgroundReference")
81 self.makeSubtask(
"assembleCoadd")
82 self.makeSubtask(
"detectCoaddSources")
85 def _makeArgumentParser(cls, **kwargs):
86 """!Build argument parser
88 Selection references are not cheap (reads Wcs), so are generated
89 only if we're not doing a batch submission.
91 parser = ArgumentParser(name=cls._DefaultName)
92 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2",
93 ContainerClass=TractDataIdContainer)
94 parser.add_id_argument(
95 "--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9")
101 Return walltime request for batch job
103 @param time: Requested time per iteration
104 @param parsedCmd: Results of argument parsing
105 @param numCores: Number of cores
106 @return float walltime request length
108 numTargets = len(parsedCmd.selectId.refList)
109 return time*numTargets/float(numCores)
112 def run(self, tractPatchRefList, butler, selectIdList=[]):
113 """!Determine which tracts are non-empty before processing
115 @param tractPatchRefList: List of tracts and patches to include in the coaddition
116 @param butler: butler reference object
117 @param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd
118 @return list of references to sel.runTract function evaluation for each tractPatchRefList member
120 pool = Pool(
"tracts")
121 pool.storeSet(butler=butler, skymap=butler.get(
122 self.config.coaddName +
"Coadd_skyMap"))
124 for patchRefList
in tractPatchRefList:
125 tractSet = set([patchRef.dataId[
"tract"]
126 for patchRef
in patchRefList])
127 assert len(tractSet) == 1
128 tractIdList.append(tractSet.pop())
130 selectDataList = [data
for data
in pool.mapNoBalance(self.
readSelection, selectIdList)
if
132 nonEmptyList = pool.mapNoBalance(
134 tractPatchRefList = [patchRefList
for patchRefList, nonEmpty
in
135 zip(tractPatchRefList, nonEmptyList)
if nonEmpty]
136 self.log.info(
"Non-empty tracts (%d): %s" % (len(tractPatchRefList),
137 [patchRefList[0].dataId[
"tract"]
for patchRefList
in
141 for data
in selectDataList:
142 data.dataRef =
getDataRef(butler, data.dataId,
"calexp")
145 return [self.
runTract(patchRefList, butler, selectDataList)
for patchRefList
in tractPatchRefList]
148 def runTract(self, patchRefList, butler, selectDataList=[]):
149 """!Run stacking on a tract
151 This method only runs on the master node.
153 @param patchRefList: List of patch data references for tract
154 @param butler: Data butler
155 @param selectDataList: List of SelectStruct for inputs
157 pool = Pool(
"stacker")
159 pool.storeSet(butler=butler, warpType=self.config.coaddName +
"Coadd_tempExp",
160 coaddType=self.config.coaddName +
"Coadd")
161 patchIdList = [patchRef.dataId
for patchRef
in patchRefList]
163 selectedData = pool.map(self.
warp, patchIdList, selectDataList)
164 if self.config.doBackgroundReference:
165 self.backgroundReference.run(patchRefList, selectDataList)
167 def refNamer(patchRef):
168 return tuple(map(int, patchRef.dataId[
"patch"].split(
",")))
170 lookup = dict(zip(map(refNamer, patchRefList), selectedData))
171 coaddData = [Struct(patchId=patchRef.dataId, selectDataList=lookup[refNamer(patchRef)])
for
172 patchRef
in patchRefList]
173 pool.map(self.
coadd, coaddData)
176 """!Read Wcs of selected inputs
178 This method only runs on slave nodes.
179 This method is similar to SelectDataIdContainer.makeDataRefList,
180 creating a Struct like a SelectStruct, except with a dataId instead
181 of a dataRef (to ease MPI).
183 @param cache: Pool cache
184 @param selectId: Data identifier for selected input
185 @return a SelectStruct with a dataId instead of dataRef
188 ref =
getDataRef(cache.butler, selectId,
"calexp")
189 self.log.info(
"Reading Wcs from %s" % (selectId,))
190 md = ref.get(
"calexp_md", immediate=
True)
191 wcs = afwImage.makeWcs(md)
192 data = Struct(dataId=selectId, wcs=wcs, dims=(
193 md.get(
"NAXIS1"), md.get(
"NAXIS2")))
195 self.log.warn(
"Unable to construct Wcs from %s" % (selectId,))
200 """!Check whether a tract has any overlapping inputs
202 This method only runs on slave nodes.
204 @param cache: Pool cache
205 @param tractId: Data identifier for tract
206 @param selectDataList: List of selection data
207 @return whether tract has any overlapping inputs
209 skymap = cache.skymap
210 tract = skymap[tractId]
211 tractWcs = tract.getWcs()
212 tractPoly = convexHull([tractWcs.pixelToSky(afwGeom.Point2D(coord)).getVector()
for
213 coord
in tract.getBBox().getCorners()])
215 for selectData
in selectIdList:
216 if not hasattr(selectData,
"poly"):
218 dims = selectData.dims
219 box = afwGeom.Box2D(afwGeom.Point2D(0, 0),
220 afwGeom.Point2D(*dims))
221 selectData.poly = convexHull([wcs.pixelToSky(coord).getVector()
222 for coord
in box.getCorners()])
223 if tractPoly.intersects(selectData.poly):
227 def warp(self, cache, patchId, selectDataList):
228 """!Warp all images for a patch
230 Only slave nodes execute this method.
232 Because only one argument may be passed, it is expected to
233 contain multiple elements, which are:
235 @param patchRef: data reference for patch
236 @param selectDataList: List of SelectStruct for inputs
237 @return selectDataList with non-overlapping elements removed
239 patchRef =
getDataRef(cache.butler, patchId, cache.coaddType)
241 with self.logOperation(
"warping %s" % (patchRef.dataId,), catch=
True):
242 self.makeCoaddTempExp.run(patchRef, selectDataList)
243 return selectDataList
246 """!Construct coadd for a patch and measure
248 Only slave nodes execute this method.
250 Because only one argument may be passed, it is expected to
251 contain multiple elements, which are:
253 @param patchRef: data reference for patch
254 @param selectDataList: List of SelectStruct for inputs
256 patchRef =
getDataRef(cache.butler, data.patchId, cache.coaddType)
257 selectDataList = data.selectDataList
259 with self.logOperation(
"coadding %s" % (patchRef.dataId,), catch=
True):
260 if self.config.doOverwriteCoadd
or not patchRef.datasetExists(cache.coaddType):
261 coaddResults = self.assembleCoadd.run(patchRef, selectDataList)
262 if coaddResults
is not None:
263 coadd = coaddResults.coaddExposure
264 elif patchRef.datasetExists(cache.coaddType):
265 self.log.info(
"%s: Reading coadd %s" % (NODE, patchRef.dataId))
266 coadd = patchRef.get(cache.coaddType, immediate=
True)
271 with self.logOperation(
"detection on %s" % (patchRef.dataId,), catch=
True):
272 idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
275 detResults = self.detectCoaddSources.runDetection(coadd, idFactory)
276 self.detectCoaddSources.write(coadd, detResults, patchRef)
279 """!Select exposures to operate upon, via the SelectImagesTask
281 This is very similar to CoaddBaseTask.selectExposures, except we return
282 a list of SelectStruct (same as the input), so we can plug the results into
283 future uses of SelectImagesTask.
285 @param patchRef data reference to a particular patch
286 @param selectDataList list of references to specific data products (i.e. visit, ccd)
287 @return filtered list of SelectStruct
290 return tuple(dataRef.dataId[k]
for k
in sorted(dataRef.dataId))
291 inputs = dict((key(select.dataRef), select)
292 for select
in selectDataList)
293 skyMap = patchRef.get(self.config.coaddName +
"Coadd_skyMap")
294 tract = skyMap[patchRef.dataId[
"tract"]]
295 patch = tract[(tuple(int(i)
296 for i
in patchRef.dataId[
"patch"].split(
",")))]
297 bbox = patch.getOuterBBox()
299 cornerPosList = afwGeom.Box2D(bbox).getCorners()
300 coordList = [wcs.pixelToSky(pos)
for pos
in cornerPosList]
301 dataRefList = self.select.runDataRef(
302 patchRef, coordList, selectDataList=selectDataList).dataRefList
303 return [inputs[key(dataRef)]
for dataRef
in dataRefList]
def selectExposures
Select exposures to operate upon, via the SelectImagesTask.
def run
Determine which tracts are non-empty before processing.
def warp
Warp all images for a patch.
def getTargetList
Get bare butler into Task.
def batchWallTime
Return walltime request for batch job.
def coadd
Construct coadd for a patch and measure.
def readSelection
Read Wcs of selected inputs.
def runTract
Run stacking on a tract.
def checkTract
Check whether a tract has any overlapping inputs.