1 from __future__
import absolute_import, division, print_function
3 from builtins
import zip
4 from builtins
import map
6 import lsst.afw.image
as afwImage
7 import lsst.afw.geom
as afwGeom
8 from lsst.afw.fits.fitsLib
import FitsError
9 from lsst.ctrl.pool.parallel
import BatchPoolTask
10 from lsst.ctrl.pool.pool
import Pool, abortOnError, NODE
11 from lsst.geom
import convexHull
12 from lsst.pex.config
import Config, Field, ConfigurableField
13 from lsst.pipe.base
import Struct, ArgumentParser
14 from lsst.pipe.tasks.coaddBase
import CoaddTaskRunner
15 from lsst.pipe.tasks.makeCoaddTempExp
import MakeCoaddTempExpTask
16 from lsst.pipe.tasks.multiBand
import DetectCoaddSourcesTask
17 from lsst.pipe.tasks.selectImages
import WcsSelectImagesTask
18 from lsst.pipe.tasks.assembleCoadd
import SafeClipAssembleCoaddTask
23 coaddName = Field(dtype=str, default=
"deep", doc=
"Name for coadd")
24 select = ConfigurableField(
25 target=WcsSelectImagesTask, doc=
"Select images to process")
26 makeCoaddTempExp = ConfigurableField(
27 target=MakeCoaddTempExpTask, doc=
"Warp images to sky")
28 doBackgroundReference = Field(
29 dtype=bool, default=
False, doc=
"Build background reference?")
30 backgroundReference = ConfigurableField(
31 target=NullSelectImagesTask, doc=
"Build background reference")
32 assembleCoadd = ConfigurableField(
33 target=SafeClipAssembleCoaddTask, doc=
"Assemble warps into coadd")
34 doDetection = Field(dtype=bool, default=
True,
35 doc=
"Run detection on the coaddition product")
36 detectCoaddSources = ConfigurableField(
37 target=DetectCoaddSourcesTask, doc=
"Detect sources on coadd")
38 doOverwriteCoadd = Field(dtype=bool, default=
False, doc=
"Overwrite coadd?")
41 self.makeCoaddTempExp.select.retarget(NullSelectImagesTask)
42 self.assembleCoadd.select.retarget(NullSelectImagesTask)
43 self.makeCoaddTempExp.doOverwrite =
False
44 self.assembleCoadd.doWrite =
False
45 self.assembleCoadd.doMatchBackgrounds =
False
46 self.makeCoaddTempExp.bgSubtracted =
True
47 self.assembleCoadd.badMaskPlanes = [
48 'BAD',
'EDGE',
'SAT',
'INTRP',
'NO_DATA']
51 if self.makeCoaddTempExp.coaddName != self.
coaddName:
53 "makeCoaddTempExp.coaddName and coaddName don't match")
54 if self.assembleCoadd.coaddName != self.
coaddName:
56 "assembleCoadd.coaddName and coaddName don't match")
63 """!Get bare butler into Task
65 @param parsedCmd results of parsing command input
67 kwargs[
"butler"] = parsedCmd.butler
68 kwargs[
"selectIdList"] = [
69 ref.dataId
for ref
in parsedCmd.selectId.refList]
70 return [(parsedCmd.id.refList, kwargs), ]
74 ConfigClass = CoaddDriverConfig
75 _DefaultName =
"coaddDriver"
76 RunnerClass = CoaddDriverTaskRunner
79 BatchPoolTask.__init__(self, *args, **kwargs)
80 self.makeSubtask(
"select")
81 self.makeSubtask(
"makeCoaddTempExp")
82 self.makeSubtask(
"backgroundReference")
83 self.makeSubtask(
"assembleCoadd")
84 self.makeSubtask(
"detectCoaddSources")
87 def _makeArgumentParser(cls, **kwargs):
88 """!Build argument parser
90 Selection references are not cheap (reads Wcs), so are generated
91 only if we're not doing a batch submission.
93 parser = ArgumentParser(name=cls._DefaultName)
94 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2",
95 ContainerClass=TractDataIdContainer)
96 parser.add_id_argument(
97 "--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9")
103 Return walltime request for batch job
105 @param time: Requested time per iteration
106 @param parsedCmd: Results of argument parsing
107 @param numCores: Number of cores
108 @return float walltime request length
110 numTargets = len(parsedCmd.selectId.refList)
111 return time*numTargets/float(numCores)
114 def run(self, tractPatchRefList, butler, selectIdList=[]):
115 """!Determine which tracts are non-empty before processing
117 @param tractPatchRefList: List of tracts and patches to include in the coaddition
118 @param butler: butler reference object
119 @param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd
120 @return list of references to sel.runTract function evaluation for each tractPatchRefList member
122 pool = Pool(
"tracts")
123 pool.storeSet(butler=butler, skymap=butler.get(
124 self.config.coaddName +
"Coadd_skyMap"))
126 for patchRefList
in tractPatchRefList:
127 tractSet = set([patchRef.dataId[
"tract"]
128 for patchRef
in patchRefList])
129 assert len(tractSet) == 1
130 tractIdList.append(tractSet.pop())
132 selectDataList = [data
for data
in pool.mapNoBalance(self.
readSelection, selectIdList)
if
134 nonEmptyList = pool.mapNoBalance(
136 tractPatchRefList = [patchRefList
for patchRefList, nonEmpty
in
137 zip(tractPatchRefList, nonEmptyList)
if nonEmpty]
138 self.log.info(
"Non-empty tracts (%d): %s" % (len(tractPatchRefList),
139 [patchRefList[0].dataId[
"tract"]
for patchRefList
in
143 for data
in selectDataList:
144 data.dataRef =
getDataRef(butler, data.dataId,
"calexp")
147 return [self.
runTract(patchRefList, butler, selectDataList)
for patchRefList
in tractPatchRefList]
150 def runTract(self, patchRefList, butler, selectDataList=[]):
151 """!Run stacking on a tract
153 This method only runs on the master node.
155 @param patchRefList: List of patch data references for tract
156 @param butler: Data butler
157 @param selectDataList: List of SelectStruct for inputs
159 pool = Pool(
"stacker")
161 pool.storeSet(butler=butler, warpType=self.config.coaddName +
"Coadd_directWarp",
162 coaddType=self.config.coaddName +
"Coadd")
163 patchIdList = [patchRef.dataId
for patchRef
in patchRefList]
165 selectedData = pool.map(self.
warp, patchIdList, selectDataList)
166 if self.config.doBackgroundReference:
167 self.backgroundReference.run(patchRefList, selectDataList)
169 def refNamer(patchRef):
170 return tuple(map(int, patchRef.dataId[
"patch"].split(
",")))
172 lookup = dict(zip(map(refNamer, patchRefList), selectedData))
173 coaddData = [Struct(patchId=patchRef.dataId, selectDataList=lookup[refNamer(patchRef)])
for
174 patchRef
in patchRefList]
175 pool.map(self.
coadd, coaddData)
178 """!Read Wcs of selected inputs
180 This method only runs on slave nodes.
181 This method is similar to SelectDataIdContainer.makeDataRefList,
182 creating a Struct like a SelectStruct, except with a dataId instead
183 of a dataRef (to ease MPI).
185 @param cache: Pool cache
186 @param selectId: Data identifier for selected input
187 @return a SelectStruct with a dataId instead of dataRef
190 ref =
getDataRef(cache.butler, selectId,
"calexp")
191 self.log.info(
"Reading Wcs from %s" % (selectId,))
192 md = ref.get(
"calexp_md", immediate=
True)
193 wcs = afwImage.makeWcs(md)
194 data = Struct(dataId=selectId, wcs=wcs, bbox=afwImage.bboxFromMetadata(md))
196 self.log.warn(
"Unable to construct Wcs from %s" % (selectId,))
201 """!Check whether a tract has any overlapping inputs
203 This method only runs on slave nodes.
205 @param cache: Pool cache
206 @param tractId: Data identifier for tract
207 @param selectDataList: List of selection data
208 @return whether tract has any overlapping inputs
210 def makePolygon(wcs, bbox):
211 """Return a polygon for the image, given Wcs and bounding box"""
212 return convexHull([wcs.pixelToSky(afwGeom.Point2D(coord)).getVector()
for
213 coord
in bbox.getCorners()])
215 skymap = cache.skymap
216 tract = skymap[tractId]
217 tractWcs = tract.getWcs()
218 tractPoly = makePolygon(tractWcs, tract.getBBox())
220 for selectData
in selectIdList:
221 if not hasattr(selectData,
"poly"):
222 selectData.poly = makePolygon(selectData.wcs, selectData.bbox)
223 if tractPoly.intersects(selectData.poly):
227 def warp(self, cache, patchId, selectDataList):
228 """!Warp all images for a patch
230 Only slave nodes execute this method.
232 Because only one argument may be passed, it is expected to
233 contain multiple elements, which are:
235 @param patchRef: data reference for patch
236 @param selectDataList: List of SelectStruct for inputs
237 @return selectDataList with non-overlapping elements removed
239 patchRef =
getDataRef(cache.butler, patchId, cache.coaddType)
241 with self.logOperation(
"warping %s" % (patchRef.dataId,), catch=
True):
242 self.makeCoaddTempExp.run(patchRef, selectDataList)
243 return selectDataList
246 """!Construct coadd for a patch and measure
248 Only slave nodes execute this method.
250 Because only one argument may be passed, it is expected to
251 contain multiple elements, which are:
253 @param patchRef: data reference for patch
254 @param selectDataList: List of SelectStruct for inputs
256 patchRef =
getDataRef(cache.butler, data.patchId, cache.coaddType)
257 selectDataList = data.selectDataList
259 with self.logOperation(
"coadding %s" % (patchRef.dataId,), catch=
True):
260 if self.config.doOverwriteCoadd
or not patchRef.datasetExists(cache.coaddType):
261 coaddResults = self.assembleCoadd.run(patchRef, selectDataList)
262 if coaddResults
is not None:
263 coadd = coaddResults.coaddExposure
264 elif patchRef.datasetExists(cache.coaddType):
265 self.log.info(
"%s: Reading coadd %s" % (NODE, patchRef.dataId))
266 coadd = patchRef.get(cache.coaddType, immediate=
True)
275 if self.config.doDetection:
276 with self.logOperation(
"detection on {}".format(patchRef.dataId),
278 idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
281 detResults = self.detectCoaddSources.runDetection(coadd, idFactory)
282 self.detectCoaddSources.write(coadd, detResults, patchRef)
284 patchRef.put(coadd, self.assembleCoadd.config.coaddName+
"Coadd")
287 """!Select exposures to operate upon, via the SelectImagesTask
289 This is very similar to CoaddBaseTask.selectExposures, except we return
290 a list of SelectStruct (same as the input), so we can plug the results into
291 future uses of SelectImagesTask.
293 @param patchRef data reference to a particular patch
294 @param selectDataList list of references to specific data products (i.e. visit, ccd)
295 @return filtered list of SelectStruct
298 return tuple(dataRef.dataId[k]
for k
in sorted(dataRef.dataId))
299 inputs = dict((key(select.dataRef), select)
300 for select
in selectDataList)
301 skyMap = patchRef.get(self.config.coaddName +
"Coadd_skyMap")
302 tract = skyMap[patchRef.dataId[
"tract"]]
303 patch = tract[(tuple(int(i)
304 for i
in patchRef.dataId[
"patch"].split(
",")))]
305 bbox = patch.getOuterBBox()
307 cornerPosList = afwGeom.Box2D(bbox).getCorners()
308 coordList = [wcs.pixelToSky(pos)
for pos
in cornerPosList]
309 dataRefList = self.select.runDataRef(
310 patchRef, coordList, selectDataList=selectDataList).dataRefList
311 return [inputs[key(dataRef)]
for dataRef
in dataRefList]
def selectExposures
Select exposures to operate upon, via the SelectImagesTask.
def run
Determine which tracts are non-empty before processing.
def warp
Warp all images for a patch.
def getTargetList
Get bare butler into Task.
def batchWallTime
Return walltime request for batch job.
def coadd
Construct coadd for a patch and measure.
def readSelection
Read Wcs of selected inputs.
def runTract
Run stacking on a tract.
def checkTract
Check whether a tract has any overlapping inputs.