1 from __future__
import absolute_import, division, print_function
3 from builtins
import zip
4 from builtins
import map
6 import lsst.afw.image
as afwImage
7 import lsst.afw.geom
as afwGeom
8 from lsst.afw.fits.fitsLib
import FitsError
9 from lsst.ctrl.pool.parallel
import BatchPoolTask
10 from lsst.ctrl.pool.pool
import Pool, abortOnError, NODE
11 from lsst.geom
import convexHull
12 from lsst.pex.config
import Config, Field, ConfigurableField
13 from lsst.pipe.base
import Struct, ArgumentParser
14 from lsst.pipe.tasks.coaddBase
import CoaddTaskRunner
15 from lsst.pipe.tasks.makeCoaddTempExp
import MakeCoaddTempExpTask
16 from lsst.pipe.tasks.multiBand
import DetectCoaddSourcesTask
17 from lsst.pipe.tasks.selectImages
import WcsSelectImagesTask
18 from lsst.pipe.tasks.assembleCoadd
import SafeClipAssembleCoaddTask
23 coaddName = Field(dtype=str, default=
"deep", doc=
"Name for coadd")
24 select = ConfigurableField(
25 target=WcsSelectImagesTask, doc=
"Select images to process")
26 makeCoaddTempExp = ConfigurableField(
27 target=MakeCoaddTempExpTask, doc=
"Warp images to sky")
28 doBackgroundReference = Field(
29 dtype=bool, default=
False, doc=
"Build background reference?")
30 backgroundReference = ConfigurableField(
31 target=NullSelectImagesTask, doc=
"Build background reference")
32 assembleCoadd = ConfigurableField(
33 target=SafeClipAssembleCoaddTask, doc=
"Assemble warps into coadd")
34 doDetection = Field(dtype=bool, default=
True,
35 doc=
"Run detection on the coaddition product")
36 detectCoaddSources = ConfigurableField(
37 target=DetectCoaddSourcesTask, doc=
"Detect sources on coadd")
38 doOverwriteCoadd = Field(dtype=bool, default=
False, doc=
"Overwrite coadd?")
48 'BAD',
'EDGE',
'SAT',
'INTRP',
'NO_DATA']
53 "makeCoaddTempExp.coaddName and coaddName don't match")
56 "assembleCoadd.coaddName and coaddName don't match")
63 """!Get bare butler into Task 65 @param parsedCmd results of parsing command input 67 kwargs[
"butler"] = parsedCmd.butler
68 kwargs[
"selectIdList"] = [
69 ref.dataId
for ref
in parsedCmd.selectId.refList]
70 return [(parsedCmd.id.refList, kwargs), ]
74 ConfigClass = CoaddDriverConfig
75 _DefaultName =
"coaddDriver" 76 RunnerClass = CoaddDriverTaskRunner
79 BatchPoolTask.__init__(self, *args, **kwargs)
80 self.makeSubtask(
"select")
81 self.makeSubtask(
"makeCoaddTempExp")
82 self.makeSubtask(
"backgroundReference")
83 self.makeSubtask(
"assembleCoadd")
84 self.makeSubtask(
"detectCoaddSources")
87 def _makeArgumentParser(cls, **kwargs):
88 """!Build argument parser 90 Selection references are not cheap (reads Wcs), so are generated 91 only if we're not doing a batch submission. 94 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2",
95 ContainerClass=TractDataIdContainer)
96 parser.add_id_argument(
97 "--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9")
103 Return walltime request for batch job 105 @param time: Requested time per iteration 106 @param parsedCmd: Results of argument parsing 107 @param numCores: Number of cores 108 @return float walltime request length 110 numTargets = len(parsedCmd.selectId.refList)
111 return time*numTargets/float(numCores)
114 def run(self, tractPatchRefList, butler, selectIdList=[]):
115 """!Determine which tracts are non-empty before processing 117 @param tractPatchRefList: List of tracts and patches to include in the coaddition 118 @param butler: butler reference object 119 @param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd 120 @return list of references to sel.runTract function evaluation for each tractPatchRefList member 122 pool = Pool(
"tracts")
123 pool.storeSet(butler=butler, skymap=butler.get(
124 self.config.coaddName +
"Coadd_skyMap"))
126 for patchRefList
in tractPatchRefList:
127 tractSet = set([patchRef.dataId[
"tract"]
128 for patchRef
in patchRefList])
129 assert len(tractSet) == 1
130 tractIdList.append(tractSet.pop())
132 selectDataList = [data
for data
in pool.mapNoBalance(self.
readSelection, selectIdList)
if 134 nonEmptyList = pool.mapNoBalance(
136 tractPatchRefList = [patchRefList
for patchRefList, nonEmpty
in 137 zip(tractPatchRefList, nonEmptyList)
if nonEmpty]
138 self.log.info(
"Non-empty tracts (%d): %s" % (len(tractPatchRefList),
139 [patchRefList[0].dataId[
"tract"]
for patchRefList
in 143 for data
in selectDataList:
144 data.dataRef =
getDataRef(butler, data.dataId,
"calexp")
147 return [self.
runTract(patchRefList, butler, selectDataList)
for patchRefList
in tractPatchRefList]
150 def runTract(self, patchRefList, butler, selectDataList=[]):
151 """!Run stacking on a tract 153 This method only runs on the master node. 155 @param patchRefList: List of patch data references for tract 156 @param butler: Data butler 157 @param selectDataList: List of SelectStruct for inputs 159 pool = Pool(
"stacker")
161 pool.storeSet(butler=butler, warpType=self.config.coaddName +
"Coadd_directWarp",
162 coaddType=self.config.coaddName +
"Coadd")
163 patchIdList = [patchRef.dataId
for patchRef
in patchRefList]
165 selectedData = pool.map(self.
warp, patchIdList, selectDataList)
166 if self.config.doBackgroundReference:
167 self.backgroundReference.
run(patchRefList, selectDataList)
169 def refNamer(patchRef):
170 return tuple(map(int, patchRef.dataId[
"patch"].split(
",")))
172 lookup = dict(zip(map(refNamer, patchRefList), selectedData))
173 coaddData = [Struct(patchId=patchRef.dataId, selectDataList=lookup[refNamer(patchRef)])
for 174 patchRef
in patchRefList]
175 pool.map(self.
coadd, coaddData)
178 """!Read Wcs of selected inputs 180 This method only runs on slave nodes. 181 This method is similar to SelectDataIdContainer.makeDataRefList, 182 creating a Struct like a SelectStruct, except with a dataId instead 183 of a dataRef (to ease MPI). 185 @param cache: Pool cache 186 @param selectId: Data identifier for selected input 187 @return a SelectStruct with a dataId instead of dataRef 190 ref =
getDataRef(cache.butler, selectId,
"calexp")
191 self.log.info(
"Reading Wcs from %s" % (selectId,))
192 md = ref.get(
"calexp_md", immediate=
True)
193 wcs = afwImage.makeWcs(md)
194 data = Struct(dataId=selectId, wcs=wcs, dims=(
195 md.get(
"NAXIS1"), md.get(
"NAXIS2")))
197 self.log.warn(
"Unable to construct Wcs from %s" % (selectId,))
202 """!Check whether a tract has any overlapping inputs 204 This method only runs on slave nodes. 206 @param cache: Pool cache 207 @param tractId: Data identifier for tract 208 @param selectDataList: List of selection data 209 @return whether tract has any overlapping inputs 211 skymap = cache.skymap
212 tract = skymap[tractId]
213 tractWcs = tract.getWcs()
214 tractPoly = convexHull([tractWcs.pixelToSky(afwGeom.Point2D(coord)).getVector()
for 215 coord
in tract.getBBox().getCorners()])
217 for selectData
in selectIdList:
218 if not hasattr(selectData,
"poly"):
220 dims = selectData.dims
221 box = afwGeom.Box2D(afwGeom.Point2D(0, 0),
222 afwGeom.Point2D(*dims))
223 selectData.poly = convexHull([wcs.pixelToSky(coord).getVector()
224 for coord
in box.getCorners()])
225 if tractPoly.intersects(selectData.poly):
229 def warp(self, cache, patchId, selectDataList):
230 """!Warp all images for a patch 232 Only slave nodes execute this method. 234 Because only one argument may be passed, it is expected to 235 contain multiple elements, which are: 237 @param patchRef: data reference for patch 238 @param selectDataList: List of SelectStruct for inputs 239 @return selectDataList with non-overlapping elements removed 241 patchRef =
getDataRef(cache.butler, patchId, cache.coaddType)
243 with self.logOperation(
"warping %s" % (patchRef.dataId,), catch=
True):
244 self.makeCoaddTempExp.
run(patchRef, selectDataList)
245 return selectDataList
248 """!Construct coadd for a patch and measure 250 Only slave nodes execute this method. 252 Because only one argument may be passed, it is expected to 253 contain multiple elements, which are: 255 @param patchRef: data reference for patch 256 @param selectDataList: List of SelectStruct for inputs 258 patchRef =
getDataRef(cache.butler, data.patchId, cache.coaddType)
259 selectDataList = data.selectDataList
261 with self.logOperation(
"coadding %s" % (patchRef.dataId,), catch=
True):
262 if self.config.doOverwriteCoadd
or not patchRef.datasetExists(cache.coaddType):
263 coaddResults = self.assembleCoadd.
run(patchRef, selectDataList)
264 if coaddResults
is not None:
265 coadd = coaddResults.coaddExposure
266 elif patchRef.datasetExists(cache.coaddType):
267 self.log.info(
"%s: Reading coadd %s" % (NODE, patchRef.dataId))
268 coadd = patchRef.get(cache.coaddType, immediate=
True)
277 if self.config.doDetection:
278 with self.logOperation(
"detection on {}".format(patchRef.dataId),
280 idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
283 detResults = self.detectCoaddSources.runDetection(coadd, idFactory)
284 self.detectCoaddSources.write(coadd, detResults, patchRef)
286 patchRef.put(coadd, self.assembleCoadd.config.coaddName+
"Coadd")
289 """!Select exposures to operate upon, via the SelectImagesTask 291 This is very similar to CoaddBaseTask.selectExposures, except we return 292 a list of SelectStruct (same as the input), so we can plug the results into 293 future uses of SelectImagesTask. 295 @param patchRef data reference to a particular patch 296 @param selectDataList list of references to specific data products (i.e. visit, ccd) 297 @return filtered list of SelectStruct 300 return tuple(dataRef.dataId[k]
for k
in sorted(dataRef.dataId))
301 inputs = dict((key(select.dataRef), select)
302 for select
in selectDataList)
303 skyMap = patchRef.get(self.config.coaddName +
"Coadd_skyMap")
304 tract = skyMap[patchRef.dataId[
"tract"]]
305 patch = tract[(tuple(int(i)
306 for i
in patchRef.dataId[
"patch"].split(
",")))]
307 bbox = patch.getOuterBBox()
309 cornerPosList = afwGeom.Box2D(bbox).getCorners()
310 coordList = [wcs.pixelToSky(pos)
for pos
in cornerPosList]
311 dataRefList = self.select.runDataRef(
312 patchRef, coordList, selectDataList=selectDataList).dataRefList
313 return [inputs[key(dataRef)]
for dataRef
in dataRefList]
def batchWallTime(cls, time, parsedCmd, numCores)
Return walltime request for batch job.
def run(self, tractPatchRefList, butler, selectIdList=[])
Determine which tracts are non-empty before processing.
def selectExposures(self, patchRef, selectDataList)
Select exposures to operate upon, via the SelectImagesTask.
def runTract(self, patchRefList, butler, selectDataList=[])
Run stacking on a tract.
def getDataRef(butler, dataId, datasetType="raw")
def coadd(self, cache, data)
Construct coadd for a patch and measure.
def warp(self, cache, patchId, selectDataList)
Warp all images for a patch.
def getTargetList(parsedCmd, kwargs)
Get bare butler into Task.
def readSelection(self, cache, selectId)
Read Wcs of selected inputs.
def __init__(self, args, kwargs)
def checkTract(self, cache, tractId, selectIdList)
Check whether a tract has any overlapping inputs.
def writeMetadata(self, dataRef)