1 from __future__
import absolute_import, division, print_function
3 from builtins
import zip
4 from builtins
import map
6 import lsst.afw.image
as afwImage
7 import lsst.afw.geom
as afwGeom
8 from lsst.afw.fits.fitsLib
import FitsError
9 from lsst.ctrl.pool.parallel
import BatchPoolTask
10 from lsst.ctrl.pool.pool
import Pool, abortOnError, NODE
11 from lsst.geom
import convexHull
12 from lsst.pex.config
import Config, Field, ConfigurableField
13 from lsst.pipe.base
import Struct, ArgumentParser
14 from lsst.pipe.tasks.coaddBase
import CoaddTaskRunner
15 from lsst.pipe.tasks.makeCoaddTempExp
import MakeCoaddTempExpTask
16 from lsst.pipe.tasks.multiBand
import DetectCoaddSourcesTask
17 from lsst.pipe.tasks.selectImages
import WcsSelectImagesTask
18 from lsst.pipe.tasks.assembleCoadd
import SafeClipAssembleCoaddTask
23 coaddName = Field(dtype=str, default=
"deep", doc=
"Name for coadd")
24 select = ConfigurableField(
25 target=WcsSelectImagesTask, doc=
"Select images to process")
26 makeCoaddTempExp = ConfigurableField(
27 target=MakeCoaddTempExpTask, doc=
"Warp images to sky")
28 doBackgroundReference = Field(
29 dtype=bool, default=
False, doc=
"Build background reference?")
30 backgroundReference = ConfigurableField(
31 target=NullSelectImagesTask, doc=
"Build background reference")
32 assembleCoadd = ConfigurableField(
33 target=SafeClipAssembleCoaddTask, doc=
"Assemble warps into coadd")
34 doDetection = Field(dtype=bool, default=
True,
35 doc=
"Run detection on the coaddition product")
36 detectCoaddSources = ConfigurableField(
37 target=DetectCoaddSourcesTask, doc=
"Detect sources on coadd")
38 doOverwriteCoadd = Field(dtype=bool, default=
False, doc=
"Overwrite coadd?")
48 'BAD',
'EDGE',
'SAT',
'INTRP',
'NO_DATA']
53 "makeCoaddTempExp.coaddName and coaddName don't match")
56 "assembleCoadd.coaddName and coaddName don't match")
63 """!Get bare butler into Task 65 @param parsedCmd results of parsing command input 67 kwargs[
"butler"] = parsedCmd.butler
68 kwargs[
"selectIdList"] = [
69 ref.dataId
for ref
in parsedCmd.selectId.refList]
70 return [(parsedCmd.id.refList, kwargs), ]
74 ConfigClass = CoaddDriverConfig
75 _DefaultName =
"coaddDriver" 76 RunnerClass = CoaddDriverTaskRunner
79 BatchPoolTask.__init__(self, *args, **kwargs)
80 self.makeSubtask(
"select")
81 self.makeSubtask(
"makeCoaddTempExp")
82 self.makeSubtask(
"backgroundReference")
83 self.makeSubtask(
"assembleCoadd")
84 self.makeSubtask(
"detectCoaddSources")
87 def _makeArgumentParser(cls, **kwargs):
88 """!Build argument parser 90 Selection references are not cheap (reads Wcs), so are generated 91 only if we're not doing a batch submission. 94 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2",
95 ContainerClass=TractDataIdContainer)
96 parser.add_id_argument(
97 "--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9")
103 Return walltime request for batch job 105 @param time: Requested time per iteration 106 @param parsedCmd: Results of argument parsing 107 @param numCores: Number of cores 108 @return float walltime request length 110 numTargets = len(parsedCmd.selectId.refList)
111 return time*numTargets/float(numCores)
114 def run(self, tractPatchRefList, butler, selectIdList=[]):
115 """!Determine which tracts are non-empty before processing 117 @param tractPatchRefList: List of tracts and patches to include in the coaddition 118 @param butler: butler reference object 119 @param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd 120 @return list of references to sel.runTract function evaluation for each tractPatchRefList member 122 pool = Pool(
"tracts")
123 pool.storeSet(butler=butler, skymap=butler.get(
124 self.config.coaddName +
"Coadd_skyMap"))
126 for patchRefList
in tractPatchRefList:
127 tractSet = set([patchRef.dataId[
"tract"]
128 for patchRef
in patchRefList])
129 assert len(tractSet) == 1
130 tractIdList.append(tractSet.pop())
132 selectDataList = [data
for data
in pool.mapNoBalance(self.
readSelection, selectIdList)
if 134 nonEmptyList = pool.mapNoBalance(
136 tractPatchRefList = [patchRefList
for patchRefList, nonEmpty
in 137 zip(tractPatchRefList, nonEmptyList)
if nonEmpty]
138 self.log.info(
"Non-empty tracts (%d): %s" % (len(tractPatchRefList),
139 [patchRefList[0].dataId[
"tract"]
for patchRefList
in 143 for data
in selectDataList:
144 data.dataRef =
getDataRef(butler, data.dataId,
"calexp")
147 return [self.
runTract(patchRefList, butler, selectDataList)
for patchRefList
in tractPatchRefList]
150 def runTract(self, patchRefList, butler, selectDataList=[]):
151 """!Run stacking on a tract 153 This method only runs on the master node. 155 @param patchRefList: List of patch data references for tract 156 @param butler: Data butler 157 @param selectDataList: List of SelectStruct for inputs 159 pool = Pool(
"stacker")
161 pool.storeSet(butler=butler, warpType=self.config.coaddName +
"Coadd_directWarp",
162 coaddType=self.config.coaddName +
"Coadd")
163 patchIdList = [patchRef.dataId
for patchRef
in patchRefList]
165 selectedData = pool.map(self.
warp, patchIdList, selectDataList)
166 if self.config.doBackgroundReference:
167 self.backgroundReference.
run(patchRefList, selectDataList)
169 def refNamer(patchRef):
170 return tuple(map(int, patchRef.dataId[
"patch"].split(
",")))
172 lookup = dict(zip(map(refNamer, patchRefList), selectedData))
173 coaddData = [Struct(patchId=patchRef.dataId, selectDataList=lookup[refNamer(patchRef)])
for 174 patchRef
in patchRefList]
175 pool.map(self.
coadd, coaddData)
178 """!Read Wcs of selected inputs 180 This method only runs on slave nodes. 181 This method is similar to SelectDataIdContainer.makeDataRefList, 182 creating a Struct like a SelectStruct, except with a dataId instead 183 of a dataRef (to ease MPI). 185 @param cache: Pool cache 186 @param selectId: Data identifier for selected input 187 @return a SelectStruct with a dataId instead of dataRef 190 ref =
getDataRef(cache.butler, selectId,
"calexp")
191 self.log.info(
"Reading Wcs from %s" % (selectId,))
192 md = ref.get(
"calexp_md", immediate=
True)
193 wcs = afwImage.makeWcs(md)
194 data = Struct(dataId=selectId, wcs=wcs, bbox=afwImage.bboxFromMetadata(md))
196 self.log.warn(
"Unable to construct Wcs from %s" % (selectId,))
201 """!Check whether a tract has any overlapping inputs 203 This method only runs on slave nodes. 205 @param cache: Pool cache 206 @param tractId: Data identifier for tract 207 @param selectDataList: List of selection data 208 @return whether tract has any overlapping inputs 210 def makePolygon(wcs, bbox):
211 """Return a polygon for the image, given Wcs and bounding box""" 212 return convexHull([wcs.pixelToSky(afwGeom.Point2D(coord)).getVector()
for 213 coord
in bbox.getCorners()])
215 skymap = cache.skymap
216 tract = skymap[tractId]
217 tractWcs = tract.getWcs()
218 tractPoly = makePolygon(tractWcs, tract.getBBox())
220 for selectData
in selectIdList:
221 if not hasattr(selectData,
"poly"):
222 selectData.poly = makePolygon(selectData.wcs, selectData.bbox)
223 if tractPoly.intersects(selectData.poly):
227 def warp(self, cache, patchId, selectDataList):
228 """!Warp all images for a patch 230 Only slave nodes execute this method. 232 Because only one argument may be passed, it is expected to 233 contain multiple elements, which are: 235 @param patchRef: data reference for patch 236 @param selectDataList: List of SelectStruct for inputs 237 @return selectDataList with non-overlapping elements removed 239 patchRef =
getDataRef(cache.butler, patchId, cache.coaddType)
241 with self.logOperation(
"warping %s" % (patchRef.dataId,), catch=
True):
242 self.makeCoaddTempExp.
run(patchRef, selectDataList)
243 return selectDataList
246 """!Construct coadd for a patch and measure 248 Only slave nodes execute this method. 250 Because only one argument may be passed, it is expected to 251 contain multiple elements, which are: 253 @param patchRef: data reference for patch 254 @param selectDataList: List of SelectStruct for inputs 256 patchRef =
getDataRef(cache.butler, data.patchId, cache.coaddType)
257 selectDataList = data.selectDataList
259 with self.logOperation(
"coadding %s" % (patchRef.dataId,), catch=
True):
260 if self.config.doOverwriteCoadd
or not patchRef.datasetExists(cache.coaddType):
261 coaddResults = self.assembleCoadd.
run(patchRef, selectDataList)
262 if coaddResults
is not None:
263 coadd = coaddResults.coaddExposure
264 elif patchRef.datasetExists(cache.coaddType):
265 self.log.info(
"%s: Reading coadd %s" % (NODE, patchRef.dataId))
266 coadd = patchRef.get(cache.coaddType, immediate=
True)
275 if self.config.doDetection:
276 with self.logOperation(
"detection on {}".format(patchRef.dataId),
278 idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
281 detResults = self.detectCoaddSources.runDetection(coadd, idFactory)
282 self.detectCoaddSources.write(coadd, detResults, patchRef)
284 patchRef.put(coadd, self.assembleCoadd.config.coaddName+
"Coadd")
287 """!Select exposures to operate upon, via the SelectImagesTask 289 This is very similar to CoaddBaseTask.selectExposures, except we return 290 a list of SelectStruct (same as the input), so we can plug the results into 291 future uses of SelectImagesTask. 293 @param patchRef data reference to a particular patch 294 @param selectDataList list of references to specific data products (i.e. visit, ccd) 295 @return filtered list of SelectStruct 298 return tuple(dataRef.dataId[k]
for k
in sorted(dataRef.dataId))
299 inputs = dict((key(select.dataRef), select)
300 for select
in selectDataList)
301 skyMap = patchRef.get(self.config.coaddName +
"Coadd_skyMap")
302 tract = skyMap[patchRef.dataId[
"tract"]]
303 patch = tract[(tuple(int(i)
304 for i
in patchRef.dataId[
"patch"].split(
",")))]
305 bbox = patch.getOuterBBox()
307 cornerPosList = afwGeom.Box2D(bbox).getCorners()
308 coordList = [wcs.pixelToSky(pos)
for pos
in cornerPosList]
309 dataRefList = self.select.runDataRef(
310 patchRef, coordList, selectDataList=selectDataList).dataRefList
311 return [inputs[key(dataRef)]
for dataRef
in dataRefList]
def batchWallTime(cls, time, parsedCmd, numCores)
Return walltime request for batch job.
def run(self, tractPatchRefList, butler, selectIdList=[])
Determine which tracts are non-empty before processing.
def selectExposures(self, patchRef, selectDataList)
Select exposures to operate upon, via the SelectImagesTask.
def runTract(self, patchRefList, butler, selectDataList=[])
Run stacking on a tract.
def getDataRef(butler, dataId, datasetType="raw")
def coadd(self, cache, data)
Construct coadd for a patch and measure.
def warp(self, cache, patchId, selectDataList)
Warp all images for a patch.
def getTargetList(parsedCmd, kwargs)
Get bare butler into Task.
def readSelection(self, cache, selectId)
Read Wcs of selected inputs.
def __init__(self, args, kwargs)
def checkTract(self, cache, tractId, selectIdList)
Check whether a tract has any overlapping inputs.
def writeMetadata(self, dataRef)