1 from __future__
import absolute_import, division, print_function
3 from builtins
import zip
4 from builtins
import map
23 coaddName = Field(dtype=str, default=
"deep", doc=
"Name for coadd")
24 select = ConfigurableField(
25 target=WcsSelectImagesTask, doc=
"Select images to process")
26 makeCoaddTempExp = ConfigurableField(
27 target=MakeCoaddTempExpTask, doc=
"Warp images to sky")
28 doBackgroundReference = Field(
29 dtype=bool, default=
False, doc=
"Build background reference?")
30 backgroundReference = ConfigurableField(
31 target=NullSelectImagesTask, doc=
"Build background reference")
32 assembleCoadd = ConfigurableField(
33 target=SafeClipAssembleCoaddTask, doc=
"Assemble warps into coadd")
34 doDetection = Field(dtype=bool, default=
True,
35 doc=
"Run detection on the coaddition product")
36 detectCoaddSources = ConfigurableField(
37 target=DetectCoaddSourcesTask, doc=
"Detect sources on coadd")
47 "makeCoaddTempExp.coaddName and coaddName don't match")
50 "assembleCoadd.coaddName and coaddName don't match")
55 def __init__(self, TaskClass, parsedCmd, doReturnResults=False):
56 CoaddTaskRunner.__init__(self, TaskClass, parsedCmd, doReturnResults)
60 return self.TaskClass(config=self.config, log=self.log, reuse=self.
reuse)
64 """!Get bare butler into Task 66 @param parsedCmd results of parsing command input 68 kwargs[
"butler"] = parsedCmd.butler
69 kwargs[
"selectIdList"] = [
70 ref.dataId
for ref
in parsedCmd.selectId.refList]
71 return [(parsedCmd.id.refList, kwargs), ]
75 """Unpickle something by calling a factory""" 76 return factory(*args, **kwargs)
80 ConfigClass = CoaddDriverConfig
81 _DefaultName =
"coaddDriver" 82 RunnerClass = CoaddDriverTaskRunner
85 BatchPoolTask.__init__(self, **kwargs)
87 self.makeSubtask(
"select")
88 self.makeSubtask(
"makeCoaddTempExp", reuse=(
"makeCoaddTempExp" in self.
reuse))
89 self.makeSubtask(
"backgroundReference")
90 self.makeSubtask(
"assembleCoadd")
91 self.makeSubtask(
"detectCoaddSources")
95 return unpickle, (self.__class__, [], dict(config=self.config, name=self._name,
96 parentTask=self._parentTask, log=self.log,
100 def _makeArgumentParser(cls, **kwargs):
101 """!Build argument parser 103 Selection references are not cheap (reads Wcs), so are generated 104 only if we're not doing a batch submission. 107 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2",
108 ContainerClass=TractDataIdContainer)
109 parser.add_id_argument(
110 "--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9")
111 parser.addReuseOption([
"makeCoaddTempExp",
"assembleCoadd",
"detectCoaddSources"])
117 Return walltime request for batch job 119 @param time: Requested time per iteration 120 @param parsedCmd: Results of argument parsing 121 @param numCores: Number of cores 122 @return float walltime request length 124 numTargets = len(parsedCmd.selectId.refList)
125 return time*numTargets/float(numCores)
128 def run(self, tractPatchRefList, butler, selectIdList=[]):
129 """!Determine which tracts are non-empty before processing 131 @param tractPatchRefList: List of tracts and patches to include in the coaddition 132 @param butler: butler reference object 133 @param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd 134 @return list of references to sel.runTract function evaluation for each tractPatchRefList member 136 pool =
Pool(
"tracts")
137 pool.storeSet(butler=butler, skymap=butler.get(
138 self.config.coaddName +
"Coadd_skyMap"))
140 for patchRefList
in tractPatchRefList:
141 tractSet = set([patchRef.dataId[
"tract"]
142 for patchRef
in patchRefList])
143 assert len(tractSet) == 1
144 tractIdList.append(tractSet.pop())
146 selectDataList = [data
for data
in pool.mapNoBalance(self.
readSelection, selectIdList)
if 148 nonEmptyList = pool.mapNoBalance(
150 tractPatchRefList = [patchRefList
for patchRefList, nonEmpty
in 151 zip(tractPatchRefList, nonEmptyList)
if nonEmpty]
152 self.log.info(
"Non-empty tracts (%d): %s" % (len(tractPatchRefList),
153 [patchRefList[0].dataId[
"tract"]
for patchRefList
in 157 for data
in selectDataList:
158 data.dataRef =
getDataRef(butler, data.dataId,
"calexp")
161 return [self.
runTract(patchRefList, butler, selectDataList)
for patchRefList
in tractPatchRefList]
164 def runTract(self, patchRefList, butler, selectDataList=[]):
165 """!Run stacking on a tract 167 This method only runs on the master node. 169 @param patchRefList: List of patch data references for tract 170 @param butler: Data butler 171 @param selectDataList: List of SelectStruct for inputs 173 pool =
Pool(
"stacker")
175 pool.storeSet(butler=butler, warpType=self.config.coaddName +
"Coadd_directWarp",
176 coaddType=self.config.coaddName +
"Coadd")
177 patchIdList = [patchRef.dataId
for patchRef
in patchRefList]
179 selectedData = pool.map(self.
warp, patchIdList, selectDataList)
180 if self.config.doBackgroundReference:
181 self.backgroundReference.
run(patchRefList, selectDataList)
183 def refNamer(patchRef):
184 return tuple(map(int, patchRef.dataId[
"patch"].split(
",")))
186 lookup = dict(zip(map(refNamer, patchRefList), selectedData))
187 coaddData = [Struct(patchId=patchRef.dataId, selectDataList=lookup[refNamer(patchRef)])
for 188 patchRef
in patchRefList]
189 pool.map(self.
coadd, coaddData)
192 """!Read Wcs of selected inputs 194 This method only runs on slave nodes. 195 This method is similar to SelectDataIdContainer.makeDataRefList, 196 creating a Struct like a SelectStruct, except with a dataId instead 197 of a dataRef (to ease MPI). 199 @param cache: Pool cache 200 @param selectId: Data identifier for selected input 201 @return a SelectStruct with a dataId instead of dataRef 204 ref =
getDataRef(cache.butler, selectId,
"calexp")
205 self.log.info(
"Reading Wcs from %s" % (selectId,))
206 md = ref.get(
"calexp_md", immediate=
True)
207 wcs = afwGeom.makeSkyWcs(md)
208 data = Struct(dataId=selectId, wcs=wcs, bbox=afwImage.bboxFromMetadata(md))
210 self.log.warn(
"Unable to construct Wcs from %s" % (selectId,))
215 """!Check whether a tract has any overlapping inputs 217 This method only runs on slave nodes. 219 @param cache: Pool cache 220 @param tractId: Data identifier for tract 221 @param selectDataList: List of selection data 222 @return whether tract has any overlapping inputs 224 def makePolygon(wcs, bbox):
225 """Return a polygon for the image, given Wcs and bounding box""" 226 return convexHull([wcs.pixelToSky(afwGeom.Point2D(coord)).getVector()
for 227 coord
in bbox.getCorners()])
229 skymap = cache.skymap
230 tract = skymap[tractId]
231 tractWcs = tract.getWcs()
232 tractPoly = makePolygon(tractWcs, tract.getBBox())
234 for selectData
in selectIdList:
235 if not hasattr(selectData,
"poly"):
236 selectData.poly = makePolygon(selectData.wcs, selectData.bbox)
237 if tractPoly.intersects(selectData.poly):
241 def warp(self, cache, patchId, selectDataList):
242 """!Warp all images for a patch 244 Only slave nodes execute this method. 246 Because only one argument may be passed, it is expected to 247 contain multiple elements, which are: 249 @param patchRef: data reference for patch 250 @param selectDataList: List of SelectStruct for inputs 251 @return selectDataList with non-overlapping elements removed 253 patchRef =
getDataRef(cache.butler, patchId, cache.coaddType)
255 with self.
logOperation(
"warping %s" % (patchRef.dataId,), catch=
True):
256 self.makeCoaddTempExp.
run(patchRef, selectDataList)
257 return selectDataList
260 """!Construct coadd for a patch and measure 262 Only slave nodes execute this method. 264 Because only one argument may be passed, it is expected to 265 contain multiple elements, which are: 267 @param patchRef: data reference for patch 268 @param selectDataList: List of SelectStruct for inputs 270 patchRef =
getDataRef(cache.butler, data.patchId, cache.coaddType)
271 selectDataList = data.selectDataList
278 "detectCoaddSources" in self.
reuse and 279 patchRef.datasetExists(self.detectCoaddSources.config.coaddName+
"Coadd_det", write=
True)
281 if "assembleCoadd" in self.
reuse:
282 if patchRef.datasetExists(cache.coaddType, write=
True):
283 self.log.info(
"%s: Skipping assembleCoadd for %s; outputs already exist." %
284 (NODE, patchRef.dataId))
285 coadd = patchRef.get(cache.coaddType, immediate=
True)
286 elif not self.config.assembleCoadd.doWrite
and self.config.doDetection
and canSkipDetection:
288 "%s: Skipping assembleCoadd and detectCoaddSources for %s; outputs already exist." %
289 (NODE, patchRef.dataId)
293 with self.
logOperation(
"coadding %s" % (patchRef.dataId,), catch=
True):
294 coaddResults = self.assembleCoadd.
run(patchRef, selectDataList)
295 if coaddResults
is not None:
296 coadd = coaddResults.coaddExposure
297 canSkipDetection =
False 305 if self.config.doDetection:
307 self.log.info(
"%s: Skipping detectCoaddSources for %s; outputs already exist." %
308 (NODE, patchRef.dataId))
310 with self.
logOperation(
"detection on {}".format(patchRef.dataId),
312 idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
313 expId = int(patchRef.get(self.config.coaddName +
"CoaddId"))
316 detResults = self.detectCoaddSources.runDetection(coadd, idFactory, expId=expId)
317 self.detectCoaddSources.write(coadd, detResults, patchRef)
319 patchRef.put(coadd, self.assembleCoadd.config.coaddName+
"Coadd")
322 """!Select exposures to operate upon, via the SelectImagesTask 324 This is very similar to CoaddBaseTask.selectExposures, except we return 325 a list of SelectStruct (same as the input), so we can plug the results into 326 future uses of SelectImagesTask. 328 @param patchRef data reference to a particular patch 329 @param selectDataList list of references to specific data products (i.e. visit, ccd) 330 @return filtered list of SelectStruct 333 return tuple(dataRef.dataId[k]
for k
in sorted(dataRef.dataId))
334 inputs = dict((key(select.dataRef), select)
335 for select
in selectDataList)
336 skyMap = patchRef.get(self.config.coaddName +
"Coadd_skyMap")
337 tract = skyMap[patchRef.dataId[
"tract"]]
338 patch = tract[(tuple(int(i)
339 for i
in patchRef.dataId[
"patch"].split(
",")))]
340 bbox = patch.getOuterBBox()
342 cornerPosList = afwGeom.Box2D(bbox).getCorners()
343 coordList = [wcs.pixelToSky(pos)
for pos
in cornerPosList]
344 dataRefList = self.select.runDataRef(
345 patchRef, coordList, selectDataList=selectDataList).dataRefList
346 return [inputs[key(dataRef)]
for dataRef
in dataRefList]
def batchWallTime(cls, time, parsedCmd, numCores)
Return walltime request for batch job.
def unpickle(factory, args, kwargs)
def run(self, tractPatchRefList, butler, selectIdList=[])
Determine which tracts are non-empty before processing.
def selectExposures(self, patchRef, selectDataList)
Select exposures to operate upon, via the SelectImagesTask.
def runTract(self, patchRefList, butler, selectDataList=[])
Run stacking on a tract.
def makeTask(self, parsedCmd=None, args=None)
def getDataRef(butler, dataId, datasetType="raw")
def __init__(self, reuse=tuple(), kwargs)
def coadd(self, cache, data)
Construct coadd for a patch and measure.
def warp(self, cache, patchId, selectDataList)
Warp all images for a patch.
def logOperation(self, operation, catch=False, trace=True)
def getTargetList(parsedCmd, kwargs)
Get bare butler into Task.
def readSelection(self, cache, selectId)
Read Wcs of selected inputs.
def __init__(self, TaskClass, parsedCmd, doReturnResults=False)
def checkTract(self, cache, tractId, selectIdList)
Check whether a tract has any overlapping inputs.
def writeMetadata(self, dataRef)