1 from __future__
import absolute_import, division, print_function
3 from builtins
import zip
4 from builtins
import map
23 coaddName = Field(dtype=str, default=
"deep", doc=
"Name for coadd")
24 select = ConfigurableField(
25 target=WcsSelectImagesTask, doc=
"Select images to process")
26 makeCoaddTempExp = ConfigurableField(
27 target=MakeCoaddTempExpTask, doc=
"Warp images to sky")
28 doBackgroundReference = Field(
29 dtype=bool, default=
False, doc=
"Build background reference?")
30 backgroundReference = ConfigurableField(
31 target=NullSelectImagesTask, doc=
"Build background reference")
32 assembleCoadd = ConfigurableField(
33 target=SafeClipAssembleCoaddTask, doc=
"Assemble warps into coadd")
34 doDetection = Field(dtype=bool, default=
True,
35 doc=
"Run detection on the coaddition product")
36 detectCoaddSources = ConfigurableField(
37 target=DetectCoaddSourcesTask, doc=
"Detect sources on coadd")
46 'BAD',
'EDGE',
'SAT',
'INTRP',
'NO_DATA']
51 "makeCoaddTempExp.coaddName and coaddName don't match")
54 "assembleCoadd.coaddName and coaddName don't match")
59 def __init__(self, TaskClass, parsedCmd, doReturnResults=False):
60 CoaddTaskRunner.__init__(self, TaskClass, parsedCmd, doReturnResults)
64 return self.TaskClass(config=self.config, log=self.log, reuse=self.
reuse)
68 """!Get bare butler into Task 70 @param parsedCmd results of parsing command input 72 kwargs[
"butler"] = parsedCmd.butler
73 kwargs[
"selectIdList"] = [
74 ref.dataId
for ref
in parsedCmd.selectId.refList]
75 return [(parsedCmd.id.refList, kwargs), ]
79 """Unpickle something by calling a factory""" 80 return factory(*args, **kwargs)
84 ConfigClass = CoaddDriverConfig
85 _DefaultName =
"coaddDriver" 86 RunnerClass = CoaddDriverTaskRunner
89 BatchPoolTask.__init__(self, **kwargs)
91 self.makeSubtask(
"select")
92 self.makeSubtask(
"makeCoaddTempExp", reuse=(
"makeCoaddTempExp" in self.
reuse))
93 self.makeSubtask(
"backgroundReference")
94 self.makeSubtask(
"assembleCoadd")
95 self.makeSubtask(
"detectCoaddSources")
99 return unpickle, (self.__class__, [], dict(config=self.config, name=self._name,
100 parentTask=self._parentTask, log=self.log,
104 def _makeArgumentParser(cls, **kwargs):
105 """!Build argument parser 107 Selection references are not cheap (reads Wcs), so are generated 108 only if we're not doing a batch submission. 111 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2",
112 ContainerClass=TractDataIdContainer)
113 parser.add_id_argument(
114 "--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9")
115 parser.addReuseOption([
"makeCoaddTempExp",
"assembleCoadd",
"detectCoaddSources"])
121 Return walltime request for batch job 123 @param time: Requested time per iteration 124 @param parsedCmd: Results of argument parsing 125 @param numCores: Number of cores 126 @return float walltime request length 128 numTargets = len(parsedCmd.selectId.refList)
129 return time*numTargets/float(numCores)
132 def run(self, tractPatchRefList, butler, selectIdList=[]):
133 """!Determine which tracts are non-empty before processing 135 @param tractPatchRefList: List of tracts and patches to include in the coaddition 136 @param butler: butler reference object 137 @param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd 138 @return list of references to sel.runTract function evaluation for each tractPatchRefList member 140 pool =
Pool(
"tracts")
141 pool.storeSet(butler=butler, skymap=butler.get(
142 self.config.coaddName +
"Coadd_skyMap"))
144 for patchRefList
in tractPatchRefList:
145 tractSet = set([patchRef.dataId[
"tract"]
146 for patchRef
in patchRefList])
147 assert len(tractSet) == 1
148 tractIdList.append(tractSet.pop())
150 selectDataList = [data
for data
in pool.mapNoBalance(self.
readSelection, selectIdList)
if 152 nonEmptyList = pool.mapNoBalance(
154 tractPatchRefList = [patchRefList
for patchRefList, nonEmpty
in 155 zip(tractPatchRefList, nonEmptyList)
if nonEmpty]
156 self.log.info(
"Non-empty tracts (%d): %s" % (len(tractPatchRefList),
157 [patchRefList[0].dataId[
"tract"]
for patchRefList
in 161 for data
in selectDataList:
162 data.dataRef =
getDataRef(butler, data.dataId,
"calexp")
165 return [self.
runTract(patchRefList, butler, selectDataList)
for patchRefList
in tractPatchRefList]
168 def runTract(self, patchRefList, butler, selectDataList=[]):
169 """!Run stacking on a tract 171 This method only runs on the master node. 173 @param patchRefList: List of patch data references for tract 174 @param butler: Data butler 175 @param selectDataList: List of SelectStruct for inputs 177 pool =
Pool(
"stacker")
179 pool.storeSet(butler=butler, warpType=self.config.coaddName +
"Coadd_directWarp",
180 coaddType=self.config.coaddName +
"Coadd")
181 patchIdList = [patchRef.dataId
for patchRef
in patchRefList]
183 selectedData = pool.map(self.
warp, patchIdList, selectDataList)
184 if self.config.doBackgroundReference:
185 self.backgroundReference.
run(patchRefList, selectDataList)
187 def refNamer(patchRef):
188 return tuple(map(int, patchRef.dataId[
"patch"].split(
",")))
190 lookup = dict(zip(map(refNamer, patchRefList), selectedData))
191 coaddData = [Struct(patchId=patchRef.dataId, selectDataList=lookup[refNamer(patchRef)])
for 192 patchRef
in patchRefList]
193 pool.map(self.
coadd, coaddData)
196 """!Read Wcs of selected inputs 198 This method only runs on slave nodes. 199 This method is similar to SelectDataIdContainer.makeDataRefList, 200 creating a Struct like a SelectStruct, except with a dataId instead 201 of a dataRef (to ease MPI). 203 @param cache: Pool cache 204 @param selectId: Data identifier for selected input 205 @return a SelectStruct with a dataId instead of dataRef 208 ref =
getDataRef(cache.butler, selectId,
"calexp")
209 self.log.info(
"Reading Wcs from %s" % (selectId,))
210 md = ref.get(
"calexp_md", immediate=
True)
211 wcs = afwImage.makeWcs(md)
212 data = Struct(dataId=selectId, wcs=wcs, bbox=afwImage.bboxFromMetadata(md))
214 self.log.warn(
"Unable to construct Wcs from %s" % (selectId,))
219 """!Check whether a tract has any overlapping inputs 221 This method only runs on slave nodes. 223 @param cache: Pool cache 224 @param tractId: Data identifier for tract 225 @param selectDataList: List of selection data 226 @return whether tract has any overlapping inputs 228 def makePolygon(wcs, bbox):
229 """Return a polygon for the image, given Wcs and bounding box""" 230 return convexHull([wcs.pixelToSky(afwGeom.Point2D(coord)).getVector()
for 231 coord
in bbox.getCorners()])
233 skymap = cache.skymap
234 tract = skymap[tractId]
235 tractWcs = tract.getWcs()
236 tractPoly = makePolygon(tractWcs, tract.getBBox())
238 for selectData
in selectIdList:
239 if not hasattr(selectData,
"poly"):
240 selectData.poly = makePolygon(selectData.wcs, selectData.bbox)
241 if tractPoly.intersects(selectData.poly):
245 def warp(self, cache, patchId, selectDataList):
246 """!Warp all images for a patch 248 Only slave nodes execute this method. 250 Because only one argument may be passed, it is expected to 251 contain multiple elements, which are: 253 @param patchRef: data reference for patch 254 @param selectDataList: List of SelectStruct for inputs 255 @return selectDataList with non-overlapping elements removed 257 patchRef =
getDataRef(cache.butler, patchId, cache.coaddType)
259 with self.
logOperation(
"warping %s" % (patchRef.dataId,), catch=
True):
260 self.makeCoaddTempExp.
run(patchRef, selectDataList)
261 return selectDataList
264 """!Construct coadd for a patch and measure 266 Only slave nodes execute this method. 268 Because only one argument may be passed, it is expected to 269 contain multiple elements, which are: 271 @param patchRef: data reference for patch 272 @param selectDataList: List of SelectStruct for inputs 274 patchRef =
getDataRef(cache.butler, data.patchId, cache.coaddType)
275 selectDataList = data.selectDataList
282 "detectCoaddSources" in self.
reuse and 283 patchRef.datasetExists(self.detectCoaddSources.config.coaddName+
"Coadd_det", write=
True)
285 if "assembleCoadd" in self.
reuse:
286 if patchRef.datasetExists(cache.coaddType, write=
True):
287 self.log.info(
"%s: Skipping assembleCoadd for %s; outputs already exist." %
288 (NODE, patchRef.dataId))
289 coadd = patchRef.get(cache.coaddType, immediate=
True)
290 elif not self.config.assembleCoadd.doWrite
and self.config.doDetection
and canSkipDetection:
292 "%s: Skipping assembleCoadd and detectCoaddSources for %s; outputs already exist." %
293 (NODE, patchRef.dataId)
297 with self.
logOperation(
"coadding %s" % (patchRef.dataId,), catch=
True):
298 coaddResults = self.assembleCoadd.
run(patchRef, selectDataList)
299 if coaddResults
is not None:
300 coadd = coaddResults.coaddExposure
301 canSkipDetection =
False 309 if self.config.doDetection:
311 self.log.info(
"%s: Skipping detectCoaddSources for %s; outputs already exist." %
312 (NODE, patchRef.dataId))
314 with self.
logOperation(
"detection on {}".format(patchRef.dataId),
316 idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
319 detResults = self.detectCoaddSources.runDetection(coadd, idFactory)
320 self.detectCoaddSources.write(coadd, detResults, patchRef)
322 patchRef.put(coadd, self.assembleCoadd.config.coaddName+
"Coadd")
325 """!Select exposures to operate upon, via the SelectImagesTask 327 This is very similar to CoaddBaseTask.selectExposures, except we return 328 a list of SelectStruct (same as the input), so we can plug the results into 329 future uses of SelectImagesTask. 331 @param patchRef data reference to a particular patch 332 @param selectDataList list of references to specific data products (i.e. visit, ccd) 333 @return filtered list of SelectStruct 336 return tuple(dataRef.dataId[k]
for k
in sorted(dataRef.dataId))
337 inputs = dict((key(select.dataRef), select)
338 for select
in selectDataList)
339 skyMap = patchRef.get(self.config.coaddName +
"Coadd_skyMap")
340 tract = skyMap[patchRef.dataId[
"tract"]]
341 patch = tract[(tuple(int(i)
342 for i
in patchRef.dataId[
"patch"].split(
",")))]
343 bbox = patch.getOuterBBox()
345 cornerPosList = afwGeom.Box2D(bbox).getCorners()
346 coordList = [wcs.pixelToSky(pos)
for pos
in cornerPosList]
347 dataRefList = self.select.runDataRef(
348 patchRef, coordList, selectDataList=selectDataList).dataRefList
349 return [inputs[key(dataRef)]
for dataRef
in dataRefList]
def batchWallTime(cls, time, parsedCmd, numCores)
Return walltime request for batch job.
def unpickle(factory, args, kwargs)
def run(self, tractPatchRefList, butler, selectIdList=[])
Determine which tracts are non-empty before processing.
def selectExposures(self, patchRef, selectDataList)
Select exposures to operate upon, via the SelectImagesTask.
def runTract(self, patchRefList, butler, selectDataList=[])
Run stacking on a tract.
def makeTask(self, parsedCmd=None, args=None)
def getDataRef(butler, dataId, datasetType="raw")
def __init__(self, reuse=tuple(), kwargs)
def coadd(self, cache, data)
Construct coadd for a patch and measure.
def warp(self, cache, patchId, selectDataList)
Warp all images for a patch.
def logOperation(self, operation, catch=False, trace=True)
def getTargetList(parsedCmd, kwargs)
Get bare butler into Task.
def readSelection(self, cache, selectId)
Read Wcs of selected inputs.
def __init__(self, TaskClass, parsedCmd, doReturnResults=False)
def checkTract(self, cache, tractId, selectIdList)
Check whether a tract has any overlapping inputs.
def writeMetadata(self, dataRef)