1 from __future__
import absolute_import, division, print_function
3 from builtins
import zip
4 from builtins
import map
23 coaddName = Field(dtype=str, default=
"deep", doc=
"Name for coadd")
24 select = ConfigurableField(
25 target=WcsSelectImagesTask, doc=
"Select images to process")
26 makeCoaddTempExp = ConfigurableField(
27 target=MakeCoaddTempExpTask, doc=
"Warp images to sky")
28 doBackgroundReference = Field(
29 dtype=bool, default=
False, doc=
"Build background reference?")
30 backgroundReference = ConfigurableField(
31 target=NullSelectImagesTask, doc=
"Build background reference")
32 assembleCoadd = ConfigurableField(
33 target=SafeClipAssembleCoaddTask, doc=
"Assemble warps into coadd")
34 doDetection = Field(dtype=bool, default=
True,
35 doc=
"Run detection on the coaddition product")
36 detectCoaddSources = ConfigurableField(
37 target=DetectCoaddSourcesTask, doc=
"Detect sources on coadd")
38 doOverwriteCoadd = Field(dtype=bool, default=
False, doc=
"Overwrite coadd?")
48 'BAD',
'EDGE',
'SAT',
'INTRP',
'NO_DATA']
53 "makeCoaddTempExp.coaddName and coaddName don't match")
56 "assembleCoadd.coaddName and coaddName don't match")
63 """!Get bare butler into Task 65 @param parsedCmd results of parsing command input 67 kwargs[
"butler"] = parsedCmd.butler
68 kwargs[
"selectIdList"] = [
69 ref.dataId
for ref
in parsedCmd.selectId.refList]
70 return [(parsedCmd.id.refList, kwargs), ]
74 ConfigClass = CoaddDriverConfig
75 _DefaultName =
"coaddDriver" 76 RunnerClass = CoaddDriverTaskRunner
79 BatchPoolTask.__init__(self, *args, **kwargs)
80 self.makeSubtask(
"select")
81 self.makeSubtask(
"makeCoaddTempExp")
82 self.makeSubtask(
"backgroundReference")
83 self.makeSubtask(
"assembleCoadd")
84 self.makeSubtask(
"detectCoaddSources")
87 def _makeArgumentParser(cls, **kwargs):
88 """!Build argument parser 90 Selection references are not cheap (reads Wcs), so are generated 91 only if we're not doing a batch submission. 94 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2",
95 ContainerClass=TractDataIdContainer)
96 parser.add_id_argument(
97 "--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9")
103 Return walltime request for batch job 105 @param time: Requested time per iteration 106 @param parsedCmd: Results of argument parsing 107 @param numCores: Number of cores 108 @return float walltime request length 110 numTargets = len(parsedCmd.selectId.refList)
111 return time*numTargets/float(numCores)
114 def run(self, tractPatchRefList, butler, selectIdList=[]):
115 """!Determine which tracts are non-empty before processing 117 @param tractPatchRefList: List of tracts and patches to include in the coaddition 118 @param butler: butler reference object 119 @param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd 120 @return list of references to sel.runTract function evaluation for each tractPatchRefList member 122 pool =
Pool(
"tracts")
123 pool.storeSet(butler=butler, skymap=butler.get(
124 self.config.coaddName +
"Coadd_skyMap"))
126 for patchRefList
in tractPatchRefList:
127 tractSet = set([patchRef.dataId[
"tract"]
128 for patchRef
in patchRefList])
129 assert len(tractSet) == 1
130 tractIdList.append(tractSet.pop())
132 selectDataList = [data
for data
in pool.mapNoBalance(self.
readSelection, selectIdList)
if 134 nonEmptyList = pool.mapNoBalance(
136 tractPatchRefList = [patchRefList
for patchRefList, nonEmpty
in 137 zip(tractPatchRefList, nonEmptyList)
if nonEmpty]
138 self.log.info(
"Non-empty tracts (%d): %s" % (len(tractPatchRefList),
139 [patchRefList[0].dataId[
"tract"]
for patchRefList
in 143 for data
in selectDataList:
144 data.dataRef =
getDataRef(butler, data.dataId,
"calexp")
147 return [self.
runTract(patchRefList, butler, selectDataList)
for patchRefList
in tractPatchRefList]
150 def runTract(self, patchRefList, butler, selectDataList=[]):
151 """!Run stacking on a tract 153 This method only runs on the master node. 155 @param patchRefList: List of patch data references for tract 156 @param butler: Data butler 157 @param selectDataList: List of SelectStruct for inputs 159 pool =
Pool(
"stacker")
161 pool.storeSet(butler=butler, warpType=self.config.coaddName +
"Coadd_directWarp",
162 coaddType=self.config.coaddName +
"Coadd")
163 patchIdList = [patchRef.dataId
for patchRef
in patchRefList]
165 selectedData = pool.map(self.
warp, patchIdList, selectDataList)
166 if self.config.doBackgroundReference:
167 self.backgroundReference.
run(patchRefList, selectDataList)
169 def refNamer(patchRef):
170 return tuple(map(int, patchRef.dataId[
"patch"].split(
",")))
172 lookup = dict(zip(map(refNamer, patchRefList), selectedData))
173 coaddData = [Struct(patchId=patchRef.dataId, selectDataList=lookup[refNamer(patchRef)])
for 174 patchRef
in patchRefList]
175 pool.map(self.
coadd, coaddData)
178 """!Read Wcs of selected inputs 180 This method only runs on slave nodes. 181 This method is similar to SelectDataIdContainer.makeDataRefList, 182 creating a Struct like a SelectStruct, except with a dataId instead 183 of a dataRef (to ease MPI). 185 @param cache: Pool cache 186 @param selectId: Data identifier for selected input 187 @return a SelectStruct with a dataId instead of dataRef 190 ref =
getDataRef(cache.butler, selectId,
"calexp")
191 self.log.info(
"Reading Wcs from %s" % (selectId,))
192 md = ref.get(
"calexp_md", immediate=
True)
193 wcs = afwImage.makeWcs(md)
194 data = Struct(dataId=selectId, wcs=wcs, bbox=afwImage.bboxFromMetadata(md))
196 self.log.warn(
"Unable to construct Wcs from %s" % (selectId,))
201 """!Check whether a tract has any overlapping inputs 203 This method only runs on slave nodes. 205 @param cache: Pool cache 206 @param tractId: Data identifier for tract 207 @param selectDataList: List of selection data 208 @return whether tract has any overlapping inputs 210 def makePolygon(wcs, bbox):
211 """Return a polygon for the image, given Wcs and bounding box""" 212 return convexHull([wcs.pixelToSky(afwGeom.Point2D(coord)).getVector()
for 213 coord
in bbox.getCorners()])
215 skymap = cache.skymap
216 tract = skymap[tractId]
217 tractWcs = tract.getWcs()
218 tractPoly = makePolygon(tractWcs, tract.getBBox())
220 for selectData
in selectIdList:
221 if not hasattr(selectData,
"poly"):
222 selectData.poly = makePolygon(selectData.wcs, selectData.bbox)
223 if tractPoly.intersects(selectData.poly):
227 def warp(self, cache, patchId, selectDataList):
228 """!Warp all images for a patch 230 Only slave nodes execute this method. 232 Because only one argument may be passed, it is expected to 233 contain multiple elements, which are: 235 @param patchRef: data reference for patch 236 @param selectDataList: List of SelectStruct for inputs 237 @return selectDataList with non-overlapping elements removed 239 patchRef =
getDataRef(cache.butler, patchId, cache.coaddType)
241 with self.
logOperation(
"warping %s" % (patchRef.dataId,), catch=
True):
242 self.makeCoaddTempExp.
run(patchRef, selectDataList)
243 return selectDataList
246 """!Construct coadd for a patch and measure 248 Only slave nodes execute this method. 250 Because only one argument may be passed, it is expected to 251 contain multiple elements, which are: 253 @param patchRef: data reference for patch 254 @param selectDataList: List of SelectStruct for inputs 256 patchRef =
getDataRef(cache.butler, data.patchId, cache.coaddType)
257 selectDataList = data.selectDataList
259 with self.
logOperation(
"coadding %s" % (patchRef.dataId,), catch=
True):
260 if self.config.doOverwriteCoadd
or not patchRef.datasetExists(cache.coaddType):
261 coaddResults = self.assembleCoadd.
run(patchRef, selectDataList)
262 if coaddResults
is not None:
263 coadd = coaddResults.coaddExposure
264 elif patchRef.datasetExists(cache.coaddType):
265 self.log.info(
"%s: Reading coadd %s" % (NODE, patchRef.dataId))
266 coadd = patchRef.get(cache.coaddType, immediate=
True)
275 if self.config.doDetection:
276 with self.
logOperation(
"detection on {}".format(patchRef.dataId),
278 idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
281 detResults = self.detectCoaddSources.runDetection(coadd, idFactory)
282 self.detectCoaddSources.write(coadd, detResults, patchRef)
284 patchRef.put(coadd, self.assembleCoadd.config.coaddName+
"Coadd")
287 """!Select exposures to operate upon, via the SelectImagesTask 289 This is very similar to CoaddBaseTask.selectExposures, except we return 290 a list of SelectStruct (same as the input), so we can plug the results into 291 future uses of SelectImagesTask. 293 @param patchRef data reference to a particular patch 294 @param selectDataList list of references to specific data products (i.e. visit, ccd) 295 @return filtered list of SelectStruct 298 return tuple(dataRef.dataId[k]
for k
in sorted(dataRef.dataId))
299 inputs = dict((key(select.dataRef), select)
300 for select
in selectDataList)
301 skyMap = patchRef.get(self.config.coaddName +
"Coadd_skyMap")
302 tract = skyMap[patchRef.dataId[
"tract"]]
303 patch = tract[(tuple(int(i)
304 for i
in patchRef.dataId[
"patch"].split(
",")))]
305 bbox = patch.getOuterBBox()
307 cornerPosList = afwGeom.Box2D(bbox).getCorners()
308 coordList = [wcs.pixelToSky(pos)
for pos
in cornerPosList]
309 dataRefList = self.select.runDataRef(
310 patchRef, coordList, selectDataList=selectDataList).dataRefList
311 return [inputs[key(dataRef)]
for dataRef
in dataRefList]
def batchWallTime(cls, time, parsedCmd, numCores)
Return walltime request for batch job.
def run(self, tractPatchRefList, butler, selectIdList=[])
Determine which tracts are non-empty before processing.
def selectExposures(self, patchRef, selectDataList)
Select exposures to operate upon, via the SelectImagesTask.
def runTract(self, patchRefList, butler, selectDataList=[])
Run stacking on a tract.
def getDataRef(butler, dataId, datasetType="raw")
def coadd(self, cache, data)
Construct coadd for a patch and measure.
def warp(self, cache, patchId, selectDataList)
Warp all images for a patch.
def logOperation(self, operation, catch=False, trace=True)
def getTargetList(parsedCmd, kwargs)
Get bare butler into Task.
def readSelection(self, cache, selectId)
Read Wcs of selected inputs.
def __init__(self, args, kwargs)
def checkTract(self, cache, tractId, selectIdList)
Check whether a tract has any overlapping inputs.
def writeMetadata(self, dataRef)