1 from __future__
import absolute_import, division, print_function
3 from argparse
import ArgumentError
5 from builtins
import zip
11 MeasureMergedCoaddSourcesTask,
12 MergeMeasurementsTask,)
26 """!Make self.refList from self.idList 28 It's difficult to make a data reference that merely points to an entire 29 tract: there is no data product solely at the tract level. Instead, we 30 generate a list of data references for patches within the tract. 32 @param namespace namespace object that is the result of an argument parser 34 datasetType = namespace.config.coaddName +
"Coadd_calexp" 36 def getPatchRefList(tract):
37 return [namespace.butler.dataRef(datasetType=datasetType,
39 filter=dataId[
"filter"],
40 patch=
"%d,%d" % patch.getIndex())
44 for dataId
in self.idList:
47 if "filter" not in dataId:
48 raise ArgumentError(
None,
"--id must include 'filter'")
50 skymap = self.getSkymap(namespace, datasetType)
53 tractId = dataId[
"tract"]
54 if tractId
not in tractRefs:
55 tractRefs[tractId] = []
57 tractRefs[tractId].append(namespace.butler.dataRef(datasetType=datasetType,
61 patch=dataId[
'patch']))
63 tractRefs[tractId] += getPatchRefList(skymap[tractId])
65 tractRefs = dict((tract.getId(), tractRefs.get(tract.getId(), []) + getPatchRefList(tract))
72 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
73 detectCoaddSources = ConfigurableField(target=DetectCoaddSourcesTask,
74 doc=
"Detect sources on coadd")
75 mergeCoaddDetections = ConfigurableField(
76 target=MergeDetectionsTask, doc=
"Merge detections")
77 measureCoaddSources = ConfigurableField(target=MeasureMergedCoaddSourcesTask,
78 doc=
"Measure merged detections")
79 mergeCoaddMeasurements = ConfigurableField(
80 target=MergeMeasurementsTask, doc=
"Merge measurements")
81 forcedPhotCoadd = ConfigurableField(target=ForcedPhotCoaddTask,
82 doc=
"Forced measurement on coadded images")
83 clobberDetections = Field(
84 dtype=bool, default=
False, doc=
"Clobber existing detections?")
85 clobberMergedDetections = Field(
86 dtype=bool, default=
False, doc=
"Clobber existing merged detections?")
87 clobberMeasurements = Field(
88 dtype=bool, default=
False, doc=
"Clobber existing measurements?")
89 clobberMergedMeasurements = Field(
90 dtype=bool, default=
False, doc=
"Clobber existing merged measurements?")
91 clobberForcedPhotometry = Field(
92 dtype=bool, default=
False, doc=
"Clobber existing forced photometry?")
94 dtype=bool, default=
False,
95 doc=(
"Are we reprocessing?\n\n" 96 "This exists as a workaround for large deblender footprints causing large memory use " 97 "and/or very slow processing. We refuse to deblend those footprints when running on a cluster " 98 "and return to reprocess on a machine with larger memory or more time " 99 "if we consider those footprints important to recover."),
103 Config.setDefaults(self)
107 for subtask
in (
"mergeCoaddDetections",
"measureCoaddSources",
108 "mergeCoaddMeasurements",
"forcedPhotCoadd"):
109 coaddName = getattr(self, subtask).coaddName
111 raise RuntimeError(
"%s.coaddName (%s) doesn't match root coaddName (%s)" %
116 """TaskRunner for running MultiBandTask 118 This is similar to the lsst.pipe.base.ButlerInitializedTaskRunner, 119 except that we have a list of data references instead of a single 120 data reference being passed to the Task.run. 124 """A variant of the base version that passes a butler argument to the task's constructor 125 parsedCmd or args must be specified. 127 if parsedCmd
is not None:
128 butler = parsedCmd.butler
129 elif args
is not None:
130 dataRefList, kwargs = args
131 butler = dataRefList[0].butlerSubset.butler
133 raise RuntimeError(
"parsedCmd or args must be specified")
134 return self.TaskClass(config=self.config, log=self.log, butler=butler)
138 """Unpickle something by calling a factory""" 139 return factory(*args, **kwargs)
143 """Multi-node driver for multiband processing""" 144 ConfigClass = MultiBandDriverConfig
145 _DefaultName =
"multiBandDriver" 146 RunnerClass = MultiBandDriverTaskRunner
148 def __init__(self, butler=None, schema=None, refObjLoader=None, **kwargs):
150 @param[in] butler: the butler can be used to retrieve schema or passed to the refObjLoader constructor 151 in case it is needed. 152 @param[in] schema: the schema of the source detection catalog used as input. 153 @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference 154 catalog. May be None if the butler argument is provided or all steps requiring a reference 155 catalog are disabled. 157 BatchPoolTask.__init__(self, **kwargs)
159 assert butler
is not None,
"Butler not provided" 160 schema = butler.get(self.config.coaddName +
161 "Coadd_det_schema", immediate=
True).schema
163 self.makeSubtask(
"detectCoaddSources")
164 self.makeSubtask(
"mergeCoaddDetections", schema=schema)
165 self.makeSubtask(
"measureCoaddSources", schema=afwTable.Schema(self.mergeCoaddDetections.schema),
166 peakSchema=afwTable.Schema(
167 self.mergeCoaddDetections.merged.getPeakSchema()),
168 refObjLoader=refObjLoader, butler=butler)
169 self.makeSubtask(
"mergeCoaddMeasurements", schema=afwTable.Schema(
170 self.measureCoaddSources.schema))
171 self.makeSubtask(
"forcedPhotCoadd", refSchema=afwTable.Schema(
172 self.mergeCoaddMeasurements.schema))
176 return unpickle, (self.__class__, [], dict(config=self.config, name=self._name,
177 parentTask=self._parentTask, log=self.log,
181 def _makeArgumentParser(cls, *args, **kwargs):
182 kwargs.pop(
"doBatch",
False)
183 parser = ArgumentParser(name=cls.
_DefaultName, *args, **kwargs)
184 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2",
185 ContainerClass=TractDataIdContainer)
190 """!Return walltime request for batch job 192 @param time: Requested time per iteration 193 @param parsedCmd: Results of argument parsing 194 @param numCores: Number of cores 197 for refList
in parsedCmd.id.refList:
198 numTargets += len(refList)
199 return time*numTargets/float(numCpus)
202 def run(self, patchRefList):
203 """!Run multiband processing on coadds 205 Only the master node runs this method. 207 No real MPI communication (scatter/gather) takes place: all I/O goes 208 through the disk. We want the intermediate stages on disk, and the 209 component Tasks are implemented around this, so we just follow suit. 211 @param patchRefList: Data references to run measurement 213 for patchRef
in patchRefList:
215 butler = patchRef.getButler()
218 raise RuntimeError(
"No valid patches")
221 pool.storeSet(butler=butler)
232 if self.config.clobberDetections:
233 detectionList = [patchRef
for patchRef
in patchRefList
if 234 patchRef.datasetExists(self.config.coaddName +
"Coadd")]
236 detectionList = [patchRef
for patchRef
in patchRefList
if not 237 patchRef.datasetExists(self.config.coaddName +
239 patchRef.datasetExists(self.config.coaddName +
244 patchRefList = [patchRef
for patchRef
in patchRefList
if 245 patchRef.datasetExists(self.config.coaddName +
"Coadd_calexp")
and 246 patchRef.datasetExists(self.config.coaddName +
"Coadd_det")]
247 dataIdList = [patchRef.dataId
for patchRef
in patchRefList]
252 for patchRef
in patchRefList:
253 dataId = patchRef.dataId
255 tract = dataId[
"tract"]
257 assert tract == dataId[
"tract"]
259 patch = dataId[
"patch"]
260 if patch
not in patches:
262 patches[patch].append(dataId)
291 if self.config.reprocessing:
292 patchReprocessing = {}
293 for dataId, reprocess
in zip(dataIdList, reprocessed):
294 patchId = dataId[
"patch"]
295 patchReprocessing[patchId] = patchReprocessing.get(
296 patchId,
False)
or reprocess
298 reprocessDataset = self.config.coaddName +
"Coadd_multibandReprocessing" 299 for patchId
in patchReprocessing:
300 if not patchReprocessing[patchId]:
302 dataId = dict(tract=tract, patch=patchId)
303 if patchReprocessing[patchId]:
304 filename = butler.get(
305 reprocessDataset +
"_filename", dataId)[0]
306 open(filename,
'a').close()
307 elif butler.datasetExists(reprocessDataset, dataId):
310 patchReprocessing[patchId] =
True 314 not self.config.reprocessing
or patchReprocessing[patchId]])
315 pool.map(self.
runForcedPhot, [dataId1
for dataId1
in dataIdList
if not self.config.reprocessing
or 316 patchReprocessing[dataId[
"patch"]]])
319 if self.config.reprocessing:
320 for patchId
in patchReprocessing:
321 if not patchReprocessing[patchId]:
323 dataId = dict(tract=tract, patch=patchId)
324 filename = butler.get(
325 reprocessDataset +
"_filename", dataId)[0]
329 """! Run detection on a patch 331 Only slave nodes execute this method. 333 @param cache: Pool cache, containing butler 334 @param patchRef: Patch on which to do detection 336 with self.
logOperation(
"do detections on {}".format(patchRef.dataId)):
337 idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
338 coadd = patchRef.get(self.config.coaddName +
"Coadd",
340 detResults = self.detectCoaddSources.
runDetection(coadd, idFactory)
341 self.detectCoaddSources.write(coadd, detResults, patchRef)
344 """!Run detection merging on a patch 346 Only slave nodes execute this method. 348 @param cache: Pool cache, containing butler 349 @param dataIdList: List of data identifiers for the patch in different filters 351 with self.
logOperation(
"merge detections from %s" % (dataIdList,)):
352 dataRefList = [
getDataRef(cache.butler, dataId, self.config.coaddName +
"Coadd_calexp")
for 353 dataId
in dataIdList]
354 if (
not self.config.clobberMergedDetections
and 355 dataRefList[0].datasetExists(self.config.coaddName +
"Coadd_mergeDet")):
357 self.mergeCoaddDetections.
run(dataRefList)
360 """!Run measurement on a patch for a single filter 362 Only slave nodes execute this method. 364 @param cache: Pool cache, with butler 365 @param dataId: Data identifier for patch 366 @return whether the patch requires reprocessing. 368 with self.
logOperation(
"measurement on %s" % (dataId,)):
370 self.config.coaddName +
"Coadd_calexp")
372 if (
not self.config.clobberMeasurements
and 373 dataRef.datasetExists(self.config.coaddName +
"Coadd_meas")):
374 if not self.config.reprocessing:
377 catalog = dataRef.get(self.config.coaddName +
"Coadd_meas")
378 bigFlag = catalog[
"deblend.parent-too-big"]
379 numOldBig = bigFlag.sum()
381 self.log.info(
"No large footprints in %s" %
384 numNewBig = sum((self.measureCoaddSources.deblend.isLargeFootprint(src.getFootprint())
for 385 src
in catalog[bigFlag]))
386 if numNewBig == numOldBig:
387 self.log.info(
"All %d formerly large footprints continue to be large in %s" %
388 (numOldBig, dataRef.dataId,))
390 self.log.info(
"Found %d large footprints to be reprocessed in %s" %
391 (numOldBig - numNewBig, dataRef.dataId))
394 self.measureCoaddSources.
run(dataRef)
398 """!Run measurement merging on a patch 400 Only slave nodes execute this method. 402 @param cache: Pool cache, containing butler 403 @param dataIdList: List of data identifiers for the patch in different filters 405 with self.
logOperation(
"merge measurements from %s" % (dataIdList,)):
406 dataRefList = [
getDataRef(cache.butler, dataId, self.config.coaddName +
"Coadd_calexp")
for 407 dataId
in dataIdList]
408 if (
not self.config.clobberMergedMeasurements
and 409 not self.config.reprocessing
and 410 dataRefList[0].datasetExists(self.config.coaddName +
"Coadd_ref")):
412 self.mergeCoaddMeasurements.
run(dataRefList)
415 """!Run forced photometry on a patch for a single filter 417 Only slave nodes execute this method. 419 @param cache: Pool cache, with butler 420 @param dataId: Data identifier for patch 422 with self.
logOperation(
"forced photometry on %s" % (dataId,)):
424 self.config.coaddName +
"Coadd_calexp")
425 if (
not self.config.clobberForcedPhotometry
and 426 not self.config.reprocessing
and 427 dataRef.datasetExists(self.config.coaddName +
"Coadd_forced_src")):
429 self.forcedPhotCoadd.
run(dataRef)
432 """We don't collect any metadata, so skip"""
def unpickle(factory, args, kwargs)
def __init__(self, butler=None, schema=None, refObjLoader=None, kwargs)
def writeMetadata(self, dataRef)
def getDataRef(butler, dataId, datasetType="raw")
def runForcedPhot(self, cache, dataId)
Run forced photometry on a patch for a single filter.
def batchWallTime(cls, time, parsedCmd, numCpus)
Return walltime request for batch job.
def runMeasureMerged(self, cache, dataId)
Run measurement on a patch for a single filter.
def logOperation(self, operation, catch=False, trace=True)
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def runDetection(self, cache, patchRef)
Run detection on a patch.
def runMergeDetections(self, cache, dataIdList)
Run detection merging on a patch.
def makeTask(self, parsedCmd=None, args=None)
def run(self, patchRefList)
Run multiband processing on coadds.
def runMergeMeasurements(self, cache, dataIdList)
Run measurement merging on a patch.