27from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
28 PipelineTask, PipelineTaskConfig, PipelineTaskConnections)
29import lsst.pipe.base.connectionTypes
as cT
32from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
34from lsst.meas.extensions.scarlet
import ScarletDeblendTask
45from lsst.obs.base
import ExposureIdInfo
48from .mergeDetections
import MergeDetectionsConfig, MergeDetectionsTask
49from .mergeMeasurements
import MergeMeasurementsConfig, MergeMeasurementsTask
50from .multiBandUtils
import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs
51from .multiBandUtils
import getInputSchema, readCatalog, _makeMakeIdFactory
52from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleConfig
53from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleTask
54from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiConfig
55from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiTask
60* deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
61* deepCoadd_mergeDet: merged detections (tract, patch)
62* deepCoadd_meas: measurements of merged detections (tract, patch, filter)
63* deepCoadd_ref: reference sources (tract, patch)
64All of these have associated *_schema catalogs that require no data ID and hold no records.
66In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
67the mergeDet, meas, and ref dataset Footprints:
68* deepCoadd_peak_schema
74 dimensions=(
"tract",
"patch",
"band",
"skymap"),
75 defaultTemplates={
"inputCoaddName":
"deep",
"outputCoaddName":
"deep"}):
76 detectionSchema = cT.InitOutput(
77 doc=
"Schema of the detection catalog",
78 name=
"{outputCoaddName}Coadd_det_schema",
79 storageClass=
"SourceCatalog",
82 doc=
"Exposure on which detections are to be performed",
83 name=
"{inputCoaddName}Coadd",
84 storageClass=
"ExposureF",
85 dimensions=(
"tract",
"patch",
"band",
"skymap")
87 outputBackgrounds = cT.Output(
88 doc=
"Output Backgrounds used in detection",
89 name=
"{outputCoaddName}Coadd_calexp_background",
90 storageClass=
"Background",
91 dimensions=(
"tract",
"patch",
"band",
"skymap")
93 outputSources = cT.Output(
94 doc=
"Detected sources catalog",
95 name=
"{outputCoaddName}Coadd_det",
96 storageClass=
"SourceCatalog",
97 dimensions=(
"tract",
"patch",
"band",
"skymap")
99 outputExposure = cT.Output(
100 doc=
"Exposure post detection",
101 name=
"{outputCoaddName}Coadd_calexp",
102 storageClass=
"ExposureF",
103 dimensions=(
"tract",
"patch",
"band",
"skymap")
107class DetectCoaddSourcesConfig(PipelineTaskConfig, pipelineConnections=DetectCoaddSourcesConnections):
109 @anchor DetectCoaddSourcesConfig_
111 @brief Configuration parameters
for the DetectCoaddSourcesTask
113 doScaleVariance = Field(dtype=bool, default=True, doc=
"Scale variance plane using empirical noise?")
114 scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc=
"Variance rescaling")
115 detection = ConfigurableField(target=DynamicDetectionTask, doc=
"Source detection")
116 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
117 doInsertFakes = Field(dtype=bool, default=
False,
118 doc=
"Run fake sources injection task",
119 deprecated=(
"doInsertFakes is no longer supported. This config will be removed "
121 insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
122 doc=
"Injection of fake sources for testing "
123 "purposes (must be retargeted)",
124 deprecated=(
"insertFakes is no longer supported. This config will "
125 "be removed after v24."))
129 doc=
"Should be set to True if fake sources have been inserted into the input data.",
132 def setDefaults(self):
133 super().setDefaults()
134 self.detection.thresholdType =
"pixel_stdev"
135 self.detection.isotropicGrow =
True
137 self.detection.reEstimateBackground =
False
138 self.detection.background.useApprox =
False
139 self.detection.background.binSize = 4096
140 self.detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER'
141 self.detection.doTempWideBackground =
True
151class DetectCoaddSourcesTask(PipelineTask, CmdLineTask):
153 @anchor DetectCoaddSourcesTask_
155 @brief Detect sources on a coadd
157 @section pipe_tasks_multiBand_Contents Contents
159 -
@ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
160 -
@ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
161 -
@ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
162 -
@ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
163 -
@ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
164 -
@ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
166 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
168 Command-line task that detects sources on a coadd of exposures obtained
with a single filter.
170 Coadding individual visits requires each exposure to be warped. This introduces covariance
in the noise
171 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
172 in the coadd to match the observed variance. This
is an approximate approach -- strictly, we should
173 propagate the full covariance matrix -- but it
is simple
and works well
in practice.
175 After scaling the variance plane, we detect sources
and generate footprints by delegating to the
@ref
176 SourceDetectionTask_
"detection" subtask.
179 deepCoadd{tract,patch,filter}: ExposureF
181 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
182 @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
184 @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
188 DetectCoaddSourcesTask delegates most of its work to the
@ref SourceDetectionTask_
"detection" subtask.
189 You can retarget this subtask
if you wish.
191 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
193 @copydoc \_\_init\_\_
195 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
199 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
201 See
@ref DetectCoaddSourcesConfig_
"DetectSourcesConfig"
203 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
205 The command line task interface supports a
206 flag
@c -d to
import @b debug.py
from your
@c PYTHONPATH; see
@ref baseDebug
for more about
@b debug.py
209 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
210 @ref SourceDetectionTask_
"SourceDetectionTask"; see the documetation
for
211 @ref SourceDetectionTask_
"SourceDetectionTask" for further information.
213 @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
214 of using DetectCoaddSourcesTask
216 DetectCoaddSourcesTask
is meant to be run after assembling a coadded image
in a given band. The purpose of
217 the task
is to update the background, detect all sources
in a single band
and generate a set of parent
218 footprints. Subsequent tasks
in the multi-band processing procedure will merge sources across bands
and,
219 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
220 reference to the coadd to be processed. A list of the available optional arguments can be obtained by
221 calling detectCoaddSources.py
with the `--help` command line argument:
223 detectCoaddSources.py --help
226 To demonstrate usage of the DetectCoaddSourcesTask
in the larger context of multi-band processing, we
227 will process HSC data
in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
228 steps 1 - 4 at
@ref pipeTasks_multiBand, one may detect all the sources
in each coadd
as follows:
230 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
232 that will process the HSC-I band data. The results are written to
233 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
235 It
is also necessary to run:
237 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
239 to generate the sources catalogs
for the HSC-R band required by the next step
in the multi-band
240 processing procedure:
@ref MergeDetectionsTask_
"MergeDetectionsTask".
242 _DefaultName = "detectCoaddSources"
243 ConfigClass = DetectCoaddSourcesConfig
244 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
245 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
248 def _makeArgumentParser(cls):
249 parser = ArgumentParser(name=cls._DefaultName)
250 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
251 ContainerClass=ExistingCoaddDataIdContainer)
254 def __init__(self, schema=None, **kwargs):
256 @brief Initialize the task. Create the
@ref SourceDetectionTask_
"detection" subtask.
258 Keyword arguments (
in addition to those forwarded to CmdLineTask.__init__):
260 @param[
in] schema: initial schema
for the output catalog, modified-
in place to include all
261 fields set by this task. If
None, the source minimal schema will be used.
262 @param[
in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
266 super().__init__(**kwargs)
268 schema = afwTable.SourceTable.makeMinimalSchema()
270 self.makeSubtask(
"detection", schema=self.schema)
271 if self.config.doScaleVariance:
272 self.makeSubtask(
"scaleVariance")
274 self.detectionSchema = afwTable.SourceCatalog(self.schema)
276 def runDataRef(self, patchRef):
278 @brief Run detection on a coadd.
280 Invokes
@ref run
and then uses
@ref write to output the
283 @param[
in] patchRef: data reference
for patch
285 if self.config.hasFakes:
286 exposure = patchRef.get(
"fakes_" + self.config.coaddName +
"Coadd", immediate=
True)
288 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
289 expId = getGen3CoaddExposureId(patchRef, coaddName=self.config.coaddName, log=self.log)
290 results = self.run(exposure, self.makeIdFactory(patchRef), expId=expId)
291 self.write(results, patchRef)
294 def runQuantum(self, butlerQC, inputRefs, outputRefs):
295 inputs = butlerQC.get(inputRefs)
296 exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"tract_patch_band")
297 inputs[
"idFactory"] = exposureIdInfo.makeSourceIdFactory()
298 inputs[
"expId"] = exposureIdInfo.expId
299 outputs = self.run(**inputs)
300 butlerQC.put(outputs, outputRefs)
302 def run(self, exposure, idFactory, expId):
304 @brief Run detection on an exposure.
306 First scale the variance plane to match the observed variance
307 using
@ref ScaleVarianceTask. Then invoke the
@ref SourceDetectionTask_
"detection" subtask to
310 @param[
in,out] exposure: Exposure on which to detect (may be backround-subtracted
and scaled,
311 depending on configuration).
312 @param[
in] idFactory: IdFactory to set source identifiers
313 @param[
in] expId: Exposure identifier (integer)
for RNG seed
315 @return a pipe.base.Struct
with fields
316 - sources: catalog of detections
317 - backgrounds: list of backgrounds
319 if self.config.doScaleVariance:
320 varScale = self.scaleVariance.run(exposure.maskedImage)
321 exposure.getMetadata().add(
"VARIANCE_SCALE", varScale)
322 backgrounds = afwMath.BackgroundList()
323 table = afwTable.SourceTable.make(self.schema, idFactory)
324 detections = self.detection.run(table, exposure, expId=expId)
325 sources = detections.sources
326 fpSets = detections.fpSets
327 if hasattr(fpSets,
"background")
and fpSets.background:
328 for bg
in fpSets.background:
329 backgrounds.append(bg)
330 return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
332 def write(self, results, patchRef):
334 @brief Write out results
from runDetection.
336 @param[
in] exposure: Exposure to write out
337 @param[
in] results: Struct returned
from runDetection
338 @param[
in] patchRef: data reference
for patch
340 coaddName = self.config.coaddName + "Coadd"
341 patchRef.put(results.outputBackgrounds, coaddName +
"_calexp_background")
342 patchRef.put(results.outputSources, coaddName +
"_det")
343 if self.config.hasFakes:
344 patchRef.put(results.outputExposure,
"fakes_" + coaddName +
"_calexp")
346 patchRef.put(results.outputExposure, coaddName +
"_calexp")
351class DeblendCoaddSourcesConfig(Config):
352 """DeblendCoaddSourcesConfig
354 Configuration parameters for the `DeblendCoaddSourcesTask`.
356 singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
357 doc="Deblend sources separately in each band")
358 multiBandDeblend = ConfigurableField(target=ScarletDeblendTask,
359 doc=
"Deblend sources simultaneously across bands")
360 simultaneous = Field(dtype=bool,
362 doc=
"Simultaneously deblend all bands? "
363 "True uses `multibandDeblend` while False uses `singleBandDeblend`")
364 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
365 hasFakes = Field(dtype=bool,
367 doc=
"Should be set to True if fake sources have been inserted into the input data.")
369 def setDefaults(self):
370 Config.setDefaults(self)
371 self.singleBandDeblend.propagateAllPeaks =
True
375 """Task runner for the `MergeSourcesTask`
377 Required because the run method requires a list of
378 dataRefs rather than a single dataRef.
381 def getTargetList(parsedCmd, **kwargs):
382 """Provide a list of patch references for each patch, tract, filter combo.
389 Keyword arguments passed to the task
394 List of tuples, where each tuple is a (dataRef, kwargs) pair.
396 refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
397 kwargs["psfCache"] = parsedCmd.psfCache
398 return [(list(p.values()), kwargs)
for t
in refDict.values()
for p
in t.values()]
401class DeblendCoaddSourcesTask(CmdLineTask):
402 """Deblend the sources in a merged catalog
404 Deblend sources from master catalog
in each coadd.
405 This can either be done separately
in each band using the HSC-SDSS deblender
406 (`DeblendCoaddSourcesTask.config.simultaneous==
False`)
407 or use SCARLET to simultaneously fit the blend
in all bands
408 (`DeblendCoaddSourcesTask.config.simultaneous==
True`).
409 The task will set its own `self.schema` atribute to the `Schema` of the
410 output deblended catalog.
411 This will include all fields
from the input `Schema`,
as well
as additional fields
414 `pipe.tasks.multiband.DeblendCoaddSourcesTask Description
415 ---------------------------------------------------------
421 Butler used to read the input schemas
from disk
or
422 construct the reference catalog loader,
if `schema`
or `peakSchema`
or
424 The schema of the merged detection catalog
as an input to this task.
426 The schema of the `PeakRecord`s
in the `Footprint`s
in the merged detection catalog
428 ConfigClass = DeblendCoaddSourcesConfig
429 RunnerClass = DeblendCoaddSourcesRunner
430 _DefaultName = "deblendCoaddSources"
431 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId", includeBand=
False)
434 def _makeArgumentParser(cls):
435 parser = ArgumentParser(name=cls._DefaultName)
436 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
437 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
438 ContainerClass=ExistingCoaddDataIdContainer)
439 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
442 def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
443 CmdLineTask.__init__(self, **kwargs)
445 assert butler
is not None,
"Neither butler nor schema is defined"
446 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
447 self.schemaMapper = afwTable.SchemaMapper(schema)
448 self.schemaMapper.addMinimalSchema(schema)
449 self.schema = self.schemaMapper.getOutputSchema()
450 if peakSchema
is None:
451 assert butler
is not None,
"Neither butler nor peakSchema is defined"
452 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
454 if self.config.simultaneous:
455 self.makeSubtask(
"multiBandDeblend", schema=self.schema, peakSchema=peakSchema)
457 self.makeSubtask(
"singleBandDeblend", schema=self.schema, peakSchema=peakSchema)
459 def getSchemaCatalogs(self):
460 """Return a dict of empty catalogs for each catalog dataset produced by this task.
465 Dictionary of empty catalogs, with catalog names
as keys.
467 catalog = afwTable.SourceCatalog(self.schema)
468 return {self.config.coaddName +
"Coadd_deblendedFlux": catalog,
469 self.config.coaddName +
"Coadd_deblendedModel": catalog}
471 def runDataRef(self, patchRefList, psfCache=100):
474 Deblend each source simultaneously or separately
475 (depending on `DeblendCoaddSourcesTask.config.simultaneous`).
476 Set `
is-primary`
and related flags.
477 Propagate flags
from individual visits.
478 Write the deblended sources out.
483 List of data references
for each filter
486 if self.config.hasFakes:
487 coaddType =
"fakes_" + self.config.coaddName
489 coaddType = self.config.coaddName
491 if self.config.simultaneous:
495 for patchRef
in patchRefList:
496 exposure = patchRef.get(coaddType +
"Coadd_calexp", immediate=
True)
497 filter = patchRef.get(coaddType +
"Coadd_filterLabel", immediate=
True)
498 filters.append(filter.bandLabel)
499 exposures.append(exposure)
501 exposures = [exposure
for _, exposure
in sorted(zip(filters, exposures))]
502 patchRefList = [patchRef
for _, patchRef
in sorted(zip(filters, patchRefList))]
505 sources = self.readSources(patchRef)
506 exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
507 templateCatalogs, fluxCatalogs = self.multiBandDeblend.run(exposure, sources)
508 for n
in range(len(patchRefList)):
509 self.write(patchRefList[n], templateCatalogs[filters[n]],
"Model")
510 if filters[n]
in fluxCatalogs:
511 self.write(patchRefList[n], fluxCatalogs[filters[n]],
"Flux")
514 for patchRef
in patchRefList:
515 exposure = patchRef.get(coaddType +
"Coadd_calexp", immediate=
True)
516 exposure.getPsf().setCacheCapacity(psfCache)
517 sources = self.readSources(patchRef)
518 self.singleBandDeblend.run(exposure, sources)
519 self.write(patchRef, sources)
521 def readSources(self, dataRef):
522 """Read merged catalog
524 Read the catalog of merged detections and create a catalog
529 dataRef: data reference
530 Data reference
for catalog of merged detections
534 sources: `SourceCatalog`
535 List of sources
in merged catalog
537 We also need to add columns to hold the measurements we
're about to make so we can measure in-place.
539 merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=
True)
540 self.log.info(
"Read %d detections: %s", len(merged), dataRef.dataId)
541 idFactory = self.makeIdFactory(dataRef)
545 maxId = np.max(merged[
"id"])
546 idFactory.notify(maxId)
547 table = afwTable.SourceTable.make(self.schema, idFactory)
548 sources = afwTable.SourceCatalog(table)
549 sources.extend(merged, self.schemaMapper)
552 def write(self, dataRef, sources, catalogType):
553 """Write the source catalog(s)
557 dataRef: Data Reference
558 Reference to the output catalog.
559 sources: `SourceCatalog`
560 Flux conserved sources to write to file.
561 If using the single band deblender, this is the catalog
563 template_sources: `SourceCatalog`
564 Source catalog using the multiband template models
567 dataRef.put(sources, self.config.coaddName + f"Coadd_deblended{catalogType}")
568 self.log.info(
"Wrote %d sources: %s", len(sources), dataRef.dataId)
571 """Write the metadata produced from processing the data.
575 List of Butler data references used to write the metadata.
576 The metadata is written to dataset type `CmdLineTask._getMetadataName`.
578 for dataRef
in dataRefList:
580 metadataName = self._getMetadataName()
581 if metadataName
is not None:
582 dataRef.put(self.getFullMetadata(), metadataName)
583 except Exception
as e:
584 self.log.warning(
"Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
587class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
588 dimensions=(
"tract",
"patch",
"band",
"skymap"),
589 defaultTemplates={
"inputCoaddName":
"deep",
590 "outputCoaddName":
"deep",
591 "deblendedCatalog":
"deblendedFlux"}):
592 inputSchema = cT.InitInput(
593 doc=
"Input schema for measure merged task produced by a deblender or detection task",
594 name=
"{inputCoaddName}Coadd_deblendedFlux_schema",
595 storageClass=
"SourceCatalog"
597 outputSchema = cT.InitOutput(
598 doc=
"Output schema after all new fields are added by task",
599 name=
"{inputCoaddName}Coadd_meas_schema",
600 storageClass=
"SourceCatalog"
602 refCat = cT.PrerequisiteInput(
603 doc=
"Reference catalog used to match measured sources against known sources",
605 storageClass=
"SimpleCatalog",
606 dimensions=(
"skypix",),
611 doc=
"Input coadd image",
612 name=
"{inputCoaddName}Coadd_calexp",
613 storageClass=
"ExposureF",
614 dimensions=(
"tract",
"patch",
"band",
"skymap")
617 doc=
"SkyMap to use in processing",
618 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
619 storageClass=
"SkyMap",
620 dimensions=(
"skymap",),
622 visitCatalogs = cT.Input(
623 doc=
"Source catalogs for visits which overlap input tract, patch, band. Will be "
624 "further filtered in the task for the purpose of propagating flags from image calibration "
625 "and characterization to coadd objects. Only used in legacy PropagateVisitFlagsTask.",
627 dimensions=(
"instrument",
"visit",
"detector"),
628 storageClass=
"SourceCatalog",
631 sourceTableHandles = cT.Input(
632 doc=(
"Source tables that are derived from the ``CalibrateTask`` sources. "
633 "These tables contain astrometry and photometry flags, and optionally "
635 name=
"sourceTable_visit",
636 storageClass=
"DataFrame",
637 dimensions=(
"instrument",
"visit"),
641 finalizedSourceTableHandles = cT.Input(
642 doc=(
"Finalized source tables from ``FinalizeCalibrationTask``. These "
643 "tables contain PSF flags from the finalized PSF estimation."),
644 name=
"finalized_src_table",
645 storageClass=
"DataFrame",
646 dimensions=(
"instrument",
"visit"),
650 inputCatalog = cT.Input(
651 doc=(
"Name of the input catalog to use."
652 "If the single band deblender was used this should be 'deblendedFlux."
653 "If the multi-band deblender was used this should be 'deblendedModel, "
654 "or deblendedFlux if the multiband deblender was configured to output "
655 "deblended flux catalogs. If no deblending was performed this should "
657 name=
"{inputCoaddName}Coadd_{deblendedCatalog}",
658 storageClass=
"SourceCatalog",
659 dimensions=(
"tract",
"patch",
"band",
"skymap"),
661 outputSources = cT.Output(
662 doc=
"Source catalog containing all the measurement information generated in this task",
663 name=
"{outputCoaddName}Coadd_meas",
664 dimensions=(
"tract",
"patch",
"band",
"skymap"),
665 storageClass=
"SourceCatalog",
667 matchResult = cT.Output(
668 doc=
"Match catalog produced by configured matcher, optional on doMatchSources",
669 name=
"{outputCoaddName}Coadd_measMatch",
670 dimensions=(
"tract",
"patch",
"band",
"skymap"),
671 storageClass=
"Catalog",
673 denormMatches = cT.Output(
674 doc=
"Denormalized Match catalog produced by configured matcher, optional on "
675 "doWriteMatchesDenormalized",
676 name=
"{outputCoaddName}Coadd_measMatchFull",
677 dimensions=(
"tract",
"patch",
"band",
"skymap"),
678 storageClass=
"Catalog",
681 def __init__(self, *, config=None):
682 super().__init__(config=config)
683 if config.doPropagateFlags
is False:
684 self.inputs -= set((
"visitCatalogs",))
685 self.inputs -= set((
"sourceTableHandles",))
686 self.inputs -= set((
"finalizedSourceTableHandles",))
687 elif config.propagateFlags.target == PropagateSourceFlagsTask:
689 self.inputs -= set((
"visitCatalogs",))
691 if not config.propagateFlags.source_flags:
692 self.inputs -= set((
"sourceTableHandles",))
693 if not config.propagateFlags.finalized_source_flags:
694 self.inputs -= set((
"finalizedSourceTableHandles",))
697 self.inputs -= set((
"sourceTableHandles",))
698 self.inputs -= set((
"finalizedSourceTableHandles",))
700 if config.doMatchSources
is False:
701 self.outputs -= set((
"matchResult",))
703 if config.doWriteMatchesDenormalized
is False:
704 self.outputs -= set((
"denormMatches",))
707class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
708 pipelineConnections=MeasureMergedCoaddSourcesConnections):
710 @anchor MeasureMergedCoaddSourcesConfig_
712 @brief Configuration parameters
for the MeasureMergedCoaddSourcesTask
714 inputCatalog = Field(dtype=str, default="deblendedFlux",
715 doc=(
"Name of the input catalog to use."
716 "If the single band deblender was used this should be 'deblendedFlux."
717 "If the multi-band deblender was used this should be 'deblendedModel."
718 "If no deblending was performed this should be 'mergeDet'"))
719 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc=
"Source measurement")
720 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
721 doPropagateFlags = Field(
722 dtype=bool, default=
True,
723 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
725 propagateFlags = ConfigurableField(target=PropagateSourceFlagsTask, doc=
"Propagate source flags to coadd")
726 doMatchSources = Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
727 match = ConfigurableField(target=DirectMatchTask, doc=
"Matching to reference catalog")
728 doWriteMatchesDenormalized = Field(
731 doc=(
"Write reference matches in denormalized format? "
732 "This format uses more disk space, but is more convenient to read."),
734 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
735 psfCache = Field(dtype=int, default=100, doc=
"Size of psfCache")
736 checkUnitsParseStrict = Field(
737 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
744 doc=
"Apply aperture corrections"
746 applyApCorr = ConfigurableField(
747 target=ApplyApCorrTask,
748 doc=
"Subtask to apply aperture corrections"
750 doRunCatalogCalculation = Field(
753 doc=
'Run catalogCalculation task'
755 catalogCalculation = ConfigurableField(
756 target=CatalogCalculationTask,
757 doc=
"Subtask to run catalogCalculation plugins on catalog"
763 doc=
"Should be set to True if fake sources have been inserted into the input data."
767 def refObjLoader(self):
768 return self.match.refObjLoader
770 def setDefaults(self):
771 super().setDefaults()
772 self.measurement.plugins.names |= [
'base_InputCount',
774 'base_LocalPhotoCalib',
776 self.measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED',
'SENSOR_EDGE',
778 self.measurement.plugins[
'base_PixelFlags'].masksFpCenter = [
'CLIPPED',
'SENSOR_EDGE',
783 refCatGen2 = getattr(self.refObjLoader,
"ref_dataset_name",
None)
784 if refCatGen2
is not None and refCatGen2 != self.connections.refCat:
786 f
"Gen2 ({refCatGen2}) and Gen3 ({self.connections.refCat}) reference catalogs "
787 f
"are different. These options must be kept in sync until Gen2 is retired."
799class MeasureMergedCoaddSourcesRunner(ButlerInitializedTaskRunner):
800 """Get the psfCache setting into MeasureMergedCoaddSourcesTask"""
802 def getTargetList(parsedCmd, **kwargs):
803 return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
806class MeasureMergedCoaddSourcesTask(PipelineTask, CmdLineTask):
808 @anchor MeasureMergedCoaddSourcesTask_
810 @brief Deblend sources
from master catalog
in each coadd seperately
and measure.
812 @section pipe_tasks_multiBand_Contents Contents
814 -
@ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
815 -
@ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
816 -
@ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
817 -
@ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
818 -
@ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
819 -
@ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
821 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
823 Command-line task that uses peaks
and footprints
from a master catalog to perform deblending
and
824 measurement
in each coadd.
826 Given a master input catalog of sources (peaks
and footprints)
or deblender outputs
827 (including a HeavyFootprint
in each band), measure each source on the
828 coadd. Repeating this procedure
with the same master catalog across multiple coadds will generate a
829 consistent set of child sources.
831 The deblender retains all peaks
and deblends any missing peaks (dropouts
in that band)
as PSFs. Source
832 properties are measured
and the
@c is-primary flag (indicating sources
with no children)
is set. Visit
833 flags are propagated to the coadd sources.
835 Optionally, we can match the coadd sources to an external reference catalog.
838 deepCoadd_mergeDet{tract,patch}
or deepCoadd_deblend{tract,patch}: SourceCatalog
839 @n deepCoadd_calexp{tract,patch,filter}: ExposureF
841 deepCoadd_meas{tract,patch,filter}: SourceCatalog
845 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
848 <DT>
@ref SingleFrameMeasurementTask_
"measurement"
849 <DD> Measure source properties of deblended sources.</DD>
850 <DT>
@ref SetPrimaryFlagsTask_
"setPrimaryFlags"
851 <DD> Set flag
'is-primary' as well
as related flags on sources.
'is-primary' is set
for sources that are
852 not at the edge of the field
and that have either
not been deblended
or are the children of deblended
854 <DT>
@ref PropagateVisitFlagsTask_
"propagateFlags"
855 <DD> Propagate flags set
in individual visits to the coadd.</DD>
856 <DT>
@ref DirectMatchTask_
"match"
857 <DD> Match input sources to a reference catalog (optional).
860 These subtasks may be retargeted
as required.
862 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
864 @copydoc \_\_init\_\_
866 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
870 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
872 See
@ref MeasureMergedCoaddSourcesConfig_
874 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
876 The command line task interface supports a
877 flag
@c -d to
import @b debug.py
from your
@c PYTHONPATH; see
@ref baseDebug
for more about
@b debug.py
880 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
881 the various sub-tasks. See the documetation
for individual sub-tasks
for more information.
883 @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
884 MeasureMergedCoaddSourcesTask
886 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
887 The next stage
in the multi-band processing procedure will merge these measurements into a suitable
888 catalog
for driving forced photometry.
890 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
892 A list of the available optional arguments can be obtained by calling measureCoaddSources.py
with the
893 `--help` command line argument:
895 measureCoaddSources.py --help
898 To demonstrate usage of the DetectCoaddSourcesTask
in the larger context of multi-band processing, we
899 will process HSC data
in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
900 step 6 at
@ref pipeTasks_multiBand, one may perform deblending
and measure sources
in the HSC-I band
903 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
905 This will process the HSC-I band data. The results are written
in
906 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
908 It
is also necessary to run
910 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
912 to generate the sources catalogs
for the HSC-R band required by the next step
in the multi-band
913 procedure:
@ref MergeMeasurementsTask_
"MergeMeasurementsTask".
915 _DefaultName = "measureCoaddSources"
916 ConfigClass = MeasureMergedCoaddSourcesConfig
917 RunnerClass = MeasureMergedCoaddSourcesRunner
918 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
920 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId", includeBand=
False)
923 def _makeArgumentParser(cls):
924 parser = ArgumentParser(name=cls._DefaultName)
925 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
926 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
927 ContainerClass=ExistingCoaddDataIdContainer)
928 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
931 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
934 @brief Initialize the task.
936 Keyword arguments (
in addition to those forwarded to CmdLineTask.__init__):
937 @param[
in] schema: the schema of the merged detection catalog used
as input to this one
938 @param[
in] peakSchema: the schema of the PeakRecords
in the Footprints
in the merged detection catalog
939 @param[
in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
940 catalog. May be
None if the loader can be constructed
from the butler argument
or all steps
941 requiring a reference catalog are disabled.
942 @param[
in] butler: a butler used to read the input schemas
from disk
or construct the reference
943 catalog loader,
if schema
or peakSchema
or refObjLoader
is None
945 The task will set its own self.schema attribute to the schema of the output measurement catalog.
946 This will include all fields
from the input schema,
as well
as additional fields
for all the
949 super().__init__(**kwargs)
950 self.deblended = self.config.inputCatalog.startswith("deblended")
951 self.inputCatalog =
"Coadd_" + self.config.inputCatalog
952 if initInputs
is not None:
953 schema = initInputs[
'inputSchema'].schema
955 assert butler
is not None,
"Neither butler nor schema is defined"
956 schema = butler.get(self.config.coaddName + self.inputCatalog +
"_schema", immediate=
True).schema
957 self.schemaMapper = afwTable.SchemaMapper(schema)
958 self.schemaMapper.addMinimalSchema(schema)
959 self.schema = self.schemaMapper.getOutputSchema()
961 self.makeSubtask(
"measurement", schema=self.schema, algMetadata=self.algMetadata)
962 self.makeSubtask(
"setPrimaryFlags", schema=self.schema)
963 if self.config.doMatchSources:
964 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
965 if self.config.doPropagateFlags:
966 self.makeSubtask(
"propagateFlags", schema=self.schema)
967 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
968 if self.config.doApCorr:
969 self.makeSubtask(
"applyApCorr", schema=self.schema)
970 if self.config.doRunCatalogCalculation:
971 self.makeSubtask(
"catalogCalculation", schema=self.schema)
973 self.outputSchema = afwTable.SourceCatalog(self.schema)
975 def runQuantum(self, butlerQC, inputRefs, outputRefs):
976 inputs = butlerQC.get(inputRefs)
978 refObjLoader = ReferenceObjectLoader([ref.datasetRef.dataId
for ref
in inputRefs.refCat],
979 inputs.pop(
'refCat'), config=self.config.refObjLoader,
981 self.match.setRefObjLoader(refObjLoader)
985 inputs[
'exposure'].getPsf().setCacheCapacity(self.config.psfCache)
988 exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"tract_patch")
989 inputs[
'exposureId'] = exposureIdInfo.expId
990 idFactory = exposureIdInfo.makeSourceIdFactory()
992 table = afwTable.SourceTable.make(self.schema, idFactory)
993 sources = afwTable.SourceCatalog(table)
994 sources.extend(inputs.pop(
'inputCatalog'), self.schemaMapper)
995 table = sources.getTable()
996 table.setMetadata(self.algMetadata)
997 inputs[
'sources'] = sources
999 skyMap = inputs.pop(
'skyMap')
1000 tractNumber = inputRefs.inputCatalog.dataId[
'tract']
1001 tractInfo = skyMap[tractNumber]
1002 patchInfo = tractInfo.getPatchInfo(inputRefs.inputCatalog.dataId[
'patch'])
1005 tractInfo=tractInfo,
1006 patchInfo=patchInfo,
1007 wcs=tractInfo.getWcs(),
1008 bbox=patchInfo.getOuterBBox()
1010 inputs[
'skyInfo'] = skyInfo
1012 if self.config.doPropagateFlags:
1013 if self.config.propagateFlags.target == PropagateSourceFlagsTask:
1015 ccdInputs = inputs[
"exposure"].getInfo().getCoaddInputs().ccds
1016 inputs[
"ccdInputs"] = ccdInputs
1018 if "sourceTableHandles" in inputs:
1019 sourceTableHandles = inputs.pop(
"sourceTableHandles")
1020 sourceTableHandleDict = {handle.dataId[
"visit"]: handle
1021 for handle
in sourceTableHandles}
1022 inputs[
"sourceTableHandleDict"] = sourceTableHandleDict
1023 if "finalizedSourceTableHandles" in inputs:
1024 finalizedSourceTableHandles = inputs.pop(
"finalizedSourceTableHandles")
1025 finalizedSourceTableHandleDict = {handle.dataId[
"visit"]: handle
1026 for handle
in finalizedSourceTableHandles}
1027 inputs[
"finalizedSourceTableHandleDict"] = finalizedSourceTableHandleDict
1031 ccdInputs = inputs[
'exposure'].getInfo().getCoaddInputs().ccds
1032 visitKey = ccdInputs.schema.find(
"visit").key
1033 ccdKey = ccdInputs.schema.find(
"ccd").key
1034 inputVisitIds = set()
1036 for ccdRecord
in ccdInputs:
1037 visit = ccdRecord.get(visitKey)
1038 ccd = ccdRecord.get(ccdKey)
1039 inputVisitIds.add((visit, ccd))
1040 ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs()
1042 inputCatalogsToKeep = []
1043 inputCatalogWcsUpdate = []
1044 for i, dataRef
in enumerate(inputRefs.visitCatalogs):
1045 key = (dataRef.dataId[
'visit'], dataRef.dataId[
'detector'])
1046 if key
in inputVisitIds:
1047 inputCatalogsToKeep.append(inputs[
'visitCatalogs'][i])
1048 inputCatalogWcsUpdate.append(ccdRecordsWcs[key])
1049 inputs[
'visitCatalogs'] = inputCatalogsToKeep
1050 inputs[
'wcsUpdates'] = inputCatalogWcsUpdate
1051 inputs[
'ccdInputs'] = ccdInputs
1053 outputs = self.run(**inputs)
1054 butlerQC.put(outputs, outputRefs)
1056 def runDataRef(self, patchRef, psfCache=100):
1058 @brief Deblend
and measure.
1060 @param[
in] patchRef: Patch reference.
1062 Set
'is-primary' and related flags. Propagate flags
1063 from individual visits. Optionally match the sources to a reference catalog
and write the matches.
1064 Finally, write the deblended sources
and measurements out.
1066 if self.config.hasFakes:
1067 coaddType =
"fakes_" + self.config.coaddName
1069 coaddType = self.config.coaddName
1070 exposure = patchRef.get(coaddType +
"Coadd_calexp", immediate=
True)
1071 exposure.getPsf().setCacheCapacity(psfCache)
1072 sources = self.readSources(patchRef)
1073 table = sources.getTable()
1074 table.setMetadata(self.algMetadata)
1075 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1077 if self.config.doPropagateFlags:
1078 ccdInputs = self.propagateFlags.getCcdInputs(exposure)
1082 expId = getGen3CoaddExposureId(patchRef, coaddName=self.config.coaddName, includeBand=
False,
1084 results = self.run(exposure=exposure, sources=sources, skyInfo=skyInfo, exposureId=expId,
1085 ccdInputs=ccdInputs, butler=patchRef.getButler())
1087 if self.config.doMatchSources:
1088 self.writeMatches(patchRef, results)
1089 self.write(patchRef, results.outputSources)
1091 def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None,
1092 butler=None, sourceTableHandleDict=None, finalizedSourceTableHandleDict=None):
1093 """Run measurement algorithms on the input exposure, and optionally populate the
1094 resulting catalog with extra information.
1098 exposure : `lsst.afw.exposure.Exposure`
1099 The input exposure on which measurements are to be performed
1101 A catalog built
from the results of merged detections,
or
1103 skyInfo : `lsst.pipe.base.Struct`
1104 A struct containing information about the position of the input exposure within
1105 a `SkyMap`, the `SkyMap`, its `Wcs`,
and its bounding box
1106 exposureId : `int`
or `bytes`
1107 packed unique number
or bytes unique to the input exposure
1109 Catalog containing information on the individual visits which went into making
1111 sourceTableHandleDict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`]
1112 Dict
for sourceTable_visit handles (key
is visit)
for propagating flags.
1113 These tables are derived
from the ``CalibrateTask`` sources,
and contain
1114 astrometry
and photometry flags,
and optionally PSF flags.
1115 finalizedSourceTableHandleDict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`], optional
1116 Dict
for finalized_src_table handles (key
is visit)
for propagating flags.
1117 These tables are derived
from ``FinalizeCalibrationTask``
and contain
1118 PSF flags
from the finalized PSF estimation.
1119 visitCatalogs : list of `lsst.afw.table.SourceCatalogs`
1120 A list of source catalogs corresponding to measurements made on the individual
1121 visits which went into the input exposure. If
None and butler
is `
None` then
1122 the task cannot propagate visit flags to the output catalog.
1123 Deprecated, to be removed
with PropagateVisitFlagsTask.
1125 If visitCatalogs
is not `
None` this should be a list of wcs objects which correspond
1126 to the input visits. Used to put all coordinates to common system. If `
None`
and
1127 butler
is `
None` then the task cannot propagate visit flags to the output catalog.
1128 Deprecated, to be removed
with PropagateVisitFlagsTask.
1129 butler : `lsst.daf.persistence.Butler`
1130 A gen2 butler used to load visit catalogs.
1131 Deprecated, to be removed
with Gen2.
1135 results : `lsst.pipe.base.Struct`
1136 Results of running measurement task. Will contain the catalog
in the
1137 sources attribute. Optionally will have results of matching to a
1138 reference catalog
in the matchResults attribute,
and denormalized
1139 matches
in the denormMatches attribute.
1141 self.measurement.run(sources, exposure, exposureId=exposureId)
1143 if self.config.doApCorr:
1144 self.applyApCorr.run(
1146 apCorrMap=exposure.getInfo().getApCorrMap()
1153 if not sources.isContiguous():
1154 sources = sources.copy(deep=
True)
1156 if self.config.doRunCatalogCalculation:
1157 self.catalogCalculation.run(sources)
1159 self.setPrimaryFlags.run(sources, skyMap=skyInfo.skyMap, tractInfo=skyInfo.tractInfo,
1160 patchInfo=skyInfo.patchInfo)
1161 if self.config.doPropagateFlags:
1162 if self.config.propagateFlags.target == PropagateSourceFlagsTask:
1164 self.propagateFlags.run(
1167 sourceTableHandleDict,
1168 finalizedSourceTableHandleDict
1172 self.propagateFlags.run(
1183 if self.config.doMatchSources:
1184 matchResult = self.match.run(sources, exposure.getInfo().getFilterLabel().bandLabel)
1185 matches = afwTable.packMatches(matchResult.matches)
1186 matches.table.setMetadata(matchResult.matchMeta)
1187 results.matchResult = matches
1188 if self.config.doWriteMatchesDenormalized:
1189 if matchResult.matches:
1190 denormMatches = denormalizeMatches(matchResult.matches, matchResult.matchMeta)
1192 self.log.warning(
"No matches, so generating dummy denormalized matches file")
1193 denormMatches = afwTable.BaseCatalog(afwTable.Schema())
1195 denormMatches.getMetadata().add(
"COMMENT",
1196 "This catalog is empty because no matches were found.")
1197 results.denormMatches = denormMatches
1198 results.denormMatches = denormMatches
1200 results.outputSources = sources
1203 def readSources(self, dataRef):
1205 @brief Read input sources.
1207 @param[
in] dataRef: Data reference
for catalog of merged detections
1208 @return List of sources
in merged catalog
1210 We also need to add columns to hold the measurements we
're about to make so we can measure in-place.
1212 merged = dataRef.get(self.config.coaddName + self.inputCatalog, immediate=True)
1213 self.log.info(
"Read %d detections: %s", len(merged), dataRef.dataId)
1214 idFactory = self.makeIdFactory(dataRef)
1216 idFactory.notify(s.getId())
1217 table = afwTable.SourceTable.make(self.schema, idFactory)
1218 sources = afwTable.SourceCatalog(table)
1219 sources.extend(merged, self.schemaMapper)
1222 def writeMatches(self, dataRef, results):
1224 @brief Write matches of the sources to the astrometric reference catalog.
1226 @param[
in] dataRef: data reference
1227 @param[
in] results: results struct
from run method
1229 if hasattr(results,
"matchResult"):
1230 dataRef.put(results.matchResult, self.config.coaddName +
"Coadd_measMatch")
1231 if hasattr(results,
"denormMatches"):
1232 dataRef.put(results.denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1234 def write(self, dataRef, sources):
1236 @brief Write the source catalog.
1238 @param[
in] dataRef: data reference
1239 @param[
in] sources: source catalog
1241 dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1242 self.log.info(
"Wrote %d sources: %s", len(sources), dataRef.dataId)
1243
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded.
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.