35 import lsst.pipe.base.connectionTypes
as cT
38 from lsst.skymap
import BaseSkyMap
40 from .references
import MultiBandReferencesTask
41 from .forcedMeasurement
import ForcedMeasurementTask
42 from .applyApCorr
import ApplyApCorrTask
43 from .catalogCalculation
import CatalogCalculationTask
46 from lsst.meas.mosaic
import applyMosaicResults
48 applyMosaicResults =
None
50 __all__ = (
"PerTractCcdDataIdContainer",
"ForcedPhotCcdConfig",
"ForcedPhotCcdTask",
"imageOverlapsTract")
54 """A data ID container which combines raw data IDs with a tract.
58 Required because we need to add "tract" to the raw data ID keys (defined as
59 whatever we use for ``src``) when no tract is provided (so that the user is
60 not required to know which tracts are spanned by the raw data ID).
62 This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is
63 being measured using the detection information, a set of reference
64 catalogs, from the set of coadds which intersect with the calexp. It needs
65 the calexp id (e.g. visit, raft, sensor), but is also uses the tract to
66 decide what set of coadds to use. The references from the tract whose
67 patches intersect with the calexp are used.
71 """Make self.refList from self.idList
73 if self.datasetType
is None:
74 raise RuntimeError(
"Must call setDatasetType first")
75 log = Log.getLogger(
"meas.base.forcedPhotCcd.PerTractCcdDataIdContainer")
77 visitTract = collections.defaultdict(set)
78 visitRefs = collections.defaultdict(list)
79 for dataId
in self.idList:
80 if "tract" not in dataId:
82 log.info(
"Reading WCS for components of dataId=%s to determine tracts", dict(dataId))
84 skymap = namespace.butler.get(namespace.config.coaddName +
"Coadd_skyMap")
86 for ref
in namespace.butler.subset(
"calexp", dataId=dataId):
87 if not ref.datasetExists(
"calexp"):
90 visit = ref.dataId[
"visit"]
91 visitRefs[visit].append(ref)
93 md = ref.get(
"calexp_md", immediate=
True)
98 tract = skymap.findTract(wcs.pixelToSky(box.getCenter()))
100 visitTract[visit].add(tract.getId())
102 self.refList.extend(ref
for ref
in namespace.butler.subset(self.datasetType, dataId=dataId))
105 for visit, tractSet
in visitTract.items():
106 for ref
in visitRefs[visit]:
107 for tract
in tractSet:
108 self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType,
109 dataId=ref.dataId, tract=tract))
111 tractCounter = collections.Counter()
112 for tractSet
in visitTract.values():
113 tractCounter.update(tractSet)
114 log.info(
"Number of visits for each tract: %s", dict(tractCounter))
118 """Return whether the given bounding box overlaps the tract given a WCS.
122 tract : `lsst.skymap.TractInfo`
123 TractInfo specifying a tract.
124 imageWcs : `lsst.afw.geom.SkyWcs`
125 World coordinate system for the image.
126 imageBox : `lsst.geom.Box2I`
127 Bounding box for the image.
132 `True` if the bounding box overlaps the tract; `False` otherwise.
134 tractPoly = tract.getOuterSkyPolygon()
138 imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners)
139 except lsst.pex.exceptions.LsstCppException
as e:
141 if (
not isinstance(e.message, lsst.pex.exceptions.DomainErrorException)
142 and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)):
147 return tractPoly.intersects(imagePoly)
151 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract"),
152 defaultTemplates={
"inputCoaddName":
"deep",
153 "inputName":
"calexp"}):
154 inputSchema = cT.InitInput(
155 doc=
"Schema for the input measurement catalogs.",
156 name=
"{inputCoaddName}Coadd_ref_schema",
157 storageClass=
"SourceCatalog",
159 outputSchema = cT.InitOutput(
160 doc=
"Schema for the output forced measurement catalogs.",
161 name=
"forced_src_schema",
162 storageClass=
"SourceCatalog",
165 doc=
"Input exposure to perform photometry on.",
167 storageClass=
"ExposureF",
168 dimensions=[
"instrument",
"visit",
"detector"],
171 doc=
"Catalog of shapes and positions at which to force photometry.",
172 name=
"{inputCoaddName}Coadd_ref",
173 storageClass=
"SourceCatalog",
174 dimensions=[
"skymap",
"tract",
"patch"],
179 doc=
"SkyMap dataset that defines the coordinate system of the reference catalog.",
180 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
181 storageClass=
"SkyMap",
182 dimensions=[
"skymap"],
185 doc=
"Output forced photometry catalog.",
187 storageClass=
"SourceCatalog",
188 dimensions=[
"instrument",
"visit",
"detector",
"skymap",
"tract"],
192 class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig,
193 pipelineConnections=ForcedPhotCcdConnections):
194 """Config class for forced measurement driver task."""
195 references = lsst.pex.config.ConfigurableField(
196 target=MultiBandReferencesTask,
197 doc=
"subtask to retrieve reference source catalog"
199 measurement = lsst.pex.config.ConfigurableField(
200 target=ForcedMeasurementTask,
201 doc=
"subtask to do forced measurement"
203 coaddName = lsst.pex.config.Field(
204 doc=
"coadd name: typically one of deep or goodSeeing",
208 doApCorr = lsst.pex.config.Field(
211 doc=
"Run subtask to apply aperture corrections"
213 applyApCorr = lsst.pex.config.ConfigurableField(
214 target=ApplyApCorrTask,
215 doc=
"Subtask to apply aperture corrections"
217 catalogCalculation = lsst.pex.config.ConfigurableField(
218 target=CatalogCalculationTask,
219 doc=
"Subtask to run catalogCalculation plugins on catalog"
221 doApplyUberCal = lsst.pex.config.Field(
223 doc=
"Apply meas_mosaic ubercal results to input calexps?",
225 deprecated=
"Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead",
227 doApplyExternalPhotoCalib = lsst.pex.config.Field(
230 doc=(
"Whether to apply external photometric calibration via an "
231 "`lsst.afw.image.PhotoCalib` object. Uses the "
232 "``externalPhotoCalibName`` field to determine which calibration "
235 doApplyExternalSkyWcs = lsst.pex.config.Field(
238 doc=(
"Whether to apply external astrometric calibration via an "
239 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` "
240 "field to determine which calibration to load."),
242 doApplySkyCorr = lsst.pex.config.Field(
245 doc=
"Apply sky correction?",
247 includePhotoCalibVar = lsst.pex.config.Field(
250 doc=
"Add photometric calibration variance to warp variance plane?",
252 externalPhotoCalibName = lsst.pex.config.ChoiceField(
254 doc=(
"Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. "
255 "Unused for Gen3 middleware."),
258 "jointcal":
"Use jointcal_photoCalib",
259 "fgcm":
"Use fgcm_photoCalib",
260 "fgcm_tract":
"Use fgcm_tract_photoCalib"
263 externalSkyWcsName = lsst.pex.config.ChoiceField(
265 doc=
"Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.",
268 "jointcal":
"Use jointcal_wcs"
272 def setDefaults(self):
276 super().setDefaults()
278 self.catalogCalculation.plugins.names = []
281 class ForcedPhotCcdTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
282 """A command-line driver for performing forced measurement on CCD images.
286 butler : `lsst.daf.persistence.butler.Butler`, optional
287 A Butler which will be passed to the references subtask to allow it to
288 load its schema from disk. Optional, but must be specified if
289 ``refSchema`` is not; if both are specified, ``refSchema`` takes
291 refSchema : `lsst.afw.table.Schema`, optional
292 The schema of the reference catalog, passed to the constructor of the
293 references subtask. Optional, but must be specified if ``butler`` is
294 not; if both are specified, ``refSchema`` takes precedence.
296 Keyword arguments are passed to the supertask constructor.
300 The `runDataRef` method takes a `~lsst.daf.persistence.ButlerDataRef` argument
301 that corresponds to a single CCD. This should contain the data ID keys that
302 correspond to the ``forced_src`` dataset (the output dataset for this
303 task), which are typically all those used to specify the ``calexp`` dataset
304 (``visit``, ``raft``, ``sensor`` for LSST data) as well as a coadd tract.
305 The tract is used to look up the appropriate coadd measurement catalogs to
306 use as references (e.g. ``deepCoadd_src``; see
307 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more
308 information). While the tract must be given as part of the dataRef, the
309 patches are determined automatically from the bounding box and WCS of the
310 calexp to be measured, and the filter used to fetch references is set via
311 the ``filter`` option in the configuration of
312 :lsst-task:`lsst.meas.base.references.BaseReferencesTask`).
315 ConfigClass = ForcedPhotCcdConfig
316 RunnerClass = pipeBase.ButlerInitializedTaskRunner
317 _DefaultName =
"forcedPhotCcd"
320 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
321 super().__init__(**kwds)
323 if initInputs
is not None:
324 refSchema = initInputs[
'inputSchema'].schema
326 self.makeSubtask(
"references", butler=butler, schema=refSchema)
327 if refSchema
is None:
328 refSchema = self.references.schema
329 self.makeSubtask(
"measurement", refSchema=refSchema)
332 if self.config.doApCorr:
333 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
334 self.makeSubtask(
'catalogCalculation', schema=self.measurement.schema)
337 def runQuantum(self, butlerQC, inputRefs, outputRefs):
338 inputs = butlerQC.get(inputRefs)
340 tract = butlerQC.quantum.dataId[
'tract']
341 skyMap = inputs.pop(
"skyMap")
342 inputs[
'refWcs'] = skyMap[tract].getWcs()
344 inputs[
'refCat'] = self.mergeAndFilterReferences(inputs[
'exposure'], inputs[
'refCat'],
347 inputs[
'measCat'], inputs[
'exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
349 inputs[
'refCat'], inputs[
'refWcs'],
351 self.attachFootprints(inputs[
'measCat'], inputs[
'refCat'], inputs[
'exposure'], inputs[
'refWcs'])
353 outputs = self.run(**inputs)
354 butlerQC.put(outputs, outputRefs)
356 def mergeAndFilterReferences(self, exposure, refCats, refWcs):
357 """Filter reference catalog so that all sources are within the
358 boundaries of the exposure.
362 exposure : `lsst.afw.image.exposure.Exposure`
363 Exposure to generate the catalog for.
364 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle`
365 Handles for catalogs of shapes and positions at which to force
367 refWcs : `lsst.afw.image.SkyWcs`
368 Reference world coordinate system.
372 refSources : `lsst.afw.table.SourceCatalog`
373 Filtered catalog of forced sources to measure.
377 Filtering the reference catalog is currently handled by Gen2
378 specific methods. To function for Gen3, this method copies
379 code segments to do the filtering and transformation. The
380 majority of this code is based on the methods of
381 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader
387 expWcs = exposure.getWcs()
388 expRegion = exposure.getBBox(lsst.afw.image.PARENT)
390 expBoxCorners = expBBox.getCorners()
391 expSkyCorners = [expWcs.pixelToSky(corner).getVector()
for
392 corner
in expBoxCorners]
401 for refCat
in refCats:
402 refCat = refCat.get()
403 if mergedRefCat
is None:
406 for record
in refCat:
407 if expPolygon.contains(record.getCoord().getVector())
and record.getParent()
in containedIds:
408 record.setFootprint(record.getFootprint().
transform(refWcs, expWcs, expRegion))
409 mergedRefCat.append(record)
410 containedIds.add(record.getId())
411 if mergedRefCat
is None:
412 raise RuntimeError(
"No reference objects for forced photometry.")
416 def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName):
417 """Generate a measurement catalog for Gen3.
421 exposureDataId : `DataId`
422 Butler dataId for this exposure.
423 exposure : `lsst.afw.image.exposure.Exposure`
424 Exposure to generate the catalog for.
425 refCat : `lsst.afw.table.SourceCatalog`
426 Catalog of shapes and positions at which to force photometry.
427 refWcs : `lsst.afw.image.SkyWcs`
428 Reference world coordinate system.
430 Type of ID packer to construct from the registry.
434 measCat : `lsst.afw.table.SourceCatalog`
435 Catalog of forced sources to measure.
437 Unique binary id associated with the input exposure
439 expId, expBits = exposureDataId.pack(idPackerName, returnMaxBits=
True)
442 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
444 return measCat, expId
446 def runDataRef(self, dataRef, psfCache=None):
447 """Perform forced measurement on a single exposure.
451 dataRef : `lsst.daf.persistence.ButlerDataRef`
452 Passed to the ``references`` subtask to obtain the reference WCS,
453 the ``getExposure`` method (implemented by derived classes) to
454 read the measurment image, and the ``fetchReferences`` method to
455 get the exposure and load the reference catalog (see
456 :lsst-task`lsst.meas.base.references.CoaddSrcReferencesTask`).
457 Refer to derived class documentation for details of the datasets
458 and data ID keys which are used.
459 psfCache : `int`, optional
460 Size of PSF cache, or `None`. The size of the PSF cache can have
461 a significant effect upon the runtime for complicated PSF models.
465 Sources are generated with ``generateMeasCat`` in the ``measurement``
466 subtask. These are passed to ``measurement``'s ``run`` method, which
467 fills the source catalog with the forced measurement results. The
468 sources are then passed to the ``writeOutputs`` method (implemented by
469 derived classes) which writes the outputs.
471 refWcs = self.references.getWcs(dataRef)
472 exposure = self.getExposure(dataRef)
473 if psfCache
is not None:
474 exposure.getPsf().setCacheSize(psfCache)
475 refCat = self.fetchReferences(dataRef, exposure)
477 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
478 idFactory=self.makeIdFactory(dataRef))
479 self.log.info(
"Performing forced measurement on %s", dataRef.dataId)
480 self.attachFootprints(measCat, refCat, exposure, refWcs)
482 exposureId = self.getExposureId(dataRef)
484 forcedPhotResult = self.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
486 self.writeOutput(dataRef, forcedPhotResult.measCat)
488 def run(self, measCat, exposure, refCat, refWcs, exposureId=None):
489 """Perform forced measurement on a single exposure.
493 measCat : `lsst.afw.table.SourceCatalog`
494 The measurement catalog, based on the sources listed in the
496 exposure : `lsst.afw.image.Exposure`
497 The measurement image upon which to perform forced detection.
498 refCat : `lsst.afw.table.SourceCatalog`
499 The reference catalog of sources to measure.
500 refWcs : `lsst.afw.image.SkyWcs`
501 The WCS for the references.
503 Optional unique exposureId used for random seed in measurement
508 result : `lsst.pipe.base.Struct`
509 Structure with fields:
512 Catalog of forced measurement results
513 (`lsst.afw.table.SourceCatalog`).
515 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
516 if self.config.doApCorr:
517 self.applyApCorr.run(
519 apCorrMap=exposure.getInfo().getApCorrMap()
521 self.catalogCalculation.run(measCat)
523 return pipeBase.Struct(measCat=measCat)
525 def makeIdFactory(self, dataRef):
526 """Create an object that generates globally unique source IDs.
528 Source IDs are created based on a per-CCD ID and the ID of the CCD
533 dataRef : `lsst.daf.persistence.ButlerDataRef`
534 Butler data reference. The ``ccdExposureId_bits`` and
535 ``ccdExposureId`` datasets are accessed. The data ID must have the
536 keys that correspond to ``ccdExposureId``, which are generally the
537 same as those that correspond to ``calexp`` (``visit``, ``raft``,
538 ``sensor`` for LSST data).
540 expBits = dataRef.get(
"ccdExposureId_bits")
541 expId = int(dataRef.get(
"ccdExposureId"))
545 return int(dataRef.get(
"ccdExposureId", immediate=
True))
548 """Get sources that overlap the exposure.
552 dataRef : `lsst.daf.persistence.ButlerDataRef`
553 Butler data reference corresponding to the image to be measured;
554 should have ``tract``, ``patch``, and ``filter`` keys.
555 exposure : `lsst.afw.image.Exposure`
556 The image to be measured (used only to obtain a WCS and bounding
561 referencs : `lsst.afw.table.SourceCatalog`
562 Catalog of sources that overlap the exposure
566 The returned catalog is sorted by ID and guarantees that all included
567 children have their parent included and that all Footprints are valid.
569 All work is delegated to the references subtask; see
570 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask`
571 for information about the default behavior.
575 unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs())
576 for record
in unfiltered:
577 if record.getFootprint()
is None or record.getFootprint().getArea() == 0:
578 if record.getParent() != 0:
579 self.log.warning(
"Skipping reference %s (child of %s) with bad Footprint",
580 record.getId(), record.getParent())
582 self.log.warning(
"Skipping reference parent %s with bad Footprint", record.getId())
583 badParents.add(record.getId())
584 elif record.getParent()
not in badParents:
585 references.append(record)
591 r"""Attach footprints to blank sources prior to measurements.
595 `~lsst.afw.detection.Footprint`\ s for forced photometry must be in the
596 pixel coordinate system of the image being measured, while the actual
597 detections may start out in a different coordinate system.
599 Subclasses of this class must implement this method to define how
600 those `~lsst.afw.detection.Footprint`\ s should be generated.
602 This default implementation transforms the
603 `~lsst.afw.detection.Footprint`\ s from the reference catalog from the
604 reference WCS to the exposure's WcS, which downgrades
605 `lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s into regular
606 `~lsst.afw.detection.Footprint`\ s, destroying deblend information.
608 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs)
611 """Read input exposure for measurement.
615 dataRef : `lsst.daf.persistence.ButlerDataRef`
616 Butler data reference.
618 exposure = dataRef.get(self.dataPrefix +
"calexp", immediate=
True)
620 if self.config.doApplyExternalPhotoCalib:
621 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
622 self.log.info(
"Applying external photoCalib from %s", source)
623 photoCalib = dataRef.get(source)
624 exposure.setPhotoCalib(photoCalib)
626 if self.config.doApplyExternalSkyWcs:
627 source = f
"{self.config.externalSkyWcsName}_wcs"
628 self.log.info(
"Applying external skyWcs from %s", source)
629 skyWcs = dataRef.get(source)
630 exposure.setWcs(skyWcs)
632 if self.config.doApplySkyCorr:
633 self.log.info(
"Apply sky correction")
634 skyCorr = dataRef.get(
"skyCorr")
635 exposure.maskedImage -= skyCorr.getImage()
640 """Write forced source table
644 dataRef : `lsst.daf.persistence.ButlerDataRef`
645 Butler data reference. The forced_src dataset (with
646 self.dataPrefix prepended) is all that will be modified.
647 sources : `lsst.afw.table.SourceCatalog`
648 Catalog of sources to save.
650 dataRef.put(sources, self.dataPrefix +
"forced_src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
653 """The schema catalogs that will be used by this task.
657 schemaCatalogs : `dict`
658 Dictionary mapping dataset type to schema catalog.
662 There is only one schema for each type of forced measurement. The
663 dataset type for this measurement is defined in the mapper.
666 catalog.getTable().setMetadata(self.measurement.algMetadata)
667 datasetType = self.dataPrefix +
"forced_src"
668 return {datasetType: catalog}
670 def _getConfigName(self):
672 return self.dataPrefix +
"forcedPhotCcd_config"
674 def _getMetadataName(self):
676 return self.dataPrefix +
"forcedPhotCcd_metadata"
679 def _makeArgumentParser(cls):
680 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
681 parser.add_id_argument(
"--id",
"forced_src", help=
"data ID with raw CCD keys [+ tract optionally], "
682 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
683 ContainerClass=PerTractCcdDataIdContainer)
static std::shared_ptr< IdFactory > makeSource(RecordId expId, int reserved)
static Key< RecordId > getParentKey()
def makeDataRefList(self, namespace)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)
std::shared_ptr< SkyWcs > makeSkyWcs(daf::base::PropertySet &metadata, bool strip=false)
lsst::geom::Box2I bboxFromMetadata(daf::base::PropertySet &metadata)
def imageOverlapsTract(tract, imageWcs, imageBox)
def attachFootprints(self, sources, refCat, exposure, refWcs, dataRef)
def writeOutput(self, dataRef, sources)
def fetchReferences(self, dataRef, exposure)
def getExposure(self, dataRef)
def getSchemaCatalogs(self)
def getExposureId(self, dataRef)