35 import lsst.pipe.base.connectionTypes
as cT
38 from lsst.skymap
import BaseSkyMap
40 from .references
import MultiBandReferencesTask
41 from .forcedMeasurement
import ForcedMeasurementTask
42 from .applyApCorr
import ApplyApCorrTask
43 from .catalogCalculation
import CatalogCalculationTask
46 from lsst.meas.mosaic
import applyMosaicResults
48 applyMosaicResults =
None
50 __all__ = (
"PerTractCcdDataIdContainer",
"ForcedPhotCcdConfig",
"ForcedPhotCcdTask",
"imageOverlapsTract")
54 """A data ID container which combines raw data IDs with a tract.
58 Required because we need to add "tract" to the raw data ID keys (defined as
59 whatever we use for ``src``) when no tract is provided (so that the user is
60 not required to know which tracts are spanned by the raw data ID).
62 This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is
63 being measured using the detection information, a set of reference
64 catalogs, from the set of coadds which intersect with the calexp. It needs
65 the calexp id (e.g. visit, raft, sensor), but is also uses the tract to
66 decide what set of coadds to use. The references from the tract whose
67 patches intersect with the calexp are used.
71 """Make self.refList from self.idList
73 if self.datasetType
is None:
74 raise RuntimeError(
"Must call setDatasetType first")
75 log = Log.getLogger(
"meas.base.forcedPhotCcd.PerTractCcdDataIdContainer")
77 visitTract = collections.defaultdict(set)
78 visitRefs = collections.defaultdict(list)
79 for dataId
in self.idList:
80 if "tract" not in dataId:
82 log.info(
"Reading WCS for components of dataId=%s to determine tracts", dict(dataId))
84 skymap = namespace.butler.get(namespace.config.coaddName +
"Coadd_skyMap")
86 for ref
in namespace.butler.subset(
"calexp", dataId=dataId):
87 if not ref.datasetExists(
"calexp"):
90 visit = ref.dataId[
"visit"]
91 visitRefs[visit].append(ref)
93 md = ref.get(
"calexp_md", immediate=
True)
98 tract = skymap.findTract(wcs.pixelToSky(box.getCenter()))
100 visitTract[visit].add(tract.getId())
102 self.refList.extend(ref
for ref
in namespace.butler.subset(self.datasetType, dataId=dataId))
105 for visit, tractSet
in visitTract.items():
106 for ref
in visitRefs[visit]:
107 for tract
in tractSet:
108 self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType,
109 dataId=ref.dataId, tract=tract))
111 tractCounter = collections.Counter()
112 for tractSet
in visitTract.values():
113 tractCounter.update(tractSet)
114 log.info(
"Number of visits for each tract: %s", dict(tractCounter))
118 """Return whether the given bounding box overlaps the tract given a WCS.
122 tract : `lsst.skymap.TractInfo`
123 TractInfo specifying a tract.
124 imageWcs : `lsst.afw.geom.SkyWcs`
125 World coordinate system for the image.
126 imageBox : `lsst.geom.Box2I`
127 Bounding box for the image.
132 `True` if the bounding box overlaps the tract; `False` otherwise.
134 tractPoly = tract.getOuterSkyPolygon()
138 imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners)
139 except lsst.pex.exceptions.LsstCppException
as e:
141 if (
not isinstance(e.message, lsst.pex.exceptions.DomainErrorException)
142 and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)):
147 return tractPoly.intersects(imagePoly)
151 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract"),
152 defaultTemplates={
"inputCoaddName":
"deep",
153 "inputName":
"calexp"}):
154 inputSchema = cT.InitInput(
155 doc=
"Schema for the input measurement catalogs.",
156 name=
"{inputCoaddName}Coadd_ref_schema",
157 storageClass=
"SourceCatalog",
159 outputSchema = cT.InitOutput(
160 doc=
"Schema for the output forced measurement catalogs.",
161 name=
"forced_src_schema",
162 storageClass=
"SourceCatalog",
165 doc=
"Input exposure to perform photometry on.",
167 storageClass=
"ExposureF",
168 dimensions=[
"instrument",
"visit",
"detector"],
171 doc=
"Catalog of shapes and positions at which to force photometry.",
172 name=
"{inputCoaddName}Coadd_ref",
173 storageClass=
"SourceCatalog",
174 dimensions=[
"skymap",
"tract",
"patch"],
178 doc=
"SkyMap dataset that defines the coordinate system of the reference catalog.",
179 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
180 storageClass=
"SkyMap",
181 dimensions=[
"skymap"],
184 doc=
"Output forced photometry catalog.",
186 storageClass=
"SourceCatalog",
187 dimensions=[
"instrument",
"visit",
"detector",
"skymap",
"tract"],
191 class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig,
192 pipelineConnections=ForcedPhotCcdConnections):
193 """Config class for forced measurement driver task."""
195 references = lsst.pex.config.ConfigurableField(
196 target=MultiBandReferencesTask,
197 doc=
"subtask to retrieve reference source catalog"
199 measurement = lsst.pex.config.ConfigurableField(
200 target=ForcedMeasurementTask,
201 doc=
"subtask to do forced measurement"
203 coaddName = lsst.pex.config.Field(
204 doc=
"coadd name: typically one of deep or goodSeeing",
208 doApCorr = lsst.pex.config.Field(
211 doc=
"Run subtask to apply aperture corrections"
213 applyApCorr = lsst.pex.config.ConfigurableField(
214 target=ApplyApCorrTask,
215 doc=
"Subtask to apply aperture corrections"
217 catalogCalculation = lsst.pex.config.ConfigurableField(
218 target=CatalogCalculationTask,
219 doc=
"Subtask to run catalogCalculation plugins on catalog"
221 doApplyUberCal = lsst.pex.config.Field(
223 doc=
"Apply meas_mosaic ubercal results to input calexps?",
225 deprecated=
"Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead",
227 doApplyExternalPhotoCalib = lsst.pex.config.Field(
230 doc=(
"Whether to apply external photometric calibration via an "
231 "`lsst.afw.image.PhotoCalib` object. Uses the "
232 "``externalPhotoCalibName`` field to determine which calibration "
235 doApplyExternalSkyWcs = lsst.pex.config.Field(
238 doc=(
"Whether to apply external astrometric calibration via an "
239 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` "
240 "field to determine which calibration to load."),
242 doApplySkyCorr = lsst.pex.config.Field(
245 doc=
"Apply sky correction?",
247 includePhotoCalibVar = lsst.pex.config.Field(
250 doc=
"Add photometric calibration variance to warp variance plane?",
252 externalPhotoCalibName = lsst.pex.config.ChoiceField(
254 doc=(
"Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. "
255 "Unused for Gen3 middleware."),
258 "jointcal":
"Use jointcal_photoCalib",
259 "fgcm":
"Use fgcm_photoCalib",
260 "fgcm_tract":
"Use fgcm_tract_photoCalib"
263 externalSkyWcsName = lsst.pex.config.ChoiceField(
265 doc=
"Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.",
268 "jointcal":
"Use jointcal_wcs"
272 def setDefaults(self):
276 super().setDefaults()
278 self.catalogCalculation.plugins.names = []
281 class ForcedPhotCcdTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
282 """A command-line driver for performing forced measurement on CCD images.
286 butler : `lsst.daf.persistence.butler.Butler`, optional
287 A Butler which will be passed to the references subtask to allow it to
288 load its schema from disk. Optional, but must be specified if
289 ``refSchema`` is not; if both are specified, ``refSchema`` takes
291 refSchema : `lsst.afw.table.Schema`, optional
292 The schema of the reference catalog, passed to the constructor of the
293 references subtask. Optional, but must be specified if ``butler`` is
294 not; if both are specified, ``refSchema`` takes precedence.
296 Keyword arguments are passed to the supertask constructor.
300 The `runDataRef` method takes a `~lsst.daf.persistence.ButlerDataRef` argument
301 that corresponds to a single CCD. This should contain the data ID keys that
302 correspond to the ``forced_src`` dataset (the output dataset for this
303 task), which are typically all those used to specify the ``calexp`` dataset
304 (``visit``, ``raft``, ``sensor`` for LSST data) as well as a coadd tract.
305 The tract is used to look up the appropriate coadd measurement catalogs to
306 use as references (e.g. ``deepCoadd_src``; see
307 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more
308 information). While the tract must be given as part of the dataRef, the
309 patches are determined automatically from the bounding box and WCS of the
310 calexp to be measured, and the filter used to fetch references is set via
311 the ``filter`` option in the configuration of
312 :lsst-task:`lsst.meas.base.references.BaseReferencesTask`).
315 ConfigClass = ForcedPhotCcdConfig
316 RunnerClass = pipeBase.ButlerInitializedTaskRunner
317 _DefaultName =
"forcedPhotCcd"
320 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
321 super().__init__(**kwds)
323 if initInputs
is not None:
324 refSchema = initInputs[
'inputSchema'].schema
326 self.makeSubtask(
"references", butler=butler, schema=refSchema)
327 if refSchema
is None:
328 refSchema = self.references.schema
329 self.makeSubtask(
"measurement", refSchema=refSchema)
332 if self.config.doApCorr:
333 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
334 self.makeSubtask(
'catalogCalculation', schema=self.measurement.schema)
337 def runQuantum(self, butlerQC, inputRefs, outputRefs):
338 inputs = butlerQC.get(inputRefs)
340 tract = butlerQC.quantum.dataId[
'tract']
341 skyMap = inputs.pop(
"skyMap")
342 inputs[
'refWcs'] = skyMap[tract].getWcs()
344 inputs[
'refCat'] = self.mergeAndFilterReferences(inputs[
'exposure'], inputs[
'refCat'],
347 inputs[
'measCat'], inputs[
'exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
349 inputs[
'refCat'], inputs[
'refWcs'],
351 self.attachFootprints(inputs[
'measCat'], inputs[
'refCat'], inputs[
'exposure'], inputs[
'refWcs'])
353 outputs = self.run(**inputs)
354 butlerQC.put(outputs, outputRefs)
356 def mergeAndFilterReferences(self, exposure, refCats, refWcs):
357 """Filter reference catalog so that all sources are within the
358 boundaries of the exposure.
362 exposure : `lsst.afw.image.exposure.Exposure`
363 Exposure to generate the catalog for.
364 refCats : sequence of `lsst.afw.table.SourceCatalog`
365 Catalogs of shapes and positions at which to force photometry.
366 refWcs : `lsst.afw.image.SkyWcs`
367 Reference world coordinate system.
371 refSources : `lsst.afw.table.SourceCatalog`
372 Filtered catalog of forced sources to measure.
376 Filtering the reference catalog is currently handled by Gen2
377 specific methods. To function for Gen3, this method copies
378 code segments to do the filtering and transformation. The
379 majority of this code is based on the methods of
380 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader
386 expWcs = exposure.getWcs()
387 expRegion = exposure.getBBox(lsst.afw.image.PARENT)
389 expBoxCorners = expBBox.getCorners()
390 expSkyCorners = [expWcs.pixelToSky(corner).getVector()
for
391 corner
in expBoxCorners]
400 for refCat
in refCats:
402 for record
in refCat:
403 if expPolygon.contains(record.getCoord().getVector())
and record.getParent()
in containedIds:
404 record.setFootprint(record.getFootprint().
transform(refWcs, expWcs, expRegion))
405 mergedRefCat.append(record)
406 containedIds.add(record.getId())
410 def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName):
411 """Generate a measurement catalog for Gen3.
415 exposureDataId : `DataId`
416 Butler dataId for this exposure.
417 exposure : `lsst.afw.image.exposure.Exposure`
418 Exposure to generate the catalog for.
419 refCat : `lsst.afw.table.SourceCatalog`
420 Catalog of shapes and positions at which to force photometry.
421 refWcs : `lsst.afw.image.SkyWcs`
422 Reference world coordinate system.
424 Type of ID packer to construct from the registry.
428 measCat : `lsst.afw.table.SourceCatalog`
429 Catalog of forced sources to measure.
431 Unique binary id associated with the input exposure
433 expId, expBits = exposureDataId.pack(idPackerName, returnMaxBits=
True)
436 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
438 return measCat, expId
440 def runDataRef(self, dataRef, psfCache=None):
441 """Perform forced measurement on a single exposure.
445 dataRef : `lsst.daf.persistence.ButlerDataRef`
446 Passed to the ``references`` subtask to obtain the reference WCS,
447 the ``getExposure`` method (implemented by derived classes) to
448 read the measurment image, and the ``fetchReferences`` method to
449 get the exposure and load the reference catalog (see
450 :lsst-task`lsst.meas.base.references.CoaddSrcReferencesTask`).
451 Refer to derived class documentation for details of the datasets
452 and data ID keys which are used.
453 psfCache : `int`, optional
454 Size of PSF cache, or `None`. The size of the PSF cache can have
455 a significant effect upon the runtime for complicated PSF models.
459 Sources are generated with ``generateMeasCat`` in the ``measurement``
460 subtask. These are passed to ``measurement``'s ``run`` method, which
461 fills the source catalog with the forced measurement results. The
462 sources are then passed to the ``writeOutputs`` method (implemented by
463 derived classes) which writes the outputs.
465 refWcs = self.references.getWcs(dataRef)
466 exposure = self.getExposure(dataRef)
467 if psfCache
is not None:
468 exposure.getPsf().setCacheSize(psfCache)
469 refCat = self.fetchReferences(dataRef, exposure)
471 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
472 idFactory=self.makeIdFactory(dataRef))
473 self.log.info(
"Performing forced measurement on %s" % (dataRef.dataId,))
474 self.attachFootprints(measCat, refCat, exposure, refWcs)
476 exposureId = self.getExposureId(dataRef)
478 forcedPhotResult = self.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
480 self.writeOutput(dataRef, forcedPhotResult.measCat)
482 def run(self, measCat, exposure, refCat, refWcs, exposureId=None):
483 """Perform forced measurement on a single exposure.
487 measCat : `lsst.afw.table.SourceCatalog`
488 The measurement catalog, based on the sources listed in the
490 exposure : `lsst.afw.image.Exposure`
491 The measurement image upon which to perform forced detection.
492 refCat : `lsst.afw.table.SourceCatalog`
493 The reference catalog of sources to measure.
494 refWcs : `lsst.afw.image.SkyWcs`
495 The WCS for the references.
497 Optional unique exposureId used for random seed in measurement
502 result : `lsst.pipe.base.Struct`
503 Structure with fields:
506 Catalog of forced measurement results
507 (`lsst.afw.table.SourceCatalog`).
509 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
510 if self.config.doApCorr:
511 self.applyApCorr.run(
513 apCorrMap=exposure.getInfo().getApCorrMap()
515 self.catalogCalculation.run(measCat)
517 return pipeBase.Struct(measCat=measCat)
519 def makeIdFactory(self, dataRef):
520 """Create an object that generates globally unique source IDs.
522 Source IDs are created based on a per-CCD ID and the ID of the CCD
527 dataRef : `lsst.daf.persistence.ButlerDataRef`
528 Butler data reference. The ``ccdExposureId_bits`` and
529 ``ccdExposureId`` datasets are accessed. The data ID must have the
530 keys that correspond to ``ccdExposureId``, which are generally the
531 same as those that correspond to ``calexp`` (``visit``, ``raft``,
532 ``sensor`` for LSST data).
534 expBits = dataRef.get(
"ccdExposureId_bits")
535 expId = int(dataRef.get(
"ccdExposureId"))
539 return int(dataRef.get(
"ccdExposureId", immediate=
True))
542 """Get sources that overlap the exposure.
546 dataRef : `lsst.daf.persistence.ButlerDataRef`
547 Butler data reference corresponding to the image to be measured;
548 should have ``tract``, ``patch``, and ``filter`` keys.
549 exposure : `lsst.afw.image.Exposure`
550 The image to be measured (used only to obtain a WCS and bounding
555 referencs : `lsst.afw.table.SourceCatalog`
556 Catalog of sources that overlap the exposure
560 The returned catalog is sorted by ID and guarantees that all included
561 children have their parent included and that all Footprints are valid.
563 All work is delegated to the references subtask; see
564 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask`
565 for information about the default behavior.
569 unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs())
570 for record
in unfiltered:
571 if record.getFootprint()
is None or record.getFootprint().getArea() == 0:
572 if record.getParent() != 0:
573 self.log.warn(
"Skipping reference %s (child of %s) with bad Footprint",
574 record.getId(), record.getParent())
576 self.log.warn(
"Skipping reference parent %s with bad Footprint", record.getId())
577 badParents.add(record.getId())
578 elif record.getParent()
not in badParents:
579 references.append(record)
585 r"""Attach footprints to blank sources prior to measurements.
589 `~lsst.afw.detection.Footprint`\ s for forced photometry must be in the
590 pixel coordinate system of the image being measured, while the actual
591 detections may start out in a different coordinate system.
593 Subclasses of this class must implement this method to define how
594 those `~lsst.afw.detection.Footprint`\ s should be generated.
596 This default implementation transforms the
597 `~lsst.afw.detection.Footprint`\ s from the reference catalog from the
598 reference WCS to the exposure's WcS, which downgrades
599 `lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s into regular
600 `~lsst.afw.detection.Footprint`\ s, destroying deblend information.
602 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs)
605 """Read input exposure for measurement.
609 dataRef : `lsst.daf.persistence.ButlerDataRef`
610 Butler data reference.
612 exposure = dataRef.get(self.dataPrefix +
"calexp", immediate=
True)
614 if self.config.doApplyExternalPhotoCalib:
615 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
616 self.log.info(
"Applying external photoCalib from %s", source)
617 photoCalib = dataRef.get(source)
618 exposure.setPhotoCalib(photoCalib)
620 if self.config.doApplyExternalSkyWcs:
621 source = f
"{self.config.externalSkyWcsName}_wcs"
622 self.log.info(
"Applying external skyWcs from %s", source)
623 skyWcs = dataRef.get(source)
624 exposure.setWcs(skyWcs)
626 if self.config.doApplySkyCorr:
627 self.log.info(
"Apply sky correction")
628 skyCorr = dataRef.get(
"skyCorr")
629 exposure.maskedImage -= skyCorr.getImage()
634 """Write forced source table
638 dataRef : `lsst.daf.persistence.ButlerDataRef`
639 Butler data reference. The forced_src dataset (with
640 self.dataPrefix prepended) is all that will be modified.
641 sources : `lsst.afw.table.SourceCatalog`
642 Catalog of sources to save.
644 dataRef.put(sources, self.dataPrefix +
"forced_src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
647 """The schema catalogs that will be used by this task.
651 schemaCatalogs : `dict`
652 Dictionary mapping dataset type to schema catalog.
656 There is only one schema for each type of forced measurement. The
657 dataset type for this measurement is defined in the mapper.
660 catalog.getTable().setMetadata(self.measurement.algMetadata)
661 datasetType = self.dataPrefix +
"forced_src"
662 return {datasetType: catalog}
664 def _getConfigName(self):
666 return self.dataPrefix +
"forcedPhotCcd_config"
668 def _getMetadataName(self):
670 return self.dataPrefix +
"forcedPhotCcd_metadata"
673 def _makeArgumentParser(cls):
674 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
675 parser.add_id_argument(
"--id",
"forced_src", help=
"data ID with raw CCD keys [+ tract optionally], "
676 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
677 ContainerClass=PerTractCcdDataIdContainer)
static std::shared_ptr< IdFactory > makeSource(RecordId expId, int reserved)
static Key< RecordId > getParentKey()
def makeDataRefList(self, namespace)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)
std::shared_ptr< SkyWcs > makeSkyWcs(daf::base::PropertySet &metadata, bool strip=false)
lsst::geom::Box2I bboxFromMetadata(daf::base::PropertySet &metadata)
def imageOverlapsTract(tract, imageWcs, imageBox)
def attachFootprints(self, sources, refCat, exposure, refWcs, dataRef)
def writeOutput(self, dataRef, sources)
def fetchReferences(self, dataRef, exposure)
def getExposure(self, dataRef)
def getSchemaCatalogs(self)
def getExposureId(self, dataRef)