Coverage for python/lsst/meas/base/forcedPhotCcd.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import collections
23import logging
24import pandas as pd
25import numpy as np
27import lsst.pex.config
28import lsst.pex.exceptions
29import lsst.pipe.base
30import lsst.geom
31import lsst.afw.detection
32import lsst.afw.geom
33import lsst.afw.image
34import lsst.afw.table
35import lsst.sphgeom
37from lsst.obs.base import ExposureIdInfo
38from lsst.pipe.base import PipelineTaskConnections
39import lsst.pipe.base.connectionTypes as cT
41import lsst.pipe.base as pipeBase
42from lsst.skymap import BaseSkyMap
44from .references import MultiBandReferencesTask
45from .forcedMeasurement import ForcedMeasurementTask
46from .applyApCorr import ApplyApCorrTask
47from .catalogCalculation import CatalogCalculationTask
49try:
50 from lsst.meas.mosaic import applyMosaicResults
51except ImportError:
52 applyMosaicResults = None
54__all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask", "imageOverlapsTract",
55 "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig")
58class PerTractCcdDataIdContainer(pipeBase.DataIdContainer):
59 """A data ID container which combines raw data IDs with a tract.
61 Notes
62 -----
63 Required because we need to add "tract" to the raw data ID keys (defined as
64 whatever we use for ``src``) when no tract is provided (so that the user is
65 not required to know which tracts are spanned by the raw data ID).
67 This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is
68 being measured using the detection information, a set of reference
69 catalogs, from the set of coadds which intersect with the calexp. It needs
70 the calexp id (e.g. visit, raft, sensor), but is also uses the tract to
71 decide what set of coadds to use. The references from the tract whose
72 patches intersect with the calexp are used.
73 """
75 def makeDataRefList(self, namespace):
76 """Make self.refList from self.idList
77 """
78 if self.datasetType is None:
79 raise RuntimeError("Must call setDatasetType first")
80 log = logging.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer")
81 skymap = None
82 visitTract = collections.defaultdict(set) # Set of tracts for each visit
83 visitRefs = collections.defaultdict(list) # List of data references for each visit
84 for dataId in self.idList:
85 if "tract" not in dataId:
86 # Discover which tracts the data overlaps
87 log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId))
88 if skymap is None:
89 skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap")
91 for ref in namespace.butler.subset("calexp", dataId=dataId):
92 if not ref.datasetExists("calexp"):
93 continue
95 visit = ref.dataId["visit"]
96 visitRefs[visit].append(ref)
98 md = ref.get("calexp_md", immediate=True)
99 wcs = lsst.afw.geom.makeSkyWcs(md)
100 box = lsst.geom.Box2D(lsst.afw.image.bboxFromMetadata(md))
101 # Going with just the nearest tract. Since we're throwing all tracts for the visit
102 # together, this shouldn't be a problem unless the tracts are much smaller than a CCD.
103 tract = skymap.findTract(wcs.pixelToSky(box.getCenter()))
104 if imageOverlapsTract(tract, wcs, box):
105 visitTract[visit].add(tract.getId())
106 else:
107 self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId))
109 # Ensure all components of a visit are kept together by putting them all in the same set of tracts
110 for visit, tractSet in visitTract.items():
111 for ref in visitRefs[visit]:
112 for tract in tractSet:
113 self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType,
114 dataId=ref.dataId, tract=tract))
115 if visitTract:
116 tractCounter = collections.Counter()
117 for tractSet in visitTract.values():
118 tractCounter.update(tractSet)
119 log.info("Number of visits for each tract: %s", dict(tractCounter))
122def imageOverlapsTract(tract, imageWcs, imageBox):
123 """Return whether the given bounding box overlaps the tract given a WCS.
125 Parameters
126 ----------
127 tract : `lsst.skymap.TractInfo`
128 TractInfo specifying a tract.
129 imageWcs : `lsst.afw.geom.SkyWcs`
130 World coordinate system for the image.
131 imageBox : `lsst.geom.Box2I`
132 Bounding box for the image.
134 Returns
135 -------
136 overlap : `bool`
137 `True` if the bounding box overlaps the tract; `False` otherwise.
138 """
139 tractPoly = tract.getOuterSkyPolygon()
141 imagePixelCorners = lsst.geom.Box2D(imageBox).getCorners()
142 try:
143 imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners)
144 except lsst.pex.exceptions.LsstCppException as e:
145 # Protecting ourselves from awful Wcs solutions in input images
146 if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException)
147 and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)):
148 raise
149 return False
151 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageSkyCorners])
152 return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by"
155class ForcedPhotCcdConnections(PipelineTaskConnections,
156 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
157 defaultTemplates={"inputCoaddName": "deep",
158 "inputName": "calexp"}):
159 inputSchema = cT.InitInput(
160 doc="Schema for the input measurement catalogs.",
161 name="{inputCoaddName}Coadd_ref_schema",
162 storageClass="SourceCatalog",
163 )
164 outputSchema = cT.InitOutput(
165 doc="Schema for the output forced measurement catalogs.",
166 name="forced_src_schema",
167 storageClass="SourceCatalog",
168 )
169 exposure = cT.Input(
170 doc="Input exposure to perform photometry on.",
171 name="{inputName}",
172 storageClass="ExposureF",
173 dimensions=["instrument", "visit", "detector"],
174 )
175 refCat = cT.Input(
176 doc="Catalog of shapes and positions at which to force photometry.",
177 name="{inputCoaddName}Coadd_ref",
178 storageClass="SourceCatalog",
179 dimensions=["skymap", "tract", "patch"],
180 multiple=True,
181 deferLoad=True,
182 )
183 skyMap = cT.Input(
184 doc="SkyMap dataset that defines the coordinate system of the reference catalog.",
185 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
186 storageClass="SkyMap",
187 dimensions=["skymap"],
188 )
189 measCat = cT.Output(
190 doc="Output forced photometry catalog.",
191 name="forced_src",
192 storageClass="SourceCatalog",
193 dimensions=["instrument", "visit", "detector", "skymap", "tract"],
194 )
197class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig,
198 pipelineConnections=ForcedPhotCcdConnections):
199 """Config class for forced measurement driver task."""
200 references = lsst.pex.config.ConfigurableField(
201 target=MultiBandReferencesTask,
202 doc="subtask to retrieve reference source catalog"
203 )
204 measurement = lsst.pex.config.ConfigurableField(
205 target=ForcedMeasurementTask,
206 doc="subtask to do forced measurement"
207 )
208 coaddName = lsst.pex.config.Field(
209 doc="coadd name: typically one of deep or goodSeeing",
210 dtype=str,
211 default="deep",
212 )
213 doApCorr = lsst.pex.config.Field(
214 dtype=bool,
215 default=True,
216 doc="Run subtask to apply aperture corrections"
217 )
218 applyApCorr = lsst.pex.config.ConfigurableField(
219 target=ApplyApCorrTask,
220 doc="Subtask to apply aperture corrections"
221 )
222 catalogCalculation = lsst.pex.config.ConfigurableField(
223 target=CatalogCalculationTask,
224 doc="Subtask to run catalogCalculation plugins on catalog"
225 )
226 doApplyUberCal = lsst.pex.config.Field(
227 dtype=bool,
228 doc="Apply meas_mosaic ubercal results to input calexps?",
229 default=False,
230 deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead",
231 )
232 doApplyExternalPhotoCalib = lsst.pex.config.Field(
233 dtype=bool,
234 default=False,
235 doc=("Whether to apply external photometric calibration via an "
236 "`lsst.afw.image.PhotoCalib` object. Uses the "
237 "``externalPhotoCalibName`` field to determine which calibration "
238 "to load."),
239 )
240 doApplyExternalSkyWcs = lsst.pex.config.Field(
241 dtype=bool,
242 default=False,
243 doc=("Whether to apply external astrometric calibration via an "
244 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` "
245 "field to determine which calibration to load."),
246 )
247 doApplySkyCorr = lsst.pex.config.Field(
248 dtype=bool,
249 default=False,
250 doc="Apply sky correction?",
251 )
252 includePhotoCalibVar = lsst.pex.config.Field(
253 dtype=bool,
254 default=False,
255 doc="Add photometric calibration variance to warp variance plane?",
256 )
257 externalPhotoCalibName = lsst.pex.config.ChoiceField(
258 dtype=str,
259 doc=("Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. "
260 "Unused for Gen3 middleware."),
261 default="jointcal",
262 allowed={
263 "jointcal": "Use jointcal_photoCalib",
264 "fgcm": "Use fgcm_photoCalib",
265 "fgcm_tract": "Use fgcm_tract_photoCalib"
266 },
267 )
268 externalSkyWcsName = lsst.pex.config.ChoiceField(
269 dtype=str,
270 doc="Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.",
271 default="jointcal",
272 allowed={
273 "jointcal": "Use jointcal_wcs"
274 },
275 )
276 footprintSource = lsst.pex.config.ChoiceField(
277 dtype=str,
278 doc="Where to obtain footprints to install in the measurement catalog, prior to measurement.",
279 allowed={
280 "transformed": "Transform footprints from the reference catalog (downgrades HeavyFootprints).",
281 "psf": ("Use the scaled shape of the PSF at the position of each source (does not generate "
282 "HeavyFootprints)."),
283 },
284 optional=True,
285 default="transformed",
286 )
287 psfFootprintScaling = lsst.pex.config.Field(
288 dtype=float,
289 doc="Scaling factor to apply to the PSF shape when footprintSource='psf' (ignored otherwise).",
290 default=3.0,
291 )
293 def setDefaults(self):
294 # Docstring inherited.
295 # Make catalogCalculation a no-op by default as no modelFlux is setup by default in
296 # ForcedMeasurementTask
297 super().setDefaults()
298 self.measurement.plugins.names |= ['base_LocalPhotoCalib', 'base_LocalWcs']
299 self.catalogCalculation.plugins.names = []
302class ForcedPhotCcdTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
303 """A command-line driver for performing forced measurement on CCD images.
305 Parameters
306 ----------
307 butler : `lsst.daf.persistence.butler.Butler`, optional
308 A Butler which will be passed to the references subtask to allow it to
309 load its schema from disk. Optional, but must be specified if
310 ``refSchema`` is not; if both are specified, ``refSchema`` takes
311 precedence.
312 refSchema : `lsst.afw.table.Schema`, optional
313 The schema of the reference catalog, passed to the constructor of the
314 references subtask. Optional, but must be specified if ``butler`` is
315 not; if both are specified, ``refSchema`` takes precedence.
316 **kwds
317 Keyword arguments are passed to the supertask constructor.
319 Notes
320 -----
321 The `runDataRef` method takes a `~lsst.daf.persistence.ButlerDataRef` argument
322 that corresponds to a single CCD. This should contain the data ID keys that
323 correspond to the ``forced_src`` dataset (the output dataset for this
324 task), which are typically all those used to specify the ``calexp`` dataset
325 (``visit``, ``raft``, ``sensor`` for LSST data) as well as a coadd tract.
326 The tract is used to look up the appropriate coadd measurement catalogs to
327 use as references (e.g. ``deepCoadd_src``; see
328 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more
329 information). While the tract must be given as part of the dataRef, the
330 patches are determined automatically from the bounding box and WCS of the
331 calexp to be measured, and the filter used to fetch references is set via
332 the ``filter`` option in the configuration of
333 :lsst-task:`lsst.meas.base.references.BaseReferencesTask`).
334 """
336 ConfigClass = ForcedPhotCcdConfig
337 RunnerClass = pipeBase.ButlerInitializedTaskRunner
338 _DefaultName = "forcedPhotCcd"
339 dataPrefix = ""
341 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
342 super().__init__(**kwds)
344 if initInputs is not None:
345 refSchema = initInputs['inputSchema'].schema
347 self.makeSubtask("references", butler=butler, schema=refSchema)
348 if refSchema is None:
349 refSchema = self.references.schema
350 self.makeSubtask("measurement", refSchema=refSchema)
351 # It is necessary to get the schema internal to the forced measurement task until such a time
352 # that the schema is not owned by the measurement task, but is passed in by an external caller
353 if self.config.doApCorr:
354 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
355 self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
356 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
358 def runQuantum(self, butlerQC, inputRefs, outputRefs):
359 inputs = butlerQC.get(inputRefs)
361 tract = butlerQC.quantum.dataId['tract']
362 skyMap = inputs.pop("skyMap")
363 inputs['refWcs'] = skyMap[tract].getWcs()
365 inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'],
366 inputs['refWcs'])
368 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
369 inputs['exposure'],
370 inputs['refCat'], inputs['refWcs'],
371 "visit_detector")
372 self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs'])
373 # TODO: apply external calibrations (DM-17062)
374 outputs = self.run(**inputs)
375 butlerQC.put(outputs, outputRefs)
377 def mergeAndFilterReferences(self, exposure, refCats, refWcs):
378 """Filter reference catalog so that all sources are within the
379 boundaries of the exposure.
381 Parameters
382 ----------
383 exposure : `lsst.afw.image.exposure.Exposure`
384 Exposure to generate the catalog for.
385 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle`
386 Handles for catalogs of shapes and positions at which to force
387 photometry.
388 refWcs : `lsst.afw.image.SkyWcs`
389 Reference world coordinate system.
391 Returns
392 -------
393 refSources : `lsst.afw.table.SourceCatalog`
394 Filtered catalog of forced sources to measure.
396 Notes
397 -----
398 Filtering the reference catalog is currently handled by Gen2
399 specific methods. To function for Gen3, this method copies
400 code segments to do the filtering and transformation. The
401 majority of this code is based on the methods of
402 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader
404 """
406 # Step 1: Determine bounds of the exposure photometry will
407 # be performed on.
408 expWcs = exposure.getWcs()
409 expRegion = exposure.getBBox(lsst.afw.image.PARENT)
410 expBBox = lsst.geom.Box2D(expRegion)
411 expBoxCorners = expBBox.getCorners()
412 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for
413 corner in expBoxCorners]
414 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners)
416 # Step 2: Filter out reference catalog sources that are
417 # not contained within the exposure boundaries, or whose
418 # parents are not within the exposure boundaries. Note
419 # that within a single input refCat, the parents always
420 # appear before the children.
421 mergedRefCat = None
422 for refCat in refCats:
423 refCat = refCat.get()
424 if mergedRefCat is None:
425 mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table)
426 containedIds = {0} # zero as a parent ID means "this is a parent"
427 for record in refCat:
428 if expPolygon.contains(record.getCoord().getVector()) and record.getParent() in containedIds:
429 record.setFootprint(record.getFootprint())
430 mergedRefCat.append(record)
431 containedIds.add(record.getId())
432 if mergedRefCat is None:
433 raise RuntimeError("No reference objects for forced photometry.")
434 mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey())
435 return mergedRefCat
437 def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName):
438 """Generate a measurement catalog for Gen3.
440 Parameters
441 ----------
442 exposureDataId : `DataId`
443 Butler dataId for this exposure.
444 exposure : `lsst.afw.image.exposure.Exposure`
445 Exposure to generate the catalog for.
446 refCat : `lsst.afw.table.SourceCatalog`
447 Catalog of shapes and positions at which to force photometry.
448 refWcs : `lsst.afw.image.SkyWcs`
449 Reference world coordinate system.
450 idPackerName : `str`
451 Type of ID packer to construct from the registry.
453 Returns
454 -------
455 measCat : `lsst.afw.table.SourceCatalog`
456 Catalog of forced sources to measure.
457 expId : `int`
458 Unique binary id associated with the input exposure
459 """
460 exposureIdInfo = ExposureIdInfo.fromDataId(exposureDataId, idPackerName)
461 idFactory = exposureIdInfo.makeSourceIdFactory()
463 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
464 idFactory=idFactory)
465 return measCat, exposureIdInfo.expId
467 def runDataRef(self, dataRef, psfCache=None):
468 """Perform forced measurement on a single exposure.
470 Parameters
471 ----------
472 dataRef : `lsst.daf.persistence.ButlerDataRef`
473 Passed to the ``references`` subtask to obtain the reference WCS,
474 the ``getExposure`` method (implemented by derived classes) to
475 read the measurment image, and the ``fetchReferences`` method to
476 get the exposure and load the reference catalog (see
477 :lsst-task`lsst.meas.base.references.CoaddSrcReferencesTask`).
478 Refer to derived class documentation for details of the datasets
479 and data ID keys which are used.
480 psfCache : `int`, optional
481 Size of PSF cache, or `None`. The size of the PSF cache can have
482 a significant effect upon the runtime for complicated PSF models.
484 Notes
485 -----
486 Sources are generated with ``generateMeasCat`` in the ``measurement``
487 subtask. These are passed to ``measurement``'s ``run`` method, which
488 fills the source catalog with the forced measurement results. The
489 sources are then passed to the ``writeOutputs`` method (implemented by
490 derived classes) which writes the outputs.
491 """
492 refWcs = self.references.getWcs(dataRef)
493 exposure = self.getExposure(dataRef)
494 if psfCache is not None:
495 exposure.getPsf().setCacheSize(psfCache)
496 refCat = self.fetchReferences(dataRef, exposure)
497 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
498 idFactory=self.makeIdFactory(dataRef))
499 self.log.info("Performing forced measurement on %s", dataRef.dataId)
500 self.attachFootprints(measCat, refCat, exposure, refWcs)
502 exposureId = self.getExposureId(dataRef)
504 forcedPhotResult = self.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
506 self.writeOutput(dataRef, forcedPhotResult.measCat)
508 def run(self, measCat, exposure, refCat, refWcs, exposureId=None):
509 """Perform forced measurement on a single exposure.
511 Parameters
512 ----------
513 measCat : `lsst.afw.table.SourceCatalog`
514 The measurement catalog, based on the sources listed in the
515 reference catalog.
516 exposure : `lsst.afw.image.Exposure`
517 The measurement image upon which to perform forced detection.
518 refCat : `lsst.afw.table.SourceCatalog`
519 The reference catalog of sources to measure.
520 refWcs : `lsst.afw.image.SkyWcs`
521 The WCS for the references.
522 exposureId : `int`
523 Optional unique exposureId used for random seed in measurement
524 task.
526 Returns
527 -------
528 result : `lsst.pipe.base.Struct`
529 Structure with fields:
531 ``measCat``
532 Catalog of forced measurement results
533 (`lsst.afw.table.SourceCatalog`).
534 """
535 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
536 if self.config.doApCorr:
537 self.applyApCorr.run(
538 catalog=measCat,
539 apCorrMap=exposure.getInfo().getApCorrMap()
540 )
541 self.catalogCalculation.run(measCat)
543 return pipeBase.Struct(measCat=measCat)
545 def makeIdFactory(self, dataRef):
546 """Create an object that generates globally unique source IDs.
548 Source IDs are created based on a per-CCD ID and the ID of the CCD
549 itself.
551 Parameters
552 ----------
553 dataRef : `lsst.daf.persistence.ButlerDataRef`
554 Butler data reference. The ``ccdExposureId_bits`` and
555 ``ccdExposureId`` datasets are accessed. The data ID must have the
556 keys that correspond to ``ccdExposureId``, which are generally the
557 same as those that correspond to ``calexp`` (``visit``, ``raft``,
558 ``sensor`` for LSST data).
559 """
560 exposureIdInfo = ExposureIdInfo(int(dataRef.get("ccdExposureId")), dataRef.get("ccdExposureId_bits"))
561 return exposureIdInfo.makeSourceIdFactory()
563 def getExposureId(self, dataRef):
564 return int(dataRef.get("ccdExposureId", immediate=True))
566 def fetchReferences(self, dataRef, exposure):
567 """Get sources that overlap the exposure.
569 Parameters
570 ----------
571 dataRef : `lsst.daf.persistence.ButlerDataRef`
572 Butler data reference corresponding to the image to be measured;
573 should have ``tract``, ``patch``, and ``filter`` keys.
574 exposure : `lsst.afw.image.Exposure`
575 The image to be measured (used only to obtain a WCS and bounding
576 box).
578 Returns
579 -------
580 referencs : `lsst.afw.table.SourceCatalog`
581 Catalog of sources that overlap the exposure
583 Notes
584 -----
585 The returned catalog is sorted by ID and guarantees that all included
586 children have their parent included and that all Footprints are valid.
588 All work is delegated to the references subtask; see
589 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask`
590 for information about the default behavior.
591 """
592 references = lsst.afw.table.SourceCatalog(self.references.schema)
593 badParents = set()
594 unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs())
595 for record in unfiltered:
596 if record.getFootprint() is None or record.getFootprint().getArea() == 0:
597 if record.getParent() != 0:
598 self.log.warning("Skipping reference %s (child of %s) with bad Footprint",
599 record.getId(), record.getParent())
600 else:
601 self.log.warning("Skipping reference parent %s with bad Footprint", record.getId())
602 badParents.add(record.getId())
603 elif record.getParent() not in badParents:
604 references.append(record)
605 # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work
606 references.sort(lsst.afw.table.SourceTable.getParentKey())
607 return references
609 def attachFootprints(self, sources, refCat, exposure, refWcs):
610 r"""Attach footprints to blank sources prior to measurements.
612 Notes
613 -----
614 `~lsst.afw.detection.Footprint`\ s for forced photometry must be in the
615 pixel coordinate system of the image being measured, while the actual
616 detections may start out in a different coordinate system.
618 Subclasses of this class may implement this method to define how
619 those `~lsst.afw.detection.Footprint`\ s should be generated.
621 This default implementation transforms depends on the
622 ``footprintSource`` configuration parameter.
623 """
624 if self.config.footprintSource == "transformed":
625 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs)
626 elif self.config.footprintSource == "psf":
627 return self.measurement.attachPsfShapeFootprints(sources, exposure,
628 scaling=self.config.psfFootprintScaling)
630 def getExposure(self, dataRef):
631 """Read input exposure for measurement.
633 Parameters
634 ----------
635 dataRef : `lsst.daf.persistence.ButlerDataRef`
636 Butler data reference.
637 """
638 exposure = dataRef.get(self.dataPrefix + "calexp", immediate=True)
640 if self.config.doApplyExternalPhotoCalib:
641 source = f"{self.config.externalPhotoCalibName}_photoCalib"
642 self.log.info("Applying external photoCalib from %s", source)
643 photoCalib = dataRef.get(source)
644 exposure.setPhotoCalib(photoCalib) # No need for calibrateImage; having the photoCalib suffices
646 if self.config.doApplyExternalSkyWcs:
647 source = f"{self.config.externalSkyWcsName}_wcs"
648 self.log.info("Applying external skyWcs from %s", source)
649 skyWcs = dataRef.get(source)
650 exposure.setWcs(skyWcs)
652 if self.config.doApplySkyCorr:
653 self.log.info("Apply sky correction")
654 skyCorr = dataRef.get("skyCorr")
655 exposure.maskedImage -= skyCorr.getImage()
657 return exposure
659 def writeOutput(self, dataRef, sources):
660 """Write forced source table
662 Parameters
663 ----------
664 dataRef : `lsst.daf.persistence.ButlerDataRef`
665 Butler data reference. The forced_src dataset (with
666 self.dataPrefix prepended) is all that will be modified.
667 sources : `lsst.afw.table.SourceCatalog`
668 Catalog of sources to save.
669 """
670 dataRef.put(sources, self.dataPrefix + "forced_src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
672 def getSchemaCatalogs(self):
673 """The schema catalogs that will be used by this task.
675 Returns
676 -------
677 schemaCatalogs : `dict`
678 Dictionary mapping dataset type to schema catalog.
680 Notes
681 -----
682 There is only one schema for each type of forced measurement. The
683 dataset type for this measurement is defined in the mapper.
684 """
685 catalog = lsst.afw.table.SourceCatalog(self.measurement.schema)
686 catalog.getTable().setMetadata(self.measurement.algMetadata)
687 datasetType = self.dataPrefix + "forced_src"
688 return {datasetType: catalog}
690 def _getConfigName(self):
691 # Documented in superclass.
692 return self.dataPrefix + "forcedPhotCcd_config"
694 def _getMetadataName(self):
695 # Documented in superclass
696 return self.dataPrefix + "forcedPhotCcd_metadata"
698 @classmethod
699 def _makeArgumentParser(cls):
700 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
701 parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], "
702 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
703 ContainerClass=PerTractCcdDataIdContainer)
704 return parser
707class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections,
708 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
709 defaultTemplates={"inputCoaddName": "goodSeeing",
710 "inputName": "calexp"}):
711 refCat = cT.Input(
712 doc="Catalog of positions at which to force photometry.",
713 name="{inputCoaddName}Diff_fullDiaObjTable",
714 storageClass="DataFrame",
715 dimensions=["skymap", "tract", "patch"],
716 multiple=True,
717 deferLoad=True,
718 )
719 exposure = cT.Input(
720 doc="Input exposure to perform photometry on.",
721 name="{inputName}",
722 storageClass="ExposureF",
723 dimensions=["instrument", "visit", "detector"],
724 )
725 measCat = cT.Output(
726 doc="Output forced photometry catalog.",
727 name="forced_src_diaObject",
728 storageClass="SourceCatalog",
729 dimensions=["instrument", "visit", "detector", "skymap", "tract"],
730 )
731 outputSchema = cT.InitOutput(
732 doc="Schema for the output forced measurement catalogs.",
733 name="forced_src_diaObject_schema",
734 storageClass="SourceCatalog",
735 )
738class ForcedPhotCcdFromDataFrameConfig(ForcedPhotCcdConfig,
739 pipelineConnections=ForcedPhotCcdFromDataFrameConnections):
740 def setDefaults(self):
741 super().setDefaults()
742 self.footprintSource = "psf"
743 self.measurement.doReplaceWithNoise = False
744 self.measurement.plugins = ["base_TransformedCentroidFromCoord", "base_PsfFlux", "base_PixelFlags"]
745 self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'}
746 self.measurement.slots.centroid = "base_TransformedCentroidFromCoord"
747 self.measurement.slots.psfFlux = "base_PsfFlux"
748 self.measurement.slots.shape = None
750 def validate(self):
751 super().validate()
752 if self.footprintSource == "transformed":
753 raise ValueError("Cannot transform footprints from reference catalog, "
754 "because DataFrames can't hold footprints.")
757class ForcedPhotCcdFromDataFrameTask(ForcedPhotCcdTask):
758 """Force Photometry on a per-detector exposure with coords from a DataFrame
760 Uses input from a DataFrame instead of SourceCatalog
761 like the base class ForcedPhotCcd does.
762 Writes out a SourceCatalog so that the downstream
763 WriteForcedSourceTableTask can be reused with output from this Task.
764 """
765 _DefaultName = "forcedPhotCcdFromDataFrame"
766 ConfigClass = ForcedPhotCcdFromDataFrameConfig
768 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
769 # Parent's init assumes that we have a reference schema; Cannot reuse
770 pipeBase.PipelineTask.__init__(self, **kwds)
772 self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema())
774 if self.config.doApCorr:
775 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
776 self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
777 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
779 def runQuantum(self, butlerQC, inputRefs, outputRefs):
780 inputs = butlerQC.get(inputRefs)
781 self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']]))
782 refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'decl']})
783 for i in inputs['refCat']],
784 inputs['exposure'].getBBox(), inputs['exposure'].getWcs())
785 inputs['refCat'] = refCat
786 inputs['refWcs'] = inputs['exposure'].getWcs()
787 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
788 inputs['exposure'], inputs['refCat'],
789 inputs['refWcs'],
790 "visit_detector")
791 self.attachFootprints(inputs["measCat"], inputs["refCat"], inputs["exposure"], inputs["refWcs"])
792 outputs = self.run(**inputs)
793 butlerQC.put(outputs, outputRefs)
795 def df2RefCat(self, dfList, exposureBBox, exposureWcs):
796 """Convert list of DataFrames to reference catalog
798 Concatenate list of DataFrames presumably from multiple patches and
799 downselect rows that overlap the exposureBBox using the exposureWcs.
801 Parameters
802 ----------
803 dfList : `list` of `pandas.DataFrame`
804 Each element containst diaObjects with ra/decl position in degrees
805 Columns 'diaObjectId', 'ra', 'decl' are expected
806 exposureBBox : `lsst.geom.Box2I`
807 Bounding box on which to select rows that overlap
808 exposureWcs : `lsst.afw.geom.SkyWcs`
809 World coordinate system to convert sky coords in ref cat to
810 pixel coords with which to compare with exposureBBox
812 Returns
813 -------
814 refCat : `lsst.afw.table.SourceTable`
815 Source Catalog with minimal schema that overlaps exposureBBox
816 """
817 df = pd.concat(dfList)
818 # translate ra/decl coords in dataframe to detector pixel coords
819 # to down select rows that overlap the detector bbox
820 mapping = exposureWcs.getTransform().getMapping()
821 x, y = mapping.applyInverse(np.array(df[['ra', 'decl']].values*2*np.pi/360).T)
822 inBBox = lsst.geom.Box2D(exposureBBox).contains(x, y)
823 refCat = self.df2SourceCat(df[inBBox])
824 return refCat
826 def df2SourceCat(self, df):
827 """Create minimal schema SourceCatalog from a pandas DataFrame.
829 The forced measurement subtask expects this as input.
831 Parameters
832 ----------
833 df : `pandas.DataFrame`
834 DiaObjects with locations and ids.
836 Returns
837 -------
838 outputCatalog : `lsst.afw.table.SourceTable`
839 Output catalog with minimal schema.
840 """
841 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
842 outputCatalog = lsst.afw.table.SourceCatalog(schema)
843 outputCatalog.reserve(len(df))
845 for diaObjectId, ra, decl in df[['ra', 'decl']].itertuples():
846 outputRecord = outputCatalog.addNew()
847 outputRecord.setId(diaObjectId)
848 outputRecord.setCoord(lsst.geom.SpherePoint(ra, decl, lsst.geom.degrees))
849 return outputCatalog