lsst.meas.base g61ffd41259+4f4d0d1589
Loading...
Searching...
No Matches
forcedPhotCcd.py
Go to the documentation of this file.
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import warnings
23
24import pandas as pd
25import numpy as np
26
27import lsst.pex.config
29import lsst.pipe.base
30import lsst.geom
32import lsst.afw.geom
33import lsst.afw.image
34import lsst.afw.table
35import lsst.sphgeom
36
37from lsst.utils.introspection import find_outside_stacklevel
38from lsst.pipe.base import PipelineTaskConnections
39import lsst.pipe.base.connectionTypes as cT
40
41import lsst.pipe.base as pipeBase
42from lsst.skymap import BaseSkyMap
43
44from .forcedMeasurement import ForcedMeasurementTask
45from .applyApCorr import ApplyApCorrTask
46from .catalogCalculation import CatalogCalculationTask
47from ._id_generator import DetectorVisitIdGeneratorConfig
48
49__all__ = ("ForcedPhotCcdConfig", "ForcedPhotCcdTask",
50 "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig")
51
52
53class ForcedPhotCcdConnections(PipelineTaskConnections,
54 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
55 defaultTemplates={"inputCoaddName": "deep",
56 "inputName": "calexp",
57 "skyWcsName": "gbdesAstrometricFit",
58 "photoCalibName": "fgcm"},
59 # TODO: remove on DM-39854
60 deprecatedTemplates={"skyWcsName": "Deprecated; will be removed after v26.",
61 "photoCalibName": "Deprecated; will be removed after v26."
62 }):
63 inputSchema = cT.InitInput(
64 doc="Schema for the input measurement catalogs.",
65 name="{inputCoaddName}Coadd_ref_schema",
66 storageClass="SourceCatalog",
67 )
68 outputSchema = cT.InitOutput(
69 doc="Schema for the output forced measurement catalogs.",
70 name="forced_src_schema",
71 storageClass="SourceCatalog",
72 )
73 exposure = cT.Input(
74 doc="Input exposure to perform photometry on.",
75 name="{inputName}",
76 storageClass="ExposureF",
77 dimensions=["instrument", "visit", "detector"],
78 )
79 refCat = cT.Input(
80 doc="Catalog of shapes and positions at which to force photometry.",
81 name="{inputCoaddName}Coadd_ref",
82 storageClass="SourceCatalog",
83 dimensions=["skymap", "tract", "patch"],
84 multiple=True,
85 deferLoad=True,
86 )
87 skyMap = cT.Input(
88 doc="SkyMap dataset that defines the coordinate system of the reference catalog.",
89 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
90 storageClass="SkyMap",
91 dimensions=["skymap"],
92 )
93 skyCorr = cT.Input(
94 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
95 name="skyCorr",
96 storageClass="Background",
97 dimensions=("instrument", "visit", "detector"),
98 )
99 visitSummary = cT.Input(
100 doc="Input visit-summary catalog with updated calibration objects.",
101 name="finalVisitSummary",
102 storageClass="ExposureCatalog",
103 dimensions=("instrument", "visit"),
104 )
105 externalSkyWcsTractCatalog = cT.Input(
106 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
107 "id for the catalog id, sorted on id for fast lookup."),
108 name="{skyWcsName}SkyWcsCatalog",
109 storageClass="ExposureCatalog",
110 dimensions=["instrument", "visit", "tract"],
111 # TODO: remove on DM-39854
112 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
113 )
114 externalSkyWcsGlobalCatalog = cT.Input(
115 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
116 "These catalogs use the detector id for the catalog id, sorted on id for "
117 "fast lookup."),
118 name="finalVisitSummary",
119 storageClass="ExposureCatalog",
120 dimensions=["instrument", "visit"],
121 # TODO: remove on DM-39854
122 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
123 )
124 externalPhotoCalibTractCatalog = cT.Input(
125 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
126 "detector id for the catalog id, sorted on id for fast lookup."),
127 name="{photoCalibName}PhotoCalibCatalog",
128 storageClass="ExposureCatalog",
129 dimensions=["instrument", "visit", "tract"],
130 # TODO: remove on DM-39854
131 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
132 )
133 externalPhotoCalibGlobalCatalog = cT.Input(
134 doc=("Per-visit photometric calibrations computed globally (with no tract "
135 "information). These catalogs use the detector id for the catalog id, "
136 "sorted on id for fast lookup."),
137 name="finalVisitSummary",
138 storageClass="ExposureCatalog",
139 dimensions=["instrument", "visit"],
140 # TODO: remove on DM-39854
141 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
142 )
143 finalizedPsfApCorrCatalog = cT.Input(
144 doc=("Per-visit finalized psf models and aperture correction maps. "
145 "These catalogs use the detector id for the catalog id, "
146 "sorted on id for fast lookup."),
147 name="finalized_psf_ap_corr_catalog",
148 storageClass="ExposureCatalog",
149 dimensions=["instrument", "visit"],
150 # TODO: remove on DM-39854
151 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
152 )
153 measCat = cT.Output(
154 doc="Output forced photometry catalog.",
155 name="forced_src",
156 storageClass="SourceCatalog",
157 dimensions=["instrument", "visit", "detector", "skymap", "tract"],
158 )
159
160 def __init__(self, *, config=None):
161 super().__init__(config=config)
162 if not config.doApplySkyCorr:
163 self.inputs.remove("skyCorr")
164 if config.doApplyExternalSkyWcs:
165 if config.useGlobalExternalSkyWcs:
166 self.inputs.remove("externalSkyWcsTractCatalog")
167 else:
168 self.inputs.remove("externalSkyWcsGlobalCatalog")
169 else:
170 self.inputs.remove("externalSkyWcsTractCatalog")
171 self.inputs.remove("externalSkyWcsGlobalCatalog")
172 if config.doApplyExternalPhotoCalib:
173 if config.useGlobalExternalPhotoCalib:
174 self.inputs.remove("externalPhotoCalibTractCatalog")
175 else:
176 self.inputs.remove("externalPhotoCalibGlobalCatalog")
177 else:
178 self.inputs.remove("externalPhotoCalibTractCatalog")
179 self.inputs.remove("externalPhotoCalibGlobalCatalog")
180 if not config.doApplyFinalizedPsf:
181 self.inputs.remove("finalizedPsfApCorrCatalog")
182
183
184class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig,
185 pipelineConnections=ForcedPhotCcdConnections):
186 """Config class for forced measurement driver task."""
187 measurement = lsst.pex.config.ConfigurableField(
188 target=ForcedMeasurementTask,
189 doc="subtask to do forced measurement"
190 )
191 coaddName = lsst.pex.config.Field(
192 doc="coadd name: typically one of deep or goodSeeing",
193 dtype=str,
194 default="deep",
195 )
196 doApCorr = lsst.pex.config.Field(
197 dtype=bool,
198 default=True,
199 doc="Run subtask to apply aperture corrections"
200 )
201 applyApCorr = lsst.pex.config.ConfigurableField(
202 target=ApplyApCorrTask,
203 doc="Subtask to apply aperture corrections"
204 )
205 catalogCalculation = lsst.pex.config.ConfigurableField(
206 target=CatalogCalculationTask,
207 doc="Subtask to run catalogCalculation plugins on catalog"
208 )
209 doApplyExternalPhotoCalib = lsst.pex.config.Field(
210 dtype=bool,
211 default=False,
212 doc=("Whether to apply external photometric calibration via an "
213 "`lsst.afw.image.PhotoCalib` object."),
214 # TODO: remove on DM-39854
215 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.",
216 )
217 useGlobalExternalPhotoCalib = lsst.pex.config.Field(
218 dtype=bool,
219 default=False,
220 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations "
221 "that are not run per-tract. When False, use per-tract photometric "
222 "calibration files."),
223 # TODO: remove on DM-39854
224 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.",
225 )
226 doApplyExternalSkyWcs = lsst.pex.config.Field(
227 dtype=bool,
228 default=False,
229 doc=("Whether to apply external astrometric calibration via an "
230 "`lsst.afw.geom.SkyWcs` object."),
231 # TODO: remove on DM-39854
232 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.",
233 )
234 useGlobalExternalSkyWcs = lsst.pex.config.Field(
235 dtype=bool,
236 default=False,
237 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations "
238 "that are not run per-tract. When False, use per-tract wcs "
239 "files."),
240 # TODO: remove on DM-39854
241 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.",
242 )
243 doApplyFinalizedPsf = lsst.pex.config.Field(
244 dtype=bool,
245 default=False,
246 doc="Whether to apply finalized psf models and aperture correction map.",
247 # TODO: remove on DM-39854
248 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.",
249 )
250 doApplySkyCorr = lsst.pex.config.Field(
251 dtype=bool,
252 default=False,
253 doc="Apply sky correction?",
254 )
255 includePhotoCalibVar = lsst.pex.config.Field(
256 dtype=bool,
257 default=False,
258 doc="Add photometric calibration variance to warp variance plane?",
259 )
260 footprintSource = lsst.pex.config.ChoiceField(
261 dtype=str,
262 doc="Where to obtain footprints to install in the measurement catalog, prior to measurement.",
263 allowed={
264 "transformed": "Transform footprints from the reference catalog (downgrades HeavyFootprints).",
265 "psf": ("Use the scaled shape of the PSF at the position of each source (does not generate "
266 "HeavyFootprints)."),
267 },
268 optional=True,
269 default="transformed",
270 )
271 psfFootprintScaling = lsst.pex.config.Field(
272 dtype=float,
273 doc="Scaling factor to apply to the PSF shape when footprintSource='psf' (ignored otherwise).",
274 default=3.0,
275 )
276 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
277
278 def setDefaults(self):
279 # Docstring inherited.
280 super().setDefaults()
281 # Footprints here will not be entirely correct, so don't try to make
282 # a biased correction for blended neighbors.
283 self.measurement.doReplaceWithNoise = False
284 # Only run a minimal set of plugins, as these measurements are only
285 # needed for PSF-like sources.
286 self.measurement.plugins.names = ["base_PixelFlags",
287 "base_TransformedCentroid",
288 "base_PsfFlux",
289 "base_LocalBackground",
290 "base_LocalPhotoCalib",
291 "base_LocalWcs",
292 ]
293 self.measurement.slots.shape = None
294 # Keep track of which footprints contain streaks
295 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK']
296 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK']
297 # Make catalogCalculation a no-op by default as no modelFlux is setup
298 # by default in ForcedMeasurementTask.
299 self.catalogCalculation.plugins.names = []
300
301
303 """A pipeline task for performing forced measurement on CCD images.
304
305 Parameters
306 ----------
307 refSchema : `lsst.afw.table.Schema`, optional
308 The schema of the reference catalog, passed to the constructor of the
309 references subtask. Optional, but must be specified if ``initInputs``
310 is not; if both are specified, ``initInputs`` takes precedence.
311 initInputs : `dict`
312 Dictionary that can contain a key ``inputSchema`` containing the
313 schema. If present will override the value of ``refSchema``.
314 **kwargs
315 Keyword arguments are passed to the supertask constructor.
316 """
317
318 ConfigClass = ForcedPhotCcdConfig
319 _DefaultName = "forcedPhotCcd"
320 dataPrefix = ""
321
322 def __init__(self, refSchema=None, initInputs=None, **kwargs):
323 super().__init__(**kwargs)
324
325 if initInputs is not None:
326 refSchema = initInputs['inputSchema'].schema
327
328 if refSchema is None:
329 raise ValueError("No reference schema provided.")
330
331 self.makeSubtask("measurement", refSchema=refSchema)
332 # It is necessary to get the schema internal to the forced measurement
333 # task until such a time that the schema is not owned by the
334 # measurement task, but is passed in by an external caller.
335 if self.config.doApCorr:
336 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
337 self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
338 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
339
340 def runQuantum(self, butlerQC, inputRefs, outputRefs):
341 inputs = butlerQC.get(inputRefs)
342
343 tract = butlerQC.quantum.dataId['tract']
344 skyMap = inputs.pop('skyMap')
345 inputs['refWcs'] = skyMap[tract].getWcs()
346
347 # Connections only exist if they are configured to be used.
348 skyCorr = inputs.pop('skyCorr', None)
349 if self.config.useGlobalExternalSkyWcs:
350 externalSkyWcsCatalog = inputs.pop('externalSkyWcsGlobalCatalog', None)
351 else:
352 externalSkyWcsCatalog = inputs.pop('externalSkyWcsTractCatalog', None)
353 if self.config.useGlobalExternalPhotoCalib:
354 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibGlobalCatalog', None)
355 else:
356 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibTractCatalog', None)
357 finalizedPsfApCorrCatalog = inputs.pop('finalizedPsfApCorrCatalog', None)
358
359 inputs['exposure'] = self.prepareCalibratedExposure(
360 inputs['exposure'],
361 skyCorr=skyCorr,
362 externalSkyWcsCatalog=externalSkyWcsCatalog,
363 externalPhotoCalibCatalog=externalPhotoCalibCatalog,
364 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog,
365 visitSummary=inputs.pop("visitSummary"),
366 )
367
368 inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'],
369 inputs['refWcs'])
370
371 if inputs['refCat'] is None:
372 self.log.info("No WCS for exposure %s. No %s catalog will be written.",
373 butlerQC.quantum.dataId, outputRefs.measCat.datasetType.name)
374 else:
375 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
376 inputs['exposure'],
377 inputs['refCat'], inputs['refWcs'])
378 self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs'])
379 outputs = self.run(**inputs)
380 butlerQC.put(outputs, outputRefs)
381
382 def prepareCalibratedExposure(self, exposure, skyCorr=None, externalSkyWcsCatalog=None,
383 externalPhotoCalibCatalog=None, finalizedPsfApCorrCatalog=None,
384 visitSummary=None):
385 """Prepare a calibrated exposure and apply external calibrations
386 and sky corrections if so configured.
387
388 Parameters
389 ----------
390 exposure : `lsst.afw.image.exposure.Exposure`
391 Input exposure to adjust calibrations.
392 skyCorr : `lsst.afw.math.backgroundList`, optional
393 Sky correction frame to apply if doApplySkyCorr=True.
394 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
395 Exposure catalog with external skyWcs to be applied if
396 config.doApplyExternalSkyWcs=True. Catalog uses the detector id
397 for the catalog id, sorted on id for fast lookup. Deprecated in
398 favor of ``visitSummary`` and will be removed after v26.
399 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
400 Exposure catalog with external photoCalib to be applied if
401 config.doApplyExternalPhotoCalib=True. Catalog uses the detector
402 id for the catalog id, sorted on id for fast lookup. Deprecated in
403 favor of ``visitSummary`` and will be removed after v26.
404 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
405 Exposure catalog with finalized psf models and aperture correction
406 maps to be applied if config.doApplyFinalizedPsf=True. Catalog
407 uses the detector id for the catalog id, sorted on id for fast
408 lookup. Deprecated in favor of ``visitSummary`` and will be removed
409 after v26.
410 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
411 Exposure catalog with update calibrations; any not-None calibration
412 objects attached will be used. These are applied first and may be
413 overridden by other arguments.
414
415 Returns
416 -------
417 exposure : `lsst.afw.image.exposure.Exposure`
418 Exposure with adjusted calibrations.
419 """
420 detectorId = exposure.getInfo().getDetector().getId()
421
422 if visitSummary is not None:
423 row = visitSummary.find(detectorId)
424 if row is None:
425 raise RuntimeError(f"Detector id {detectorId} not found in visitSummary.")
426 if (photoCalib := row.getPhotoCalib()) is not None:
427 exposure.setPhotoCalib(photoCalib)
428 if (skyWcs := row.getWcs()) is not None:
429 exposure.setWcs(skyWcs)
430 if (psf := row.getPsf()) is not None:
431 exposure.setPsf(psf)
432 if (apCorrMap := row.getApCorrMap()) is not None:
433 exposure.info.setApCorrMap(apCorrMap)
434
435 if externalPhotoCalibCatalog is not None:
436 # TODO: remove on DM-39854
437 warnings.warn(
438 "The 'externalPhotoCalibCatalog' argument is deprecated in favor of 'visitSummary' and will "
439 "be removed after v26.",
440 FutureWarning,
441 stacklevel=find_outside_stacklevel("lsst.meas.base"),
442 )
443 row = externalPhotoCalibCatalog.find(detectorId)
444 if row is None:
445 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog; "
446 "Using original photoCalib.", detectorId)
447 else:
448 photoCalib = row.getPhotoCalib()
449 if photoCalib is None:
450 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog; "
451 "Using original photoCalib.", detectorId)
452 else:
453 exposure.setPhotoCalib(photoCalib)
454
455 if externalSkyWcsCatalog is not None:
456 # TODO: remove on DM-39854
457 warnings.warn(
458 "The 'externalSkyWcsCatalog' argument is deprecated in favor of 'visitSummary' and will "
459 "be removed after v26.",
460 FutureWarning,
461 stacklevel=find_outside_stacklevel("lsst.meas.base"),
462 )
463 row = externalSkyWcsCatalog.find(detectorId)
464 if row is None:
465 self.log.warning("Detector id %s not found in externalSkyWcsCatalog; "
466 "Using original skyWcs.", detectorId)
467 else:
468 skyWcs = row.getWcs()
469 if skyWcs is None:
470 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog; "
471 "Using original skyWcs.", detectorId)
472 else:
473 exposure.setWcs(skyWcs)
474
475 if finalizedPsfApCorrCatalog is not None:
476 # TODO: remove on DM-39854
477 warnings.warn(
478 "The 'finalizedPsfApCorrCatalog' argument is deprecated in favor of 'visitSummary' and will "
479 "be removed after v26.",
480 FutureWarning,
481 stacklevel=find_outside_stacklevel("lsst.meas.base"),
482 )
483 row = finalizedPsfApCorrCatalog.find(detectorId)
484 if row is None:
485 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; "
486 "Using original psf.", detectorId)
487 else:
488 psf = row.getPsf()
489 apCorrMap = row.getApCorrMap()
490 if psf is None or apCorrMap is None:
491 self.log.warning("Detector id %s has None for psf/apCorrMap in "
492 "finalizedPsfApCorrCatalog; Using original psf.", detectorId)
493 else:
494 exposure.setPsf(psf)
495 exposure.setApCorrMap(apCorrMap)
496
497 if skyCorr is not None:
498 exposure.maskedImage -= skyCorr.getImage()
499
500 return exposure
501
502 def mergeAndFilterReferences(self, exposure, refCats, refWcs):
503 """Filter reference catalog so that all sources are within the
504 boundaries of the exposure.
505
506 Parameters
507 ----------
508 exposure : `lsst.afw.image.exposure.Exposure`
509 Exposure to generate the catalog for.
510 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle`
511 Handles for catalogs of shapes and positions at which to force
512 photometry.
513 refWcs : `lsst.afw.image.SkyWcs`
514 Reference world coordinate system.
515
516 Returns
517 -------
518 refSources : `lsst.afw.table.SourceCatalog`
519 Filtered catalog of forced sources to measure.
520
521 Notes
522 -----
523 The majority of this code is based on the methods of
524 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader
525
526 """
527 mergedRefCat = None
528
529 # Step 1: Determine bounds of the exposure photometry will
530 # be performed on.
531 expWcs = exposure.getWcs()
532 if expWcs is None:
533 self.log.info("Exposure has no WCS. Returning None for mergedRefCat.")
534 else:
535 expRegion = exposure.getBBox(lsst.afw.image.PARENT)
536 expBBox = lsst.geom.Box2D(expRegion)
537 expBoxCorners = expBBox.getCorners()
538 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for
539 corner in expBoxCorners]
540 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners)
541
542 # Step 2: Filter out reference catalog sources that are
543 # not contained within the exposure boundaries, or whose
544 # parents are not within the exposure boundaries. Note
545 # that within a single input refCat, the parents always
546 # appear before the children.
547 for refCat in refCats:
548 refCat = refCat.get()
549 if mergedRefCat is None:
550 mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table)
551 containedIds = {0} # zero as a parent ID means "this is a parent"
552 for record in refCat:
553 if (expPolygon.contains(record.getCoord().getVector()) and record.getParent()
554 in containedIds):
555 record.setFootprint(record.getFootprint())
556 mergedRefCat.append(record)
557 containedIds.add(record.getId())
558 if mergedRefCat is None:
559 raise RuntimeError("No reference objects for forced photometry.")
560 mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey())
561 return mergedRefCat
562
563 def generateMeasCat(self, dataId, exposure, refCat, refWcs):
564 """Generate a measurement catalog.
565
566 Parameters
567 ----------
568 dataId : `lsst.daf.butler.DataCoordinate`
569 Butler data ID for this image, with ``{visit, detector}`` keys.
570 exposure : `lsst.afw.image.exposure.Exposure`
571 Exposure to generate the catalog for.
572 refCat : `lsst.afw.table.SourceCatalog`
573 Catalog of shapes and positions at which to force photometry.
574 refWcs : `lsst.afw.image.SkyWcs`
575 Reference world coordinate system.
576 This parameter is not currently used.
577
578 Returns
579 -------
580 measCat : `lsst.afw.table.SourceCatalog`
581 Catalog of forced sources to measure.
582 expId : `int`
583 Unique binary id associated with the input exposure
584 """
585 id_generator = self.config.idGenerator.apply(dataId)
586 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
587 idFactory=id_generator.make_table_id_factory())
588 return measCat, id_generator.catalog_id
589
590 def run(self, measCat, exposure, refCat, refWcs, exposureId=None):
591 """Perform forced measurement on a single exposure.
592
593 Parameters
594 ----------
595 measCat : `lsst.afw.table.SourceCatalog`
596 The measurement catalog, based on the sources listed in the
597 reference catalog.
598 exposure : `lsst.afw.image.Exposure`
599 The measurement image upon which to perform forced detection.
600 refCat : `lsst.afw.table.SourceCatalog`
601 The reference catalog of sources to measure.
602 refWcs : `lsst.afw.image.SkyWcs`
603 The WCS for the references.
604 exposureId : `int`
605 Optional unique exposureId used for random seed in measurement
606 task.
607
608 Returns
609 -------
610 result : `lsst.pipe.base.Struct`
611 Structure with fields:
612
613 ``measCat``
614 Catalog of forced measurement results
615 (`lsst.afw.table.SourceCatalog`).
616 """
617 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
618 if self.config.doApCorr:
619 apCorrMap = exposure.getInfo().getApCorrMap()
620 if apCorrMap is None:
621 self.log.warning("Forced exposure image does not have valid aperture correction; skipping.")
622 else:
623 self.applyApCorr.run(
624 catalog=measCat,
625 apCorrMap=apCorrMap,
626 )
627 self.catalogCalculation.run(measCat)
628
629 return pipeBase.Struct(measCat=measCat)
630
631 def attachFootprints(self, sources, refCat, exposure, refWcs):
632 """Attach footprints to blank sources prior to measurements.
633
634 Notes
635 -----
636 `~lsst.afw.detection.Footprint` objects for forced photometry must
637 be in the pixel coordinate system of the image being measured, while
638 the actual detections may start out in a different coordinate system.
639
640 Subclasses of this class may implement this method to define how
641 those `~lsst.afw.detection.Footprint` objects should be generated.
642
643 This default implementation transforms depends on the
644 ``footprintSource`` configuration parameter.
645 """
646 if self.config.footprintSource == "transformed":
647 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs)
648 elif self.config.footprintSource == "psf":
649 return self.measurement.attachPsfShapeFootprints(sources, exposure,
650 scaling=self.config.psfFootprintScaling)
651
652
653class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections,
654 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
655 defaultTemplates={"inputCoaddName": "goodSeeing",
656 "inputName": "calexp",
657 "skyWcsName": "gbdesAstrometricFit",
658 "photoCalibName": "fgcm"},
659 # TODO: remove on DM-39854
660 deprecatedTemplates={
661 "skyWcsName": "Deprecated; will be removed after v26.",
662 "photoCalibName": "Deprecated; will be removed after v26."
663 }):
664 refCat = cT.Input(
665 doc="Catalog of positions at which to force photometry.",
666 name="{inputCoaddName}Diff_fullDiaObjTable",
667 storageClass="DataFrame",
668 dimensions=["skymap", "tract", "patch"],
669 multiple=True,
670 deferLoad=True,
671 )
672 exposure = cT.Input(
673 doc="Input exposure to perform photometry on.",
674 name="{inputName}",
675 storageClass="ExposureF",
676 dimensions=["instrument", "visit", "detector"],
677 )
678 skyCorr = cT.Input(
679 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
680 name="skyCorr",
681 storageClass="Background",
682 dimensions=("instrument", "visit", "detector"),
683 )
684 visitSummary = cT.Input(
685 doc="Input visit-summary catalog with updated calibration objects.",
686 name="finalVisitSummary",
687 storageClass="ExposureCatalog",
688 dimensions=("instrument", "visit"),
689 )
690 externalSkyWcsTractCatalog = cT.Input(
691 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
692 "id for the catalog id, sorted on id for fast lookup."),
693 name="{skyWcsName}SkyWcsCatalog",
694 storageClass="ExposureCatalog",
695 dimensions=["instrument", "visit", "tract"],
696 # TODO: remove on DM-39854
697 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
698 )
699 externalSkyWcsGlobalCatalog = cT.Input(
700 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
701 "These catalogs use the detector id for the catalog id, sorted on id for "
702 "fast lookup."),
703 name="{skyWcsName}SkyWcsCatalog",
704 storageClass="ExposureCatalog",
705 dimensions=["instrument", "visit"],
706 # TODO: remove on DM-39854
707 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
708 )
709 externalPhotoCalibTractCatalog = cT.Input(
710 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
711 "detector id for the catalog id, sorted on id for fast lookup."),
712 name="{photoCalibName}PhotoCalibCatalog",
713 storageClass="ExposureCatalog",
714 dimensions=["instrument", "visit", "tract"],
715 # TODO: remove on DM-39854
716 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
717 )
718 externalPhotoCalibGlobalCatalog = cT.Input(
719 doc=("Per-visit photometric calibrations computed globally (with no tract "
720 "information). These catalogs use the detector id for the catalog id, "
721 "sorted on id for fast lookup."),
722 name="{photoCalibName}PhotoCalibCatalog",
723 storageClass="ExposureCatalog",
724 dimensions=["instrument", "visit"],
725 # TODO: remove on DM-39854
726 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
727 )
728 finalizedPsfApCorrCatalog = cT.Input(
729 doc=("Per-visit finalized psf models and aperture correction maps. "
730 "These catalogs use the detector id for the catalog id, "
731 "sorted on id for fast lookup."),
732 name="finalized_psf_ap_corr_catalog",
733 storageClass="ExposureCatalog",
734 dimensions=["instrument", "visit"],
735 # TODO: remove on DM-39854
736 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26."
737 )
738 measCat = cT.Output(
739 doc="Output forced photometry catalog.",
740 name="forced_src_diaObject",
741 storageClass="SourceCatalog",
742 dimensions=["instrument", "visit", "detector", "skymap", "tract"],
743 )
744 outputSchema = cT.InitOutput(
745 doc="Schema for the output forced measurement catalogs.",
746 name="forced_src_diaObject_schema",
747 storageClass="SourceCatalog",
748 )
749
750 def __init__(self, *, config=None):
751 super().__init__(config=config)
752 if not config.doApplySkyCorr:
753 self.inputs.remove("skyCorr")
754 if config.doApplyExternalSkyWcs:
755 if config.useGlobalExternalSkyWcs:
756 self.inputs.remove("externalSkyWcsTractCatalog")
757 else:
758 self.inputs.remove("externalSkyWcsGlobalCatalog")
759 else:
760 self.inputs.remove("externalSkyWcsTractCatalog")
761 self.inputs.remove("externalSkyWcsGlobalCatalog")
762 if config.doApplyExternalPhotoCalib:
763 if config.useGlobalExternalPhotoCalib:
764 self.inputs.remove("externalPhotoCalibTractCatalog")
765 else:
766 self.inputs.remove("externalPhotoCalibGlobalCatalog")
767 else:
768 self.inputs.remove("externalPhotoCalibTractCatalog")
769 self.inputs.remove("externalPhotoCalibGlobalCatalog")
770 if not config.doApplyFinalizedPsf:
771 self.inputs.remove("finalizedPsfApCorrCatalog")
772
773
775 pipelineConnections=ForcedPhotCcdFromDataFrameConnections):
776 def setDefaults(self):
777 super().setDefaults()
778 self.footprintSource = "psf"
779 self.measurement.doReplaceWithNoise = False
780 # Only run a minimal set of plugins, as these measurements are only
781 # needed for PSF-like sources.
782 self.measurement.plugins.names = ["base_PixelFlags",
783 "base_TransformedCentroidFromCoord",
784 "base_PsfFlux",
785 "base_LocalBackground",
786 "base_LocalPhotoCalib",
787 "base_LocalWcs",
788 ]
789 self.measurement.slots.shape = None
790 # Keep track of which footprints contain streaks
791 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK']
792 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK']
793 # Make catalogCalculation a no-op by default as no modelFlux is setup
794 # by default in ForcedMeasurementTask.
795 self.catalogCalculation.plugins.names = []
796
797 self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'}
798 self.measurement.slots.centroid = "base_TransformedCentroidFromCoord"
799 self.measurement.slots.psfFlux = "base_PsfFlux"
800
801 def validate(self):
802 super().validate()
803 if self.footprintSource == "transformed":
804 raise ValueError("Cannot transform footprints from reference catalog, "
805 "because DataFrames can't hold footprints.")
806
807
809 """Force Photometry on a per-detector exposure with coords from a DataFrame
810
811 Uses input from a DataFrame instead of SourceCatalog
812 like the base class ForcedPhotCcd does.
813 Writes out a SourceCatalog so that the downstream
814 WriteForcedSourceTableTask can be reused with output from this Task.
815 """
816 _DefaultName = "forcedPhotCcdFromDataFrame"
817 ConfigClass = ForcedPhotCcdFromDataFrameConfig
818
819 def __init__(self, refSchema=None, initInputs=None, **kwargs):
820 # Parent's init assumes that we have a reference schema; Cannot reuse
821 pipeBase.PipelineTask.__init__(self, **kwargs)
822
823 self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema())
824
825 if self.config.doApCorr:
826 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
827 self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
828 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
829
830 def runQuantum(self, butlerQC, inputRefs, outputRefs):
831 inputs = butlerQC.get(inputRefs)
832
833 # When run with dataframes, we do not need a reference wcs.
834 inputs['refWcs'] = None
835
836 # Connections only exist if they are configured to be used.
837 skyCorr = inputs.pop('skyCorr', None)
838 if self.config.useGlobalExternalSkyWcs:
839 externalSkyWcsCatalog = inputs.pop('externalSkyWcsGlobalCatalog', None)
840 else:
841 externalSkyWcsCatalog = inputs.pop('externalSkyWcsTractCatalog', None)
842 if self.config.useGlobalExternalPhotoCalib:
843 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibGlobalCatalog', None)
844 else:
845 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibTractCatalog', None)
846 finalizedPsfApCorrCatalog = inputs.pop('finalizedPsfApCorrCatalog', None)
847
848 inputs['exposure'] = self.prepareCalibratedExposure(
849 inputs['exposure'],
850 skyCorr=skyCorr,
851 externalSkyWcsCatalog=externalSkyWcsCatalog,
852 externalPhotoCalibCatalog=externalPhotoCalibCatalog,
853 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog,
854 visitSummary=inputs.pop("visitSummary"),
855 )
856
857 self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']]))
858 if inputs["exposure"].getWcs() is not None:
859 refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'dec']})
860 for i in inputs['refCat']],
861 inputs['exposure'].getBBox(), inputs['exposure'].getWcs())
862 inputs['refCat'] = refCat
863 # generateMeasCat does not use the refWcs.
864 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(
865 inputRefs.exposure.dataId, inputs['exposure'], inputs['refCat'], inputs['refWcs']
866 )
867 # attachFootprints only uses refWcs in ``transformed`` mode, which is not
868 # supported in the DataFrame-backed task.
869 self.attachFootprints(inputs["measCat"], inputs["refCat"], inputs["exposure"], inputs["refWcs"])
870 outputs = self.run(**inputs)
871
872 butlerQC.put(outputs, outputRefs)
873 else:
874 self.log.info("No WCS for %s. Skipping and no %s catalog will be written.",
875 butlerQC.quantum.dataId, outputRefs.measCat.datasetType.name)
876
877 def df2RefCat(self, dfList, exposureBBox, exposureWcs):
878 """Convert list of DataFrames to reference catalog
879
880 Concatenate list of DataFrames presumably from multiple patches and
881 downselect rows that overlap the exposureBBox using the exposureWcs.
882
883 Parameters
884 ----------
885 dfList : `list` of `pandas.DataFrame`
886 Each element containst diaObjects with ra/dec position in degrees
887 Columns 'diaObjectId', 'ra', 'dec' are expected
888 exposureBBox : `lsst.geom.Box2I`
889 Bounding box on which to select rows that overlap
890 exposureWcs : `lsst.afw.geom.SkyWcs`
891 World coordinate system to convert sky coords in ref cat to
892 pixel coords with which to compare with exposureBBox
893
894 Returns
895 -------
896 refCat : `lsst.afw.table.SourceTable`
897 Source Catalog with minimal schema that overlaps exposureBBox
898 """
899 df = pd.concat(dfList)
900 # translate ra/dec coords in dataframe to detector pixel coords
901 # to down select rows that overlap the detector bbox
902 mapping = exposureWcs.getTransform().getMapping()
903 x, y = mapping.applyInverse(np.array(df[['ra', 'dec']].values*2*np.pi/360).T)
904 inBBox = lsst.geom.Box2D(exposureBBox).contains(x, y)
905 refCat = self.df2SourceCat(df[inBBox])
906 return refCat
907
908 def df2SourceCat(self, df):
909 """Create minimal schema SourceCatalog from a pandas DataFrame.
910
911 The forced measurement subtask expects this as input.
912
913 Parameters
914 ----------
915 df : `pandas.DataFrame`
916 DiaObjects with locations and ids.
917
918 Returns
919 -------
920 outputCatalog : `lsst.afw.table.SourceTable`
921 Output catalog with minimal schema.
922 """
924 outputCatalog = lsst.afw.table.SourceCatalog(schema)
925 outputCatalog.reserve(len(df))
926
927 for diaObjectId, ra, dec in df[['ra', 'dec']].itertuples():
928 outputRecord = outputCatalog.addNew()
929 outputRecord.setId(diaObjectId)
930 outputRecord.setCoord(lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees))
931 return outputCatalog
static Schema makeMinimalSchema()
static Key< RecordId > getParentKey()
df2RefCat(self, dfList, exposureBBox, exposureWcs)
runQuantum(self, butlerQC, inputRefs, outputRefs)