Coverage for python/lsst/meas/base/forcedPhotCcd.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import collections
23import logging
24import pandas as pd
25import numpy as np
27import lsst.pex.config
28import lsst.pex.exceptions
29import lsst.pipe.base
30import lsst.geom
31import lsst.afw.geom
32import lsst.afw.image
33import lsst.afw.table
34import lsst.sphgeom
36from lsst.pipe.base import PipelineTaskConnections
37import lsst.pipe.base.connectionTypes as cT
39import lsst.pipe.base as pipeBase
40from lsst.skymap import BaseSkyMap
42from .references import MultiBandReferencesTask
43from .forcedMeasurement import ForcedMeasurementTask
44from .applyApCorr import ApplyApCorrTask
45from .catalogCalculation import CatalogCalculationTask
47try:
48 from lsst.meas.mosaic import applyMosaicResults
49except ImportError:
50 applyMosaicResults = None
52__all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask", "imageOverlapsTract",
53 "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig")
56class PerTractCcdDataIdContainer(pipeBase.DataIdContainer):
57 """A data ID container which combines raw data IDs with a tract.
59 Notes
60 -----
61 Required because we need to add "tract" to the raw data ID keys (defined as
62 whatever we use for ``src``) when no tract is provided (so that the user is
63 not required to know which tracts are spanned by the raw data ID).
65 This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is
66 being measured using the detection information, a set of reference
67 catalogs, from the set of coadds which intersect with the calexp. It needs
68 the calexp id (e.g. visit, raft, sensor), but is also uses the tract to
69 decide what set of coadds to use. The references from the tract whose
70 patches intersect with the calexp are used.
71 """
73 def makeDataRefList(self, namespace):
74 """Make self.refList from self.idList
75 """
76 if self.datasetType is None:
77 raise RuntimeError("Must call setDatasetType first")
78 log = logging.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer")
79 skymap = None
80 visitTract = collections.defaultdict(set) # Set of tracts for each visit
81 visitRefs = collections.defaultdict(list) # List of data references for each visit
82 for dataId in self.idList:
83 if "tract" not in dataId:
84 # Discover which tracts the data overlaps
85 log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId))
86 if skymap is None:
87 skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap")
89 for ref in namespace.butler.subset("calexp", dataId=dataId):
90 if not ref.datasetExists("calexp"):
91 continue
93 visit = ref.dataId["visit"]
94 visitRefs[visit].append(ref)
96 md = ref.get("calexp_md", immediate=True)
97 wcs = lsst.afw.geom.makeSkyWcs(md)
98 box = lsst.geom.Box2D(lsst.afw.image.bboxFromMetadata(md))
99 # Going with just the nearest tract. Since we're throwing all tracts for the visit
100 # together, this shouldn't be a problem unless the tracts are much smaller than a CCD.
101 tract = skymap.findTract(wcs.pixelToSky(box.getCenter()))
102 if imageOverlapsTract(tract, wcs, box):
103 visitTract[visit].add(tract.getId())
104 else:
105 self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId))
107 # Ensure all components of a visit are kept together by putting them all in the same set of tracts
108 for visit, tractSet in visitTract.items():
109 for ref in visitRefs[visit]:
110 for tract in tractSet:
111 self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType,
112 dataId=ref.dataId, tract=tract))
113 if visitTract:
114 tractCounter = collections.Counter()
115 for tractSet in visitTract.values():
116 tractCounter.update(tractSet)
117 log.info("Number of visits for each tract: %s", dict(tractCounter))
120def imageOverlapsTract(tract, imageWcs, imageBox):
121 """Return whether the given bounding box overlaps the tract given a WCS.
123 Parameters
124 ----------
125 tract : `lsst.skymap.TractInfo`
126 TractInfo specifying a tract.
127 imageWcs : `lsst.afw.geom.SkyWcs`
128 World coordinate system for the image.
129 imageBox : `lsst.geom.Box2I`
130 Bounding box for the image.
132 Returns
133 -------
134 overlap : `bool`
135 `True` if the bounding box overlaps the tract; `False` otherwise.
136 """
137 tractPoly = tract.getOuterSkyPolygon()
139 imagePixelCorners = lsst.geom.Box2D(imageBox).getCorners()
140 try:
141 imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners)
142 except lsst.pex.exceptions.LsstCppException as e:
143 # Protecting ourselves from awful Wcs solutions in input images
144 if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException)
145 and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)):
146 raise
147 return False
149 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageSkyCorners])
150 return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by"
153class ForcedPhotCcdConnections(PipelineTaskConnections,
154 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
155 defaultTemplates={"inputCoaddName": "deep",
156 "inputName": "calexp"}):
157 inputSchema = cT.InitInput(
158 doc="Schema for the input measurement catalogs.",
159 name="{inputCoaddName}Coadd_ref_schema",
160 storageClass="SourceCatalog",
161 )
162 outputSchema = cT.InitOutput(
163 doc="Schema for the output forced measurement catalogs.",
164 name="forced_src_schema",
165 storageClass="SourceCatalog",
166 )
167 exposure = cT.Input(
168 doc="Input exposure to perform photometry on.",
169 name="{inputName}",
170 storageClass="ExposureF",
171 dimensions=["instrument", "visit", "detector"],
172 )
173 refCat = cT.Input(
174 doc="Catalog of shapes and positions at which to force photometry.",
175 name="{inputCoaddName}Coadd_ref",
176 storageClass="SourceCatalog",
177 dimensions=["skymap", "tract", "patch"],
178 multiple=True,
179 deferLoad=True,
180 )
181 skyMap = cT.Input(
182 doc="SkyMap dataset that defines the coordinate system of the reference catalog.",
183 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
184 storageClass="SkyMap",
185 dimensions=["skymap"],
186 )
187 measCat = cT.Output(
188 doc="Output forced photometry catalog.",
189 name="forced_src",
190 storageClass="SourceCatalog",
191 dimensions=["instrument", "visit", "detector", "skymap", "tract"],
192 )
195class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig,
196 pipelineConnections=ForcedPhotCcdConnections):
197 """Config class for forced measurement driver task."""
198 references = lsst.pex.config.ConfigurableField(
199 target=MultiBandReferencesTask,
200 doc="subtask to retrieve reference source catalog"
201 )
202 measurement = lsst.pex.config.ConfigurableField(
203 target=ForcedMeasurementTask,
204 doc="subtask to do forced measurement"
205 )
206 coaddName = lsst.pex.config.Field(
207 doc="coadd name: typically one of deep or goodSeeing",
208 dtype=str,
209 default="deep",
210 )
211 doApCorr = lsst.pex.config.Field(
212 dtype=bool,
213 default=True,
214 doc="Run subtask to apply aperture corrections"
215 )
216 applyApCorr = lsst.pex.config.ConfigurableField(
217 target=ApplyApCorrTask,
218 doc="Subtask to apply aperture corrections"
219 )
220 catalogCalculation = lsst.pex.config.ConfigurableField(
221 target=CatalogCalculationTask,
222 doc="Subtask to run catalogCalculation plugins on catalog"
223 )
224 doApplyUberCal = lsst.pex.config.Field(
225 dtype=bool,
226 doc="Apply meas_mosaic ubercal results to input calexps?",
227 default=False,
228 deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead",
229 )
230 doApplyExternalPhotoCalib = lsst.pex.config.Field(
231 dtype=bool,
232 default=False,
233 doc=("Whether to apply external photometric calibration via an "
234 "`lsst.afw.image.PhotoCalib` object. Uses the "
235 "``externalPhotoCalibName`` field to determine which calibration "
236 "to load."),
237 )
238 doApplyExternalSkyWcs = lsst.pex.config.Field(
239 dtype=bool,
240 default=False,
241 doc=("Whether to apply external astrometric calibration via an "
242 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` "
243 "field to determine which calibration to load."),
244 )
245 doApplySkyCorr = lsst.pex.config.Field(
246 dtype=bool,
247 default=False,
248 doc="Apply sky correction?",
249 )
250 includePhotoCalibVar = lsst.pex.config.Field(
251 dtype=bool,
252 default=False,
253 doc="Add photometric calibration variance to warp variance plane?",
254 )
255 externalPhotoCalibName = lsst.pex.config.ChoiceField(
256 dtype=str,
257 doc=("Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. "
258 "Unused for Gen3 middleware."),
259 default="jointcal",
260 allowed={
261 "jointcal": "Use jointcal_photoCalib",
262 "fgcm": "Use fgcm_photoCalib",
263 "fgcm_tract": "Use fgcm_tract_photoCalib"
264 },
265 )
266 externalSkyWcsName = lsst.pex.config.ChoiceField(
267 dtype=str,
268 doc="Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.",
269 default="jointcal",
270 allowed={
271 "jointcal": "Use jointcal_wcs"
272 },
273 )
275 def setDefaults(self):
276 # Docstring inherited.
277 # Make catalogCalculation a no-op by default as no modelFlux is setup by default in
278 # ForcedMeasurementTask
279 super().setDefaults()
280 self.measurement.plugins.names |= ['base_LocalPhotoCalib', 'base_LocalWcs']
281 self.catalogCalculation.plugins.names = []
284class ForcedPhotCcdTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
285 """A command-line driver for performing forced measurement on CCD images.
287 Parameters
288 ----------
289 butler : `lsst.daf.persistence.butler.Butler`, optional
290 A Butler which will be passed to the references subtask to allow it to
291 load its schema from disk. Optional, but must be specified if
292 ``refSchema`` is not; if both are specified, ``refSchema`` takes
293 precedence.
294 refSchema : `lsst.afw.table.Schema`, optional
295 The schema of the reference catalog, passed to the constructor of the
296 references subtask. Optional, but must be specified if ``butler`` is
297 not; if both are specified, ``refSchema`` takes precedence.
298 **kwds
299 Keyword arguments are passed to the supertask constructor.
301 Notes
302 -----
303 The `runDataRef` method takes a `~lsst.daf.persistence.ButlerDataRef` argument
304 that corresponds to a single CCD. This should contain the data ID keys that
305 correspond to the ``forced_src`` dataset (the output dataset for this
306 task), which are typically all those used to specify the ``calexp`` dataset
307 (``visit``, ``raft``, ``sensor`` for LSST data) as well as a coadd tract.
308 The tract is used to look up the appropriate coadd measurement catalogs to
309 use as references (e.g. ``deepCoadd_src``; see
310 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more
311 information). While the tract must be given as part of the dataRef, the
312 patches are determined automatically from the bounding box and WCS of the
313 calexp to be measured, and the filter used to fetch references is set via
314 the ``filter`` option in the configuration of
315 :lsst-task:`lsst.meas.base.references.BaseReferencesTask`).
316 """
318 ConfigClass = ForcedPhotCcdConfig
319 RunnerClass = pipeBase.ButlerInitializedTaskRunner
320 _DefaultName = "forcedPhotCcd"
321 dataPrefix = ""
323 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
324 super().__init__(**kwds)
326 if initInputs is not None:
327 refSchema = initInputs['inputSchema'].schema
329 self.makeSubtask("references", butler=butler, schema=refSchema)
330 if refSchema is None:
331 refSchema = self.references.schema
332 self.makeSubtask("measurement", refSchema=refSchema)
333 # It is necessary to get the schema internal to the forced measurement task until such a time
334 # that the schema is not owned by the measurement task, but is passed in by an external caller
335 if self.config.doApCorr:
336 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
337 self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
338 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
340 def runQuantum(self, butlerQC, inputRefs, outputRefs):
341 inputs = butlerQC.get(inputRefs)
343 tract = butlerQC.quantum.dataId['tract']
344 skyMap = inputs.pop("skyMap")
345 inputs['refWcs'] = skyMap[tract].getWcs()
347 inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'],
348 inputs['refWcs'])
350 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
351 inputs['exposure'],
352 inputs['refCat'], inputs['refWcs'],
353 "visit_detector")
354 self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs'])
355 # TODO: apply external calibrations (DM-17062)
356 outputs = self.run(**inputs)
357 butlerQC.put(outputs, outputRefs)
359 def mergeAndFilterReferences(self, exposure, refCats, refWcs):
360 """Filter reference catalog so that all sources are within the
361 boundaries of the exposure.
363 Parameters
364 ----------
365 exposure : `lsst.afw.image.exposure.Exposure`
366 Exposure to generate the catalog for.
367 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle`
368 Handles for catalogs of shapes and positions at which to force
369 photometry.
370 refWcs : `lsst.afw.image.SkyWcs`
371 Reference world coordinate system.
373 Returns
374 -------
375 refSources : `lsst.afw.table.SourceCatalog`
376 Filtered catalog of forced sources to measure.
378 Notes
379 -----
380 Filtering the reference catalog is currently handled by Gen2
381 specific methods. To function for Gen3, this method copies
382 code segments to do the filtering and transformation. The
383 majority of this code is based on the methods of
384 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader
386 """
388 # Step 1: Determine bounds of the exposure photometry will
389 # be performed on.
390 expWcs = exposure.getWcs()
391 expRegion = exposure.getBBox(lsst.afw.image.PARENT)
392 expBBox = lsst.geom.Box2D(expRegion)
393 expBoxCorners = expBBox.getCorners()
394 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for
395 corner in expBoxCorners]
396 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners)
398 # Step 2: Filter out reference catalog sources that are
399 # not contained within the exposure boundaries, or whose
400 # parents are not within the exposure boundaries. Note
401 # that within a single input refCat, the parents always
402 # appear before the children.
403 mergedRefCat = None
404 for refCat in refCats:
405 refCat = refCat.get()
406 if mergedRefCat is None:
407 mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table)
408 containedIds = {0} # zero as a parent ID means "this is a parent"
409 for record in refCat:
410 if expPolygon.contains(record.getCoord().getVector()) and record.getParent() in containedIds:
411 record.setFootprint(record.getFootprint().transform(refWcs, expWcs, expRegion))
412 mergedRefCat.append(record)
413 containedIds.add(record.getId())
414 if mergedRefCat is None:
415 raise RuntimeError("No reference objects for forced photometry.")
416 mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey())
417 return mergedRefCat
419 def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName):
420 """Generate a measurement catalog for Gen3.
422 Parameters
423 ----------
424 exposureDataId : `DataId`
425 Butler dataId for this exposure.
426 exposure : `lsst.afw.image.exposure.Exposure`
427 Exposure to generate the catalog for.
428 refCat : `lsst.afw.table.SourceCatalog`
429 Catalog of shapes and positions at which to force photometry.
430 refWcs : `lsst.afw.image.SkyWcs`
431 Reference world coordinate system.
432 idPackerName : `str`
433 Type of ID packer to construct from the registry.
435 Returns
436 -------
437 measCat : `lsst.afw.table.SourceCatalog`
438 Catalog of forced sources to measure.
439 expId : `int`
440 Unique binary id associated with the input exposure
441 """
442 expId, expBits = exposureDataId.pack(idPackerName, returnMaxBits=True)
443 idFactory = lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits)
445 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
446 idFactory=idFactory)
447 return measCat, expId
449 def runDataRef(self, dataRef, psfCache=None):
450 """Perform forced measurement on a single exposure.
452 Parameters
453 ----------
454 dataRef : `lsst.daf.persistence.ButlerDataRef`
455 Passed to the ``references`` subtask to obtain the reference WCS,
456 the ``getExposure`` method (implemented by derived classes) to
457 read the measurment image, and the ``fetchReferences`` method to
458 get the exposure and load the reference catalog (see
459 :lsst-task`lsst.meas.base.references.CoaddSrcReferencesTask`).
460 Refer to derived class documentation for details of the datasets
461 and data ID keys which are used.
462 psfCache : `int`, optional
463 Size of PSF cache, or `None`. The size of the PSF cache can have
464 a significant effect upon the runtime for complicated PSF models.
466 Notes
467 -----
468 Sources are generated with ``generateMeasCat`` in the ``measurement``
469 subtask. These are passed to ``measurement``'s ``run`` method, which
470 fills the source catalog with the forced measurement results. The
471 sources are then passed to the ``writeOutputs`` method (implemented by
472 derived classes) which writes the outputs.
473 """
474 refWcs = self.references.getWcs(dataRef)
475 exposure = self.getExposure(dataRef)
476 if psfCache is not None:
477 exposure.getPsf().setCacheSize(psfCache)
478 refCat = self.fetchReferences(dataRef, exposure)
480 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
481 idFactory=self.makeIdFactory(dataRef))
482 self.log.info("Performing forced measurement on %s", dataRef.dataId)
483 self.attachFootprints(measCat, refCat, exposure, refWcs)
485 exposureId = self.getExposureId(dataRef)
487 forcedPhotResult = self.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
489 self.writeOutput(dataRef, forcedPhotResult.measCat)
491 def run(self, measCat, exposure, refCat, refWcs, exposureId=None):
492 """Perform forced measurement on a single exposure.
494 Parameters
495 ----------
496 measCat : `lsst.afw.table.SourceCatalog`
497 The measurement catalog, based on the sources listed in the
498 reference catalog.
499 exposure : `lsst.afw.image.Exposure`
500 The measurement image upon which to perform forced detection.
501 refCat : `lsst.afw.table.SourceCatalog`
502 The reference catalog of sources to measure.
503 refWcs : `lsst.afw.image.SkyWcs`
504 The WCS for the references.
505 exposureId : `int`
506 Optional unique exposureId used for random seed in measurement
507 task.
509 Returns
510 -------
511 result : `lsst.pipe.base.Struct`
512 Structure with fields:
514 ``measCat``
515 Catalog of forced measurement results
516 (`lsst.afw.table.SourceCatalog`).
517 """
518 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
519 if self.config.doApCorr:
520 self.applyApCorr.run(
521 catalog=measCat,
522 apCorrMap=exposure.getInfo().getApCorrMap()
523 )
524 self.catalogCalculation.run(measCat)
526 return pipeBase.Struct(measCat=measCat)
528 def makeIdFactory(self, dataRef):
529 """Create an object that generates globally unique source IDs.
531 Source IDs are created based on a per-CCD ID and the ID of the CCD
532 itself.
534 Parameters
535 ----------
536 dataRef : `lsst.daf.persistence.ButlerDataRef`
537 Butler data reference. The ``ccdExposureId_bits`` and
538 ``ccdExposureId`` datasets are accessed. The data ID must have the
539 keys that correspond to ``ccdExposureId``, which are generally the
540 same as those that correspond to ``calexp`` (``visit``, ``raft``,
541 ``sensor`` for LSST data).
542 """
543 expBits = dataRef.get("ccdExposureId_bits")
544 expId = int(dataRef.get("ccdExposureId"))
545 return lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits)
547 def getExposureId(self, dataRef):
548 return int(dataRef.get("ccdExposureId", immediate=True))
550 def fetchReferences(self, dataRef, exposure):
551 """Get sources that overlap the exposure.
553 Parameters
554 ----------
555 dataRef : `lsst.daf.persistence.ButlerDataRef`
556 Butler data reference corresponding to the image to be measured;
557 should have ``tract``, ``patch``, and ``filter`` keys.
558 exposure : `lsst.afw.image.Exposure`
559 The image to be measured (used only to obtain a WCS and bounding
560 box).
562 Returns
563 -------
564 referencs : `lsst.afw.table.SourceCatalog`
565 Catalog of sources that overlap the exposure
567 Notes
568 -----
569 The returned catalog is sorted by ID and guarantees that all included
570 children have their parent included and that all Footprints are valid.
572 All work is delegated to the references subtask; see
573 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask`
574 for information about the default behavior.
575 """
576 references = lsst.afw.table.SourceCatalog(self.references.schema)
577 badParents = set()
578 unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs())
579 for record in unfiltered:
580 if record.getFootprint() is None or record.getFootprint().getArea() == 0:
581 if record.getParent() != 0:
582 self.log.warning("Skipping reference %s (child of %s) with bad Footprint",
583 record.getId(), record.getParent())
584 else:
585 self.log.warning("Skipping reference parent %s with bad Footprint", record.getId())
586 badParents.add(record.getId())
587 elif record.getParent() not in badParents:
588 references.append(record)
589 # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work
590 references.sort(lsst.afw.table.SourceTable.getParentKey())
591 return references
593 def attachFootprints(self, sources, refCat, exposure, refWcs):
594 r"""Attach footprints to blank sources prior to measurements.
596 Notes
597 -----
598 `~lsst.afw.detection.Footprint`\ s for forced photometry must be in the
599 pixel coordinate system of the image being measured, while the actual
600 detections may start out in a different coordinate system.
602 Subclasses of this class must implement this method to define how
603 those `~lsst.afw.detection.Footprint`\ s should be generated.
605 This default implementation transforms the
606 `~lsst.afw.detection.Footprint`\ s from the reference catalog from the
607 reference WCS to the exposure's WcS, which downgrades
608 `lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s into regular
609 `~lsst.afw.detection.Footprint`\ s, destroying deblend information.
610 """
611 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs)
613 def getExposure(self, dataRef):
614 """Read input exposure for measurement.
616 Parameters
617 ----------
618 dataRef : `lsst.daf.persistence.ButlerDataRef`
619 Butler data reference.
620 """
621 exposure = dataRef.get(self.dataPrefix + "calexp", immediate=True)
623 if self.config.doApplyExternalPhotoCalib:
624 source = f"{self.config.externalPhotoCalibName}_photoCalib"
625 self.log.info("Applying external photoCalib from %s", source)
626 photoCalib = dataRef.get(source)
627 exposure.setPhotoCalib(photoCalib) # No need for calibrateImage; having the photoCalib suffices
629 if self.config.doApplyExternalSkyWcs:
630 source = f"{self.config.externalSkyWcsName}_wcs"
631 self.log.info("Applying external skyWcs from %s", source)
632 skyWcs = dataRef.get(source)
633 exposure.setWcs(skyWcs)
635 if self.config.doApplySkyCorr:
636 self.log.info("Apply sky correction")
637 skyCorr = dataRef.get("skyCorr")
638 exposure.maskedImage -= skyCorr.getImage()
640 return exposure
642 def writeOutput(self, dataRef, sources):
643 """Write forced source table
645 Parameters
646 ----------
647 dataRef : `lsst.daf.persistence.ButlerDataRef`
648 Butler data reference. The forced_src dataset (with
649 self.dataPrefix prepended) is all that will be modified.
650 sources : `lsst.afw.table.SourceCatalog`
651 Catalog of sources to save.
652 """
653 dataRef.put(sources, self.dataPrefix + "forced_src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
655 def getSchemaCatalogs(self):
656 """The schema catalogs that will be used by this task.
658 Returns
659 -------
660 schemaCatalogs : `dict`
661 Dictionary mapping dataset type to schema catalog.
663 Notes
664 -----
665 There is only one schema for each type of forced measurement. The
666 dataset type for this measurement is defined in the mapper.
667 """
668 catalog = lsst.afw.table.SourceCatalog(self.measurement.schema)
669 catalog.getTable().setMetadata(self.measurement.algMetadata)
670 datasetType = self.dataPrefix + "forced_src"
671 return {datasetType: catalog}
673 def _getConfigName(self):
674 # Documented in superclass.
675 return self.dataPrefix + "forcedPhotCcd_config"
677 def _getMetadataName(self):
678 # Documented in superclass
679 return self.dataPrefix + "forcedPhotCcd_metadata"
681 @classmethod
682 def _makeArgumentParser(cls):
683 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
684 parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], "
685 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
686 ContainerClass=PerTractCcdDataIdContainer)
687 return parser
690class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections,
691 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
692 defaultTemplates={"inputCoaddName": "goodSeeing",
693 "inputName": "calexp"}):
694 refCat = cT.Input(
695 doc="Catalog of positions at which to force photometry.",
696 name="{inputCoaddName}Diff_fullDiaObjTable",
697 storageClass="DataFrame",
698 dimensions=["skymap", "tract", "patch"],
699 multiple=True,
700 deferLoad=True,
701 )
702 exposure = cT.Input(
703 doc="Input exposure to perform photometry on.",
704 name="{inputName}",
705 storageClass="ExposureF",
706 dimensions=["instrument", "visit", "detector"],
707 )
708 measCat = cT.Output(
709 doc="Output forced photometry catalog.",
710 name="forced_src_diaObject",
711 storageClass="SourceCatalog",
712 dimensions=["instrument", "visit", "detector", "skymap", "tract"],
713 )
714 outputSchema = cT.InitOutput(
715 doc="Schema for the output forced measurement catalogs.",
716 name="forced_src_diaObject_schema",
717 storageClass="SourceCatalog",
718 )
721class ForcedPhotCcdFromDataFrameConfig(ForcedPhotCcdConfig,
722 pipelineConnections=ForcedPhotCcdFromDataFrameConnections):
723 def setDefaults(self):
724 self.measurement.doReplaceWithNoise = False
725 self.measurement.plugins = ["base_TransformedCentroidFromCoord", "base_PsfFlux"]
726 self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'}
727 self.measurement.slots.centroid = "base_TransformedCentroidFromCoord"
728 self.measurement.slots.psfFlux = "base_PsfFlux"
729 self.measurement.slots.shape = None
730 self.catalogCalculation.plugins.names = []
733class ForcedPhotCcdFromDataFrameTask(ForcedPhotCcdTask):
734 """Force Photometry on a per-detector exposure with coords from a DataFrame
736 Uses input from a DataFrame instead of SourceCatalog
737 like the base class ForcedPhotCcd does.
738 Writes out a SourceCatalog so that the downstream WriteForcedSourceTableTask
739 can be reused with output from this Task.
740 """
741 _DefaultName = "forcedPhotCcdFromDataFrame"
742 ConfigClass = ForcedPhotCcdFromDataFrameConfig
744 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
745 # Parent's init assumes that we have a reference schema; Cannot reuse
746 pipeBase.PipelineTask.__init__(self, **kwds)
748 self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema())
750 if self.config.doApCorr:
751 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
752 self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
753 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
755 def runQuantum(self, butlerQC, inputRefs, outputRefs):
756 inputs = butlerQC.get(inputRefs)
757 self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']]))
758 refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'decl']})
759 for i in inputs['refCat']],
760 inputs['exposure'].getBBox(), inputs['exposure'].getWcs())
761 inputs['refCat'] = refCat
762 inputs['refWcs'] = inputs['exposure'].getWcs()
763 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
764 inputs['exposure'], inputs['refCat'],
765 inputs['refWcs'],
766 "visit_detector")
767 outputs = self.run(**inputs)
768 butlerQC.put(outputs, outputRefs)
770 def df2RefCat(self, dfList, exposureBBox, exposureWcs):
771 """Convert list of DataFrames to reference catalog
773 Concatenate list of DataFrames presumably from multiple patches and
774 downselect rows that overlap the exposureBBox using the exposureWcs.
776 Parameters
777 ----------
778 dfList : `list` of `pandas.DataFrame`
779 Each element containst diaObjects with ra/decl position in degrees
780 Columns 'diaObjectId', 'ra', 'decl' are expected
781 exposureBBox : `lsst.geom.Box2I`
782 Bounding box on which to select rows that overlap
783 exposureWcs : `lsst.afw.geom.SkyWcs`
784 World coordinate system to convert sky coords in ref cat to
785 pixel coords with which to compare with exposureBBox
787 Returns
788 -------
789 refCat : `lsst.afw.table.SourceTable`
790 Source Catalog with minimal schema that overlaps exposureBBox
791 """
792 df = pd.concat(dfList)
793 # translate ra/decl coords in dataframe to detector pixel coords
794 # to down select rows that overlap the detector bbox
795 mapping = exposureWcs.getTransform().getMapping()
796 x, y = mapping.applyInverse(np.array(df[['ra', 'decl']].values*2*np.pi/360).T)
797 inBBox = exposureBBox.contains(x, y)
798 refCat = self.df2SourceCat(df[inBBox])
799 return refCat
801 def df2SourceCat(self, df):
802 """Create minimal schema SourceCatalog from a pandas DataFrame.
804 The forced measurement subtask expects this as input.
806 Parameters
807 ----------
808 df : `pandas.DataFrame`
809 DiaObjects with locations and ids.
811 Returns
812 -------
813 outputCatalog : `lsst.afw.table.SourceTable`
814 Output catalog with minimal schema.
815 """
816 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
817 outputCatalog = lsst.afw.table.SourceCatalog(schema)
818 outputCatalog.reserve(len(df))
820 for id, diaObjectId, ra, decl in df[['diaObjectId', 'ra', 'decl']].itertuples():
821 outputRecord = outputCatalog.addNew()
822 outputRecord.setId(diaObjectId)
823 outputRecord.setCoord(lsst.geom.SpherePoint(ra, decl, lsst.geom.degrees))
824 return outputCatalog