Coverage for python/lsst/meas/base/forcedPhotCcd.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import collections
24import lsst.pex.config
25import lsst.pex.exceptions
26from lsst.log import Log
27import lsst.pipe.base
28import lsst.geom
29import lsst.afw.geom
30import lsst.afw.image
31import lsst.afw.table
32import lsst.sphgeom
34from lsst.pipe.base import PipelineTaskConnections
35import lsst.pipe.base.connectionTypes as cT
36from .forcedPhotImage import ForcedPhotImageTask, ForcedPhotImageConfig
38try:
39 from lsst.meas.mosaic import applyMosaicResults
40except ImportError:
41 applyMosaicResults = None
43__all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask", "imageOverlapsTract")
46class PerTractCcdDataIdContainer(lsst.pipe.base.DataIdContainer):
47 """A data ID container which combines raw data IDs with a tract.
49 Notes
50 -----
51 Required because we need to add "tract" to the raw data ID keys (defined as
52 whatever we use for ``src``) when no tract is provided (so that the user is
53 not required to know which tracts are spanned by the raw data ID).
55 This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is
56 being measured using the detection information, a set of reference
57 catalogs, from the set of coadds which intersect with the calexp. It needs
58 the calexp id (e.g. visit, raft, sensor), but is also uses the tract to
59 decide what set of coadds to use. The references from the tract whose
60 patches intersect with the calexp are used.
61 """
63 def makeDataRefList(self, namespace):
64 """Make self.refList from self.idList
65 """
66 if self.datasetType is None:
67 raise RuntimeError("Must call setDatasetType first")
68 log = Log.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer")
69 skymap = None
70 visitTract = collections.defaultdict(set) # Set of tracts for each visit
71 visitRefs = collections.defaultdict(list) # List of data references for each visit
72 for dataId in self.idList:
73 if "tract" not in dataId:
74 # Discover which tracts the data overlaps
75 log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId))
76 if skymap is None:
77 skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap")
79 for ref in namespace.butler.subset("calexp", dataId=dataId):
80 if not ref.datasetExists("calexp"):
81 continue
83 visit = ref.dataId["visit"]
84 visitRefs[visit].append(ref)
86 md = ref.get("calexp_md", immediate=True)
87 wcs = lsst.afw.geom.makeSkyWcs(md)
88 box = lsst.geom.Box2D(lsst.afw.image.bboxFromMetadata(md))
89 # Going with just the nearest tract. Since we're throwing all tracts for the visit
90 # together, this shouldn't be a problem unless the tracts are much smaller than a CCD.
91 tract = skymap.findTract(wcs.pixelToSky(box.getCenter()))
92 if imageOverlapsTract(tract, wcs, box):
93 visitTract[visit].add(tract.getId())
94 else:
95 self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId))
97 # Ensure all components of a visit are kept together by putting them all in the same set of tracts
98 for visit, tractSet in visitTract.items():
99 for ref in visitRefs[visit]:
100 for tract in tractSet:
101 self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType,
102 dataId=ref.dataId, tract=tract))
103 if visitTract:
104 tractCounter = collections.Counter()
105 for tractSet in visitTract.values():
106 tractCounter.update(tractSet)
107 log.info("Number of visits for each tract: %s", dict(tractCounter))
110def imageOverlapsTract(tract, imageWcs, imageBox):
111 """Return whether the given bounding box overlaps the tract given a WCS.
113 Parameters
114 ----------
115 tract : `lsst.skymap.TractInfo`
116 TractInfo specifying a tract.
117 imageWcs : `lsst.afw.geom.SkyWcs`
118 World coordinate system for the image.
119 imageBox : `lsst.geom.Box2I`
120 Bounding box for the image.
122 Returns
123 -------
124 overlap : `bool`
125 `True` if the bounding box overlaps the tract; `False` otherwise.
126 """
127 tractPoly = tract.getOuterSkyPolygon()
129 imagePixelCorners = lsst.geom.Box2D(imageBox).getCorners()
130 try:
131 imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners)
132 except lsst.pex.exceptions.LsstCppException as e:
133 # Protecting ourselves from awful Wcs solutions in input images
134 if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException)
135 and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)):
136 raise
137 return False
139 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageSkyCorners])
140 return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by"
143class ForcedPhotCcdConnections(PipelineTaskConnections,
144 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
145 defaultTemplates={"inputCoaddName": "deep",
146 "inputName": "calexp"}):
147 inputSchema = cT.InitInput(
148 doc="Schema for the input measurement catalogs.",
149 name="{inputCoaddName}Coadd_ref_schema",
150 storageClass="SourceCatalog",
151 )
152 outputSchema = cT.InitOutput(
153 doc="Schema for the output forced measurement catalogs.",
154 name="forced_src_schema",
155 storageClass="SourceCatalog",
156 )
157 exposure = cT.Input(
158 doc="Input exposure to perform photometry on.",
159 name="{inputName}",
160 storageClass="ExposureF",
161 dimensions=["instrument", "visit", "detector"],
162 )
163 refCat = cT.Input(
164 doc="Catalog of shapes and positions at which to force photometry.",
165 name="{inputCoaddName}Coadd_ref",
166 storageClass="SourceCatalog",
167 dimensions=["skymap", "tract", "patch"],
168 )
169 refWcs = cT.Input(
170 doc="Reference world coordinate system.",
171 name="{inputCoaddName}Coadd.wcs",
172 storageClass="Wcs",
173 dimensions=["band", "skymap", "tract", "patch"],
174 )
175 measCat = cT.Output(
176 doc="Output forced photometry catalog.",
177 name="forced_src",
178 storageClass="SourceCatalog",
179 dimensions=["instrument", "visit", "detector", "skymap", "tract"],
180 )
183class ForcedPhotCcdConfig(ForcedPhotImageConfig, pipelineConnections=ForcedPhotCcdConnections):
184 doApplyUberCal = lsst.pex.config.Field(
185 dtype=bool,
186 doc="Apply meas_mosaic ubercal results to input calexps?",
187 default=False,
188 deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead",
189 )
190 doApplyExternalPhotoCalib = lsst.pex.config.Field(
191 dtype=bool,
192 default=False,
193 doc=("Whether to apply external photometric calibration via an "
194 "`lsst.afw.image.PhotoCalib` object. Uses the "
195 "``externalPhotoCalibName`` field to determine which calibration "
196 "to load."),
197 )
198 doApplyExternalSkyWcs = lsst.pex.config.Field(
199 dtype=bool,
200 default=False,
201 doc=("Whether to apply external astrometric calibration via an "
202 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` "
203 "field to determine which calibration to load."),
204 )
205 doApplySkyCorr = lsst.pex.config.Field(
206 dtype=bool,
207 default=False,
208 doc="Apply sky correction?",
209 )
210 includePhotoCalibVar = lsst.pex.config.Field(
211 dtype=bool,
212 default=False,
213 doc="Add photometric calibration variance to warp variance plane?",
214 )
215 externalPhotoCalibName = lsst.pex.config.ChoiceField(
216 dtype=str,
217 doc=("Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. "
218 "Unused for Gen3 middleware."),
219 default="jointcal",
220 allowed={
221 "jointcal": "Use jointcal_photoCalib",
222 "fgcm": "Use fgcm_photoCalib",
223 "fgcm_tract": "Use fgcm_tract_photoCalib"
224 },
225 )
226 externalSkyWcsName = lsst.pex.config.ChoiceField(
227 dtype=str,
228 doc="Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.",
229 default="jointcal",
230 allowed={
231 "jointcal": "Use jointcal_wcs"
232 },
233 )
236class ForcedPhotCcdTask(ForcedPhotImageTask):
237 """A command-line driver for performing forced measurement on CCD images.
239 Notes
240 -----
241 This task is a subclass of
242 :lsst-task:`lsst.meas.base.forcedPhotImage.ForcedPhotImageTask` which is
243 specifically for doing forced measurement on a single CCD exposure, using
244 as a reference catalog the detections which were made on overlapping
245 coadds.
247 The `run` method (inherited from `ForcedPhotImageTask`) takes a
248 `~lsst.daf.persistence.ButlerDataRef` argument that corresponds to a single
249 CCD. This should contain the data ID keys that correspond to the
250 ``forced_src`` dataset (the output dataset for this task), which are
251 typically all those used to specify the ``calexp`` dataset (``visit``,
252 ``raft``, ``sensor`` for LSST data) as well as a coadd tract. The tract is
253 used to look up the appropriate coadd measurement catalogs to use as
254 references (e.g. ``deepCoadd_src``; see
255 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more
256 information). While the tract must be given as part of the dataRef, the
257 patches are determined automatically from the bounding box and WCS of the
258 calexp to be measured, and the filter used to fetch references is set via
259 the ``filter`` option in the configuration of
260 :lsst-task:`lsst.meas.base.references.BaseReferencesTask`).
262 In addition to the `run` method, `ForcedPhotCcdTask` overrides several
263 methods of `ForcedPhotImageTask` to specialize it for single-CCD
264 processing, including `~ForcedPhotImageTask.makeIdFactory`,
265 `~ForcedPhotImageTask.fetchReferences`, and
266 `~ForcedPhotImageTask.getExposure`. None of these should be called
267 directly by the user, though it may be useful to override them further in
268 subclasses.
269 """
271 ConfigClass = ForcedPhotCcdConfig
272 RunnerClass = lsst.pipe.base.ButlerInitializedTaskRunner
273 _DefaultName = "forcedPhotCcd"
274 dataPrefix = ""
276 def runQuantum(self, butlerQC, inputRefs, outputRefs):
277 inputs = butlerQC.get(inputRefs)
278 inputs['refCat'] = self.filterReferences(inputs['exposure'], inputs['refCat'], inputs['refWcs'])
279 inputs['measCat'] = self.generateMeasCat(inputRefs.exposure.dataId,
280 inputs['exposure'],
281 inputs['refCat'], inputs['refWcs'],
282 "visit_detector")
283 # TODO: apply external calibrations (DM-17062)
284 outputs = self.run(**inputs)
285 butlerQC.put(outputs, outputRefs)
287 def filterReferences(self, exposure, refCat, refWcs):
288 """Filter reference catalog so that all sources are within the
289 boundaries of the exposure.
291 Parameters
292 ----------
293 exposure : `lsst.afw.image.exposure.Exposure`
294 Exposure to generate the catalog for.
295 refCat : `lsst.afw.table.SourceCatalog`
296 Catalog of shapes and positions at which to force photometry.
297 refWcs : `lsst.afw.image.SkyWcs`
298 Reference world coordinate system.
300 Returns
301 -------
302 refSources : `lsst.afw.table.SourceCatalog`
303 Filtered catalog of forced sources to measure.
305 Notes
306 -----
307 Filtering the reference catalog is currently handled by Gen2
308 specific methods. To function for Gen3, this method copies
309 code segments to do the filtering and transformation. The
310 majority of this code is based on the methods of
311 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader
313 """
315 # Step 1: Determine bounds of the exposure photometry will
316 # be performed on.
317 expWcs = exposure.getWcs()
318 expRegion = exposure.getBBox(lsst.afw.image.PARENT)
319 expBBox = lsst.geom.Box2D(expRegion)
320 expBoxCorners = expBBox.getCorners()
321 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for
322 corner in expBoxCorners]
323 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners)
325 # Step 2: Filter out reference catalog sources that are
326 # not contained within the exposure boundaries.
327 sources = type(refCat)(refCat.table)
328 for record in refCat:
329 if expPolygon.contains(record.getCoord().getVector()):
330 sources.append(record)
331 refCatIdDict = {ref.getId(): ref.getParent() for ref in sources}
333 # Step 3: Cull sources that do not have their parent
334 # source in the filtered catalog. Save two copies of each
335 # source.
336 refSources = type(refCat)(refCat.table)
337 for record in refCat:
338 if expPolygon.contains(record.getCoord().getVector()):
339 recordId = record.getId()
340 topId = recordId
341 while (topId > 0):
342 if topId in refCatIdDict:
343 topId = refCatIdDict[topId]
344 else:
345 break
346 if topId == 0:
347 refSources.append(record)
349 # Step 4: Transform source footprints from the reference
350 # coordinates to the exposure coordinates.
351 for refRecord in refSources:
352 refRecord.setFootprint(refRecord.getFootprint().transform(refWcs,
353 expWcs, expRegion))
354 # Step 5: Replace reference catalog with filtered source list.
355 return refSources
357 def makeIdFactory(self, dataRef):
358 """Create an object that generates globally unique source IDs.
360 Source IDs are created based on a per-CCD ID and the ID of the CCD
361 itself.
363 Parameters
364 ----------
365 dataRef : `lsst.daf.persistence.ButlerDataRef`
366 Butler data reference. The ``ccdExposureId_bits`` and
367 ``ccdExposureId`` datasets are accessed. The data ID must have the
368 keys that correspond to ``ccdExposureId``, which are generally the
369 same as those that correspond to ``calexp`` (``visit``, ``raft``,
370 ``sensor`` for LSST data).
371 """
372 expBits = dataRef.get("ccdExposureId_bits")
373 expId = int(dataRef.get("ccdExposureId"))
374 return lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits)
376 def getExposureId(self, dataRef):
377 return int(dataRef.get("ccdExposureId", immediate=True))
379 def fetchReferences(self, dataRef, exposure):
380 """Get sources that overlap the exposure.
382 Parameters
383 ----------
384 dataRef : `lsst.daf.persistence.ButlerDataRef`
385 Butler data reference corresponding to the image to be measured;
386 should have ``tract``, ``patch``, and ``filter`` keys.
387 exposure : `lsst.afw.image.Exposure`
388 The image to be measured (used only to obtain a WCS and bounding
389 box).
391 Returns
392 -------
393 referencs : `lsst.afw.table.SourceCatalog`
394 Catalog of sources that overlap the exposure
396 Notes
397 -----
398 The returned catalog is sorted by ID and guarantees that all included
399 children have their parent included and that all Footprints are valid.
401 All work is delegated to the references subtask; see
402 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask`
403 for information about the default behavior.
404 """
405 references = lsst.afw.table.SourceCatalog(self.references.schema)
406 badParents = set()
407 unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs())
408 for record in unfiltered:
409 if record.getFootprint() is None or record.getFootprint().getArea() == 0:
410 if record.getParent() != 0:
411 self.log.warn("Skipping reference %s (child of %s) with bad Footprint",
412 record.getId(), record.getParent())
413 else:
414 self.log.warn("Skipping reference parent %s with bad Footprint", record.getId())
415 badParents.add(record.getId())
416 elif record.getParent() not in badParents:
417 references.append(record)
418 # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work
419 references.sort(lsst.afw.table.SourceTable.getParentKey())
420 return references
422 def getExposure(self, dataRef):
423 """Read input exposure for measurement.
425 Parameters
426 ----------
427 dataRef : `lsst.daf.persistence.ButlerDataRef`
428 Butler data reference.
429 """
430 exposure = ForcedPhotImageTask.getExposure(self, dataRef)
432 if self.config.doApplyExternalPhotoCalib:
433 source = f"{self.config.externalPhotoCalibName}_photoCalib"
434 self.log.info("Applying external photoCalib from %s", source)
435 photoCalib = dataRef.get(source)
436 exposure.setPhotoCalib(photoCalib) # No need for calibrateImage; having the photoCalib suffices
438 if self.config.doApplyExternalSkyWcs:
439 source = f"{self.config.externalSkyWcsName}_wcs"
440 self.log.info("Applying external skyWcs from %s", source)
441 skyWcs = dataRef.get(source)
442 exposure.setWcs(skyWcs)
444 if self.config.doApplySkyCorr:
445 self.log.info("Apply sky correction")
446 skyCorr = dataRef.get("skyCorr")
447 exposure.maskedImage -= skyCorr.getImage()
449 return exposure
451 def _getConfigName(self):
452 # Documented in superclass.
453 return self.dataPrefix + "forcedPhotCcd_config"
455 def _getMetadataName(self):
456 # Documented in superclass
457 return self.dataPrefix + "forcedPhotCcd_metadata"
459 @classmethod
460 def _makeArgumentParser(cls):
461 parser = lsst.pipe.base.ArgumentParser(name=cls._DefaultName)
462 parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], "
463 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
464 ContainerClass=PerTractCcdDataIdContainer)
465 return parser