Coverage for python/lsst/fgcmcal/fgcmBuildStars.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM.
25This task finds all the visits and calexps in a repository (or a subset
26based on command line parameters) and extract all the potential calibration
27stars for input into fgcm. This task additionally uses fgcm to match
28star observations into unique stars, and performs as much cleaning of
29the input catalog as possible.
30"""
32import os
33import sys
34import time
35import traceback
37import numpy as np
39import lsst.pex.config as pexConfig
40import lsst.pipe.base as pipeBase
41import lsst.afw.table as afwTable
42import lsst.geom as geom
43from lsst.daf.base import PropertyList
44from lsst.daf.base.dateTime import DateTime
45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
47from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
48from .utilities import computeApproxPixelAreaFields, computeApertureRadius
50import fgcm
52REFSTARS_FORMAT_VERSION = 1
54__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask', 'FgcmBuildStarsRunner']
57class FgcmBuildStarsConfig(pexConfig.Config):
58 """Config for FgcmBuildStarsTask"""
60 instFluxField = pexConfig.Field(
61 doc=("Name of the source instFlux field to use. The associated flag field "
62 "('<name>_flag') will be implicitly included in badFlags"),
63 dtype=str,
64 default='slot_CalibFlux_instFlux',
65 )
66 minPerBand = pexConfig.Field(
67 doc="Minimum observations per band",
68 dtype=int,
69 default=2,
70 )
71 matchRadius = pexConfig.Field(
72 doc="Match radius (arcseconds)",
73 dtype=float,
74 default=1.0,
75 )
76 isolationRadius = pexConfig.Field(
77 doc="Isolation radius (arcseconds)",
78 dtype=float,
79 default=2.0,
80 )
81 densityCutNside = pexConfig.Field(
82 doc="Density cut healpix nside",
83 dtype=int,
84 default=128,
85 )
86 densityCutMaxPerPixel = pexConfig.Field(
87 doc="Density cut number of stars per pixel",
88 dtype=int,
89 default=1000,
90 )
91 matchNside = pexConfig.Field(
92 doc="Healpix Nside for matching",
93 dtype=int,
94 default=4096,
95 )
96 coarseNside = pexConfig.Field(
97 doc="Healpix coarse Nside for partitioning matches",
98 dtype=int,
99 default=8,
100 )
101 filterMap = pexConfig.DictField(
102 doc="Mapping from 'filterName' to band.",
103 keytype=str,
104 itemtype=str,
105 default={},
106 )
107 requiredBands = pexConfig.ListField(
108 doc="Bands required for each star",
109 dtype=str,
110 default=(),
111 )
112 primaryBands = pexConfig.ListField(
113 doc=("Bands for 'primary' star matches. "
114 "A star must be observed in one of these bands to be considered "
115 "as a calibration star."),
116 dtype=str,
117 default=None
118 )
119 referenceCCD = pexConfig.Field(
120 doc="Reference CCD for scanning visits",
121 dtype=int,
122 default=13,
123 )
124 checkAllCcds = pexConfig.Field(
125 doc=("Check repo for all CCDs for each visit specified. To be used when the "
126 "full set of ids (visit/ccd) are not specified on the command line. For "
127 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
128 "faster than the alternatives."),
129 dtype=bool,
130 default=True,
131 )
132 visitDataRefName = pexConfig.Field(
133 doc="dataRef name for the 'visit' field",
134 dtype=str,
135 default="visit"
136 )
137 ccdDataRefName = pexConfig.Field(
138 doc="dataRef name for the 'ccd' field",
139 dtype=str,
140 default="ccd"
141 )
142 applyJacobian = pexConfig.Field(
143 doc="Apply Jacobian correction?",
144 dtype=bool,
145 deprecated=("This field is no longer used, and has been deprecated by DM-20163. "
146 "It will be removed after v20."),
147 default=False
148 )
149 jacobianName = pexConfig.Field(
150 doc="Name of field with jacobian correction",
151 dtype=str,
152 deprecated=("This field is no longer used, and has been deprecated by DM-20163. "
153 "It will be removed after v20."),
154 default="base_Jacobian_value"
155 )
156 doApplyWcsJacobian = pexConfig.Field(
157 doc="Apply the jacobian of the WCS to the star observations prior to fit?",
158 dtype=bool,
159 default=True
160 )
161 psfCandidateName = pexConfig.Field(
162 doc="Name of field with psf candidate flag for propagation",
163 dtype=str,
164 default="calib_psf_candidate"
165 )
166 doSubtractLocalBackground = pexConfig.Field(
167 doc=("Subtract the local background before performing calibration? "
168 "This is only supported for circular aperture calibration fluxes."),
169 dtype=bool,
170 default=False
171 )
172 localBackgroundFluxField = pexConfig.Field(
173 doc="Name of the local background instFlux field to use.",
174 dtype=str,
175 default='base_LocalBackground_instFlux'
176 )
177 sourceSelector = sourceSelectorRegistry.makeField(
178 doc="How to select sources",
179 default="science"
180 )
181 apertureInnerInstFluxField = pexConfig.Field(
182 doc="Field that contains inner aperture for aperture correction proxy",
183 dtype=str,
184 default='base_CircularApertureFlux_12_0_instFlux'
185 )
186 apertureOuterInstFluxField = pexConfig.Field(
187 doc="Field that contains outer aperture for aperture correction proxy",
188 dtype=str,
189 default='base_CircularApertureFlux_17_0_instFlux'
190 )
191 doReferenceMatches = pexConfig.Field(
192 doc="Match reference catalog as additional constraint on calibration",
193 dtype=bool,
194 default=True,
195 )
196 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
197 target=FgcmLoadReferenceCatalogTask,
198 doc="FGCM reference object loader",
199 )
200 nVisitsPerCheckpoint = pexConfig.Field(
201 doc="Number of visits read between checkpoints",
202 dtype=int,
203 default=500,
204 )
206 def setDefaults(self):
207 sourceSelector = self.sourceSelector["science"]
208 sourceSelector.setDefaults()
210 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
212 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge',
213 'base_PixelFlags_flag_interpolatedCenter',
214 'base_PixelFlags_flag_saturatedCenter',
215 'base_PixelFlags_flag_crCenter',
216 'base_PixelFlags_flag_bad',
217 'base_PixelFlags_flag_interpolated',
218 'base_PixelFlags_flag_saturated',
219 'slot_Centroid_flag',
220 fluxFlagName]
222 if self.doSubtractLocalBackground:
223 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
224 sourceSelector.flags.bad.append(localBackgroundFlagName)
226 sourceSelector.doFlags = True
227 sourceSelector.doUnresolved = True
228 sourceSelector.doSignalToNoise = True
229 sourceSelector.doIsolated = True
231 sourceSelector.signalToNoise.fluxField = self.instFluxField
232 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
233 sourceSelector.signalToNoise.minimum = 10.0
234 sourceSelector.signalToNoise.maximum = 1000.0
236 # FGCM operates on unresolved sources, and this setting is
237 # appropriate for the current base_ClassificationExtendedness
238 sourceSelector.unresolved.maximum = 0.5
241class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
242 """Subclass of TaskRunner for fgcmBuildStarsTask
244 fgcmBuildStarsTask.run() takes a number of arguments, one of which is the
245 butler (for persistence and mapper data), and a list of dataRefs
246 extracted from the command line. Note that FGCM runs on a large set of
247 dataRefs, and not on single dataRef/tract/patch.
248 This class transforms the process arguments generated by the ArgumentParser
249 into the arguments expected by FgcmBuildStarsTask.run().
250 This runner does not use any parallelization.
252 """
254 @staticmethod
255 def getTargetList(parsedCmd):
256 """
257 Return a list with one element: a tuple with the butler and
258 list of dataRefs
259 """
260 # we want to combine the butler with any (or no!) dataRefs
261 return [(parsedCmd.butler, parsedCmd.id.refList)]
263 def __call__(self, args):
264 """
265 Parameters
266 ----------
267 args: `tuple` with (butler, dataRefList)
269 Returns
270 -------
271 exitStatus: `list` with `lsst.pipe.base.Struct`
272 exitStatus (0: success; 1: failure)
273 """
274 butler, dataRefList = args
276 task = self.TaskClass(config=self.config, log=self.log)
278 exitStatus = 0
279 if self.doRaise:
280 task.runDataRef(butler, dataRefList)
281 else:
282 try:
283 task.runDataRef(butler, dataRefList)
284 except Exception as e:
285 exitStatus = 1
286 task.log.fatal("Failed: %s" % e)
287 if not isinstance(e, pipeBase.TaskError):
288 traceback.print_exc(file=sys.stderr)
290 task.writeMetadata(butler)
292 # The task does not return any results:
293 return [pipeBase.Struct(exitStatus=exitStatus)]
295 def run(self, parsedCmd):
296 """
297 Run the task, with no multiprocessing
299 Parameters
300 ----------
301 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
302 """
304 resultList = []
306 if self.precall(parsedCmd):
307 targetList = self.getTargetList(parsedCmd)
308 resultList = self(targetList[0])
310 return resultList
313class FgcmBuildStarsTask(pipeBase.CmdLineTask):
314 """
315 Build stars for the FGCM global calibration
316 """
318 ConfigClass = FgcmBuildStarsConfig
319 RunnerClass = FgcmBuildStarsRunner
320 _DefaultName = "fgcmBuildStars"
322 def __init__(self, butler=None, **kwargs):
323 """
324 Instantiate an `FgcmBuildStarsTask`.
326 Parameters
327 ----------
328 butler : `lsst.daf.persistence.Butler`
329 """
331 pipeBase.CmdLineTask.__init__(self, **kwargs)
332 self.makeSubtask("sourceSelector")
333 # Only log warning and fatal errors from the sourceSelector
334 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
336 @classmethod
337 def _makeArgumentParser(cls):
338 """Create an argument parser"""
340 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
341 parser.add_id_argument("--id", "calexp", help="Data ID, e.g. --id visit=6789")
343 return parser
345 # no saving of metadata for now
346 def _getMetadataName(self):
347 return None
349 @pipeBase.timeMethod
350 def runDataRef(self, butler, dataRefs):
351 """
352 Cross-match and make star list for FGCM Input
354 Parameters
355 ----------
356 butler: `lsst.daf.persistence.Butler`
357 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
358 Data references for the input visits.
359 If this is an empty list, all visits with src catalogs in
360 the repository are used.
361 Only one individual dataRef from a visit need be specified
362 and the code will find the other source catalogs from
363 each visit.
365 Raises
366 ------
367 RuntimeErrror: Raised if `config.doReferenceMatches` is set and
368 an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
369 fails if the calibFlux is not a CircularAperture flux.
370 """
372 self.log.info("Running with %d dataRefs" % (len(dataRefs)))
374 if self.config.doReferenceMatches:
375 # Ensure that we have a LUT
376 if not butler.datasetExists('fgcmLookUpTable'):
377 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
378 # Compute aperture radius if necessary. This is useful to do now before
379 # any heavy lifting has happened (fail early).
380 calibFluxApertureRadius = None
381 if self.config.doSubtractLocalBackground:
382 sourceSchema = butler.get('src_schema').schema
383 try:
384 calibFluxApertureRadius = computeApertureRadius(sourceSchema,
385 self.config.instFluxField)
386 except (RuntimeError, LookupError):
387 raise RuntimeError("Could not determine aperture radius from %s. "
388 "Cannot use doSubtractLocalBackground." %
389 (self.config.instFluxField))
391 groupedDataRefs = self.findAndGroupDataRefs(butler, dataRefs)
393 camera = butler.get('camera')
395 # Make the visit catalog if necessary
396 # First check if the visit catalog is in the current path
397 visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
398 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
399 if os.path.exists(filename):
400 # This file exists and we should continue processing
401 inVisitCat = visitCatDataRef.get()
402 if len(inVisitCat) != len(groupedDataRefs):
403 raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
404 "number of visits. Cannot continue.")
405 else:
406 inVisitCat = None
408 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
409 visitCatDataRef=visitCatDataRef,
410 inVisitCat=inVisitCat)
412 # Persist the visitCat as a checkpoint file.
413 visitCatDataRef.put(visitCat)
415 starObsDataRef = butler.dataRef('fgcmStarObservations')
416 filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
417 if os.path.exists(filename):
418 inStarObsCat = starObsDataRef.get()
419 else:
420 inStarObsCat = None
422 rad = calibFluxApertureRadius
423 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
424 visitCat,
425 calibFluxApertureRadius=rad,
426 starObsDataRef=starObsDataRef,
427 visitCatDataRef=visitCatDataRef,
428 inStarObsCat=inStarObsCat)
429 visitCatDataRef.put(visitCat)
430 starObsDataRef.put(fgcmStarObservationCat)
432 # Always do the matching.
433 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(butler,
434 visitCat,
435 fgcmStarObservationCat)
437 # Persist catalogs via the butler
438 butler.put(fgcmStarIdCat, 'fgcmStarIds')
439 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
440 if fgcmRefCat is not None:
441 butler.put(fgcmRefCat, 'fgcmReferenceStars')
443 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs,
444 visitCatDataRef=None, inVisitCat=None):
445 """
446 Make a visit catalog with all the keys from each visit
448 Parameters
449 ----------
450 camera: `lsst.afw.cameraGeom.Camera`
451 Camera from the butler
452 groupedDataRefs: `dict`
453 Dictionary with visit keys, and `list`s of
454 `lsst.daf.persistence.ButlerDataRef`
455 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
456 Dataref to write visitCat for checkpoints
457 inVisitCat: `afw.table.BaseCatalog`
458 Input (possibly incomplete) visit catalog
460 Returns
461 -------
462 visitCat: `afw.table.BaseCatalog`
463 """
465 self.log.info("Assembling visitCatalog from %d %ss" %
466 (len(groupedDataRefs), self.config.visitDataRefName))
468 nCcd = len(camera)
470 if inVisitCat is None:
471 schema = self._makeFgcmVisitSchema(nCcd)
473 visitCat = afwTable.BaseCatalog(schema)
474 visitCat.reserve(len(groupedDataRefs))
476 for i, visit in enumerate(sorted(groupedDataRefs)):
477 rec = visitCat.addNew()
478 rec['visit'] = visit
479 rec['used'] = 0
480 rec['sources_read'] = 0
481 else:
482 visitCat = inVisitCat
484 # No matter what, fill the catalog. This will check if it was
485 # already read.
486 self._fillVisitCatalog(visitCat, groupedDataRefs,
487 visitCatDataRef=visitCatDataRef)
489 return visitCat
491 def _fillVisitCatalog(self, visitCat, groupedDataRefs,
492 visitCatDataRef=None):
493 """
494 Fill the visit catalog with visit metadata
496 Parameters
497 ----------
498 visitCat: `afw.table.BaseCatalog`
499 Catalog with schema from _makeFgcmVisitSchema()
500 groupedDataRefs: `dict`
501 Dictionary with visit keys, and `list`s of
502 `lsst.daf.persistence.ButlerDataRef
503 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
504 Dataref to write visitCat for checkpoints
505 """
507 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
509 for i, visit in enumerate(sorted(groupedDataRefs)):
510 # We don't use the bypasses since we need the psf info which does
511 # not have a bypass
512 # TODO: When DM-15500 is implemented in the Gen3 Butler, this
513 # can be fixed
515 # Do not read those that have already been read
516 if visitCat['used'][i]:
517 continue
519 if (i % self.config.nVisitsPerCheckpoint) == 0:
520 self.log.info("Retrieving metadata for %s %d (%d/%d)" %
521 (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
522 # Save checkpoint if desired
523 if visitCatDataRef is not None:
524 visitCatDataRef.put(visitCat)
526 # Note that the reference ccd is first in the list (if available).
528 # The first dataRef in the group will be the reference ccd (if available)
529 dataRef = groupedDataRefs[visit][0]
531 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox,
532 flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
534 visitInfo = exp.getInfo().getVisitInfo()
535 f = exp.getFilter()
536 psf = exp.getPsf()
538 rec = visitCat[i]
539 rec['visit'] = visit
540 rec['filtername'] = f.getName()
541 radec = visitInfo.getBoresightRaDec()
542 rec['telra'] = radec.getRa().asDegrees()
543 rec['teldec'] = radec.getDec().asDegrees()
544 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
545 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
546 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
547 rec['exptime'] = visitInfo.getExposureTime()
548 # convert from Pa to millibar
549 # Note that I don't know if this unit will need to be per-camera config
550 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
551 # Flag to signify if this is a "deep" field. Not currently used
552 rec['deepFlag'] = 0
553 # Relative flat scaling (1.0 means no relative scaling)
554 rec['scaling'][:] = 1.0
555 # Median delta aperture, to be measured from stars
556 rec['deltaAper'] = 0.0
558 rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
560 if dataRef.datasetExists(datasetType='calexpBackground'):
561 # Get background for reference CCD
562 # This approximation is good enough for now
563 bgStats = (bg[0].getStatsImage().getImage().array
564 for bg in dataRef.get(datasetType='calexpBackground'))
565 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
566 else:
567 self.log.warn('Sky background not found for visit %d / ccd %d' %
568 (visit, dataRef.dataId[self.config.ccdDataRefName]))
569 rec['skyBackground'] = -1.0
571 rec['used'] = 1
573 def findAndGroupDataRefs(self, butler, dataRefs):
574 """
575 Find and group dataRefs (by visit). If dataRefs is an empty list,
576 this will look for all source catalogs in a given repo.
578 Parameters
579 ----------
580 butler: `lsst.daf.persistence.Butler`
581 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
582 Data references for the input visits.
583 If this is an empty list, all visits with src catalogs in
584 the repository are used.
586 Returns
587 -------
588 groupedDataRefs: `dict`
589 Dictionary with visit keys, and `list`s of `lsst.daf.persistence.ButlerDataRef`
590 """
592 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
594 camera = butler.get('camera')
596 ccdIds = []
597 for detector in camera:
598 ccdIds.append(detector.getId())
600 # TODO: related to DM-13730, this dance of looking for source visits
601 # will be unnecessary with Gen3 Butler. This should be part of
602 # DM-13730.
604 nVisits = 0
606 groupedDataRefs = {}
607 for dataRef in dataRefs:
608 visit = dataRef.dataId[self.config.visitDataRefName]
609 # If we don't have the dataset, just continue
610 if not dataRef.datasetExists(datasetType='src'):
611 continue
612 # If we need to check all ccds, do it here
613 if self.config.checkAllCcds:
614 if visit in groupedDataRefs:
615 # We already have found this visit
616 continue
617 dataId = dataRef.dataId.copy()
618 # For each ccd we must check that a valid source catalog exists.
619 for ccdId in ccdIds:
620 dataId[self.config.ccdDataRefName] = ccdId
621 if butler.datasetExists('src', dataId=dataId):
622 goodDataRef = butler.dataRef('src', dataId=dataId)
623 if visit in groupedDataRefs:
624 if (goodDataRef.dataId[self.config.ccdDataRefName] not in
625 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
626 groupedDataRefs[visit].append(goodDataRef)
627 else:
628 # This is a new visit
629 nVisits += 1
630 groupedDataRefs[visit] = [goodDataRef]
631 else:
632 # We have already confirmed that the dataset exists, so no need
633 # to check here.
634 if visit in groupedDataRefs:
635 if (dataRef.dataId[self.config.ccdDataRefName] not in
636 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]):
637 groupedDataRefs[visit].append(dataRef)
638 else:
639 # This is a new visit
640 nVisits += 1
641 groupedDataRefs[visit] = [dataRef]
643 if (nVisits % 100) == 0 and nVisits > 0:
644 self.log.info("Found %d unique %ss..." % (nVisits,
645 self.config.visitDataRefName))
647 self.log.info("Found %d unique %ss total." % (nVisits,
648 self.config.visitDataRefName))
650 # Put them in ccd order, with the reference ccd first (if available)
651 def ccdSorter(dataRef):
652 ccdId = dataRef.dataId[self.config.ccdDataRefName]
653 if ccdId == self.config.referenceCCD:
654 return -100
655 else:
656 return ccdId
658 # If we did not check all ccds, put them in ccd order
659 if not self.config.checkAllCcds:
660 for visit in groupedDataRefs:
661 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
663 return groupedDataRefs
665 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
666 calibFluxApertureRadius=None,
667 visitCatDataRef=None,
668 starObsDataRef=None,
669 inStarObsCat=None):
670 """
671 Compile all good star observations from visits in visitCat. Checkpoint files
672 will be stored if both visitCatDataRef and starObsDataRef are not None.
674 Parameters
675 ----------
676 groupedDataRefs: `dict` of `list`s
677 Lists of `lsst.daf.persistence.ButlerDataRef`, grouped by visit.
678 visitCat: `afw.table.BaseCatalog`
679 Catalog with visit data for FGCM
680 calibFluxApertureRadius: `float`, optional
681 Aperture radius for calibration flux. Default is None.
682 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
683 Dataref to write visitCat for checkpoints
684 starObsDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
685 Dataref to write the star observation catalog for checkpoints.
686 inStarObsCat: `afw.table.BaseCatalog`
687 Input (possibly incomplete) observation catalog
689 Returns
690 -------
691 fgcmStarObservations: `afw.table.BaseCatalog`
692 Full catalog of good observations.
694 Raises
695 ------
696 RuntimeError: Raised if doSubtractLocalBackground is True and
697 calibFluxApertureRadius is not set.
698 """
699 startTime = time.time()
701 if (visitCatDataRef is not None and starObsDataRef is None or
702 visitCatDataRef is None and starObsDataRef is not None):
703 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
704 "no checkpoint files will be persisted.")
706 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
707 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
709 # create our source schema. Use the first valid dataRef
710 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
711 sourceSchema = dataRef.get('src_schema', immediate=True).schema
713 # Construct a mapping from ccd number to index
714 camera = dataRef.get('camera')
715 ccdMapping = {}
716 for ccdIndex, detector in enumerate(camera):
717 ccdMapping[detector.getId()] = ccdIndex
719 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
721 sourceMapper = self._makeSourceMapper(sourceSchema)
723 # We also have a temporary catalog that will accumulate aperture measurements
724 aperMapper = self._makeAperMapper(sourceSchema)
726 outputSchema = sourceMapper.getOutputSchema()
728 if inStarObsCat is not None:
729 fullCatalog = inStarObsCat
730 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
731 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
732 if not comp1 or not comp2:
733 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
734 else:
735 fullCatalog = afwTable.BaseCatalog(outputSchema)
737 # FGCM will provide relative calibration for the flux in config.instFluxField
739 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
740 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey()
741 visitKey = outputSchema['visit'].asKey()
742 ccdKey = outputSchema['ccd'].asKey()
743 instMagKey = outputSchema['instMag'].asKey()
744 instMagErrKey = outputSchema['instMagErr'].asKey()
746 # Prepare local background if desired
747 if self.config.doSubtractLocalBackground:
748 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
749 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
750 else:
751 localBackground = 0.0
753 aperOutputSchema = aperMapper.getOutputSchema()
755 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
756 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey()
757 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
758 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey()
759 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
760 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
761 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
762 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
764 k = 2.5 / np.log(10.)
766 # loop over visits
767 for ctr, visit in enumerate(visitCat):
768 if visit['sources_read']:
769 continue
771 expTime = visit['exptime']
773 nStarInVisit = 0
775 # Reset the aperture catalog (per visit)
776 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
778 for dataRef in groupedDataRefs[visit['visit']]:
780 ccdId = dataRef.dataId[self.config.ccdDataRefName]
782 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
784 # If we are subtracting the local background, then correct here
785 # before we do the s/n selection. This ensures we do not have
786 # bad stars after local background subtraction.
788 if self.config.doSubtractLocalBackground:
789 # At the moment we only adjust the flux and not the flux
790 # error by the background because the error on
791 # base_LocalBackground_instFlux is the rms error in the
792 # background annulus, not the error on the mean in the
793 # background estimate (which is much smaller, by sqrt(n)
794 # pixels used to estimate the background, which we do not
795 # have access to in this task). In the default settings,
796 # the annulus is sufficiently large such that these
797 # additional errors are are negligibly small (much less
798 # than a mmag in quadrature).
800 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
801 sources[instFluxKey] -= localBackground
803 goodSrc = self.sourceSelector.selectSources(sources)
805 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
806 tempCat.reserve(goodSrc.selected.sum())
807 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
808 tempCat[visitKey][:] = visit['visit']
809 tempCat[ccdKey][:] = ccdId
811 # Compute "instrumental magnitude" by scaling flux with exposure time.
812 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
813 visit['scaling'][ccdMapping[ccdId]])
814 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
816 # Compute instMagErr from instFluxErr / instFlux, any scaling
817 # will cancel out.
819 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] /
820 sources[instFluxKey][goodSrc.selected])
822 # Compute the jacobian from an approximate PixelAreaBoundedField
823 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'],
824 tempCat['y'])
826 # Apply the jacobian if configured
827 if self.config.doApplyWcsJacobian:
828 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
830 fullCatalog.extend(tempCat)
832 # And the aperture information
833 # This does not need the jacobian because it is all locally relative
834 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
835 tempAperCat.reserve(goodSrc.selected.sum())
836 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
838 with np.warnings.catch_warnings():
839 # Ignore warnings, we will filter infinities and
840 # nans below.
841 np.warnings.simplefilter("ignore")
843 tempAperCat[instMagInKey][:] = -2.5*np.log10(
844 sources[instFluxAperInKey][goodSrc.selected])
845 tempAperCat[instMagErrInKey][:] = (2.5/np.log(10.))*(
846 sources[instFluxErrAperInKey][goodSrc.selected] /
847 sources[instFluxAperInKey][goodSrc.selected])
848 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
849 sources[instFluxAperOutKey][goodSrc.selected])
850 tempAperCat[instMagErrOutKey][:] = (2.5/np.log(10.))*(
851 sources[instFluxErrAperOutKey][goodSrc.selected] /
852 sources[instFluxAperOutKey][goodSrc.selected])
854 aperVisitCatalog.extend(tempAperCat)
856 nStarInVisit += len(tempCat)
858 # Compute the median delta-aper
859 if not aperVisitCatalog.isContiguous():
860 aperVisitCatalog = aperVisitCatalog.copy(deep=True)
862 instMagIn = aperVisitCatalog[instMagInKey]
863 instMagErrIn = aperVisitCatalog[instMagErrInKey]
864 instMagOut = aperVisitCatalog[instMagOutKey]
865 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
867 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
868 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
870 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
871 visit['sources_read'] = 1
873 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" %
874 (nStarInVisit, visit['visit'], visit['deltaAper']))
876 if ((ctr % self.config.nVisitsPerCheckpoint) == 0 and
877 starObsDataRef is not None and visitCatDataRef is not None):
878 # We need to persist both the stars and the visit catalog which gets
879 # additional metadata from each visit.
880 starObsDataRef.put(fullCatalog)
881 visitCatDataRef.put(visitCat)
883 self.log.info("Found all good star observations in %.2f s" %
884 (time.time() - startTime))
886 return fullCatalog
888 def fgcmMatchStars(self, butler, visitCat, obsCat):
889 """
890 Use FGCM code to match observations into unique stars.
892 Parameters
893 ----------
894 butler: `lsst.daf.persistence.Butler`
895 visitCat: `afw.table.BaseCatalog`
896 Catalog with visit data for fgcm
897 obsCat: `afw.table.BaseCatalog`
898 Full catalog of star observations for fgcm
900 Returns
901 -------
902 fgcmStarIdCat: `afw.table.BaseCatalog`
903 Catalog of unique star identifiers and index keys
904 fgcmStarIndicesCat: `afwTable.BaseCatalog`
905 Catalog of unique star indices
906 fgcmRefCat: `afw.table.BaseCatalog`
907 Catalog of matched reference stars.
908 Will be None if `config.doReferenceMatches` is False.
909 """
911 if self.config.doReferenceMatches:
912 # Make a subtask for reference loading
913 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
915 # get filter names into a numpy array...
916 # This is the type that is expected by the fgcm code
917 visitFilterNames = np.zeros(len(visitCat), dtype='a10')
918 for i in range(len(visitCat)):
919 visitFilterNames[i] = visitCat[i]['filtername']
921 # match to put filterNames with observations
922 visitIndex = np.searchsorted(visitCat['visit'],
923 obsCat['visit'])
925 obsFilterNames = visitFilterNames[visitIndex]
927 if self.config.doReferenceMatches:
928 # Get the reference filter names, using the LUT
929 lutCat = butler.get('fgcmLookUpTable')
931 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
932 zip(lutCat[0]['filterNames'].split(','),
933 lutCat[0]['stdFilterNames'].split(','))}
934 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
935 zip(lutCat[0]['stdFilterNames'].split(','),
936 lutCat[0]['lambdaStdFilter'])}
938 del lutCat
940 referenceFilterNames = self._getReferenceFilterNames(visitCat,
941 stdFilterDict,
942 stdLambdaDict)
943 self.log.info("Using the following reference filters: %s" %
944 (', '.join(referenceFilterNames)))
946 else:
947 # This should be an empty list
948 referenceFilterNames = []
950 # make the fgcm starConfig dict
952 starConfig = {'logger': self.log,
953 'filterToBand': self.config.filterMap,
954 'requiredBands': self.config.requiredBands,
955 'minPerBand': self.config.minPerBand,
956 'matchRadius': self.config.matchRadius,
957 'isolationRadius': self.config.isolationRadius,
958 'matchNSide': self.config.matchNside,
959 'coarseNSide': self.config.coarseNside,
960 'densNSide': self.config.densityCutNside,
961 'densMaxPerPixel': self.config.densityCutMaxPerPixel,
962 'primaryBands': self.config.primaryBands,
963 'referenceFilterNames': referenceFilterNames}
965 # initialize the FgcmMakeStars object
966 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
968 # make the primary stars
969 # note that the ra/dec native Angle format is radians
970 # We determine the conversion from the native units (typically
971 # radians) to degrees for the first observation. This allows us
972 # to treate ra/dec as numpy arrays rather than Angles, which would
973 # be approximately 600x slower.
974 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
975 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
976 obsCat['dec'] * conv,
977 filterNameArray=obsFilterNames,
978 bandSelected=False)
980 # and match all the stars
981 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
982 obsCat['dec'] * conv,
983 obsFilterNames)
985 if self.config.doReferenceMatches:
986 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
988 # now persist
990 objSchema = self._makeFgcmObjSchema()
992 # make catalog and records
993 fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
994 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
995 for i in range(fgcmMakeStars.objIndexCat.size):
996 fgcmStarIdCat.addNew()
998 # fill the catalog
999 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
1000 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
1001 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
1002 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
1003 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
1005 obsSchema = self._makeFgcmObsSchema()
1007 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
1008 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
1009 for i in range(fgcmMakeStars.obsIndexCat.size):
1010 fgcmStarIndicesCat.addNew()
1012 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
1014 if self.config.doReferenceMatches:
1015 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
1017 fgcmRefCat = afwTable.BaseCatalog(refSchema)
1018 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
1020 for i in range(fgcmMakeStars.referenceCat.size):
1021 fgcmRefCat.addNew()
1023 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
1024 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
1025 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
1027 md = PropertyList()
1028 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
1029 md.set("FILTERNAMES", referenceFilterNames)
1030 fgcmRefCat.setMetadata(md)
1032 else:
1033 fgcmRefCat = None
1035 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
1037 def _makeFgcmVisitSchema(self, nCcd):
1038 """
1039 Make a schema for an fgcmVisitCatalog
1041 Parameters
1042 ----------
1043 nCcd: `int`
1044 Number of CCDs in the camera
1046 Returns
1047 -------
1048 schema: `afwTable.Schema`
1049 """
1051 schema = afwTable.Schema()
1052 schema.addField('visit', type=np.int32, doc="Visit number")
1053 # Note that the FGCM code currently handles filternames up to 2 characters long
1054 schema.addField('filtername', type=str, size=10, doc="Filter name")
1055 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
1056 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
1057 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
1058 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
1059 schema.addField('mjd', type=np.float64, doc="MJD of visit")
1060 schema.addField('exptime', type=np.float32, doc="Exposure time")
1061 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
1062 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
1063 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
1064 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
1065 # the following field is not used yet
1066 schema.addField('deepFlag', type=np.int32, doc="Deep observation")
1067 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
1068 size=nCcd)
1069 schema.addField('used', type=np.int32, doc="This visit has been ingested.")
1070 schema.addField('sources_read', type=np.int32, doc="This visit had sources read.")
1072 return schema
1074 def _makeSourceMapper(self, sourceSchema):
1075 """
1076 Make a schema mapper for fgcm sources
1078 Parameters
1079 ----------
1080 sourceSchema: `afwTable.Schema`
1081 Default source schema from the butler
1083 Returns
1084 -------
1085 sourceMapper: `afwTable.schemaMapper`
1086 Mapper to the FGCM source schema
1087 """
1089 # create a mapper to the preferred output
1090 sourceMapper = afwTable.SchemaMapper(sourceSchema)
1092 # map to ra/dec
1093 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
1094 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
1095 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
1096 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
1097 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
1098 'psf_candidate')
1100 # and add the fields we want
1101 sourceMapper.editOutputSchema().addField(
1102 "visit", type=np.int32, doc="Visit number")
1103 sourceMapper.editOutputSchema().addField(
1104 "ccd", type=np.int32, doc="CCD number")
1105 sourceMapper.editOutputSchema().addField(
1106 "instMag", type=np.float32, doc="Instrumental magnitude")
1107 sourceMapper.editOutputSchema().addField(
1108 "instMagErr", type=np.float32, doc="Instrumental magnitude error")
1109 sourceMapper.editOutputSchema().addField(
1110 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
1112 return sourceMapper
1114 def _makeAperMapper(self, sourceSchema):
1115 """
1116 Make a schema mapper for fgcm aperture measurements
1118 Parameters
1119 ----------
1120 sourceSchema: `afwTable.Schema`
1121 Default source schema from the butler
1123 Returns
1124 -------
1125 aperMapper: `afwTable.schemaMapper`
1126 Mapper to the FGCM aperture schema
1127 """
1129 aperMapper = afwTable.SchemaMapper(sourceSchema)
1130 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
1131 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
1132 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64,
1133 doc="Magnitude at inner aperture")
1134 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64,
1135 doc="Magnitude error at inner aperture")
1136 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64,
1137 doc="Magnitude at outer aperture")
1138 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64,
1139 doc="Magnitude error at outer aperture")
1141 return aperMapper
1143 def _makeFgcmObjSchema(self):
1144 """
1145 Make a schema for the objIndexCat from fgcmMakeStars
1147 Returns
1148 -------
1149 schema: `afwTable.Schema`
1150 """
1152 objSchema = afwTable.Schema()
1153 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
1154 # Will investigate making these angles...
1155 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
1156 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
1157 objSchema.addField('obsArrIndex', type=np.int32,
1158 doc='Index in obsIndexTable for first observation')
1159 objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
1161 return objSchema
1163 def _makeFgcmObsSchema(self):
1164 """
1165 Make a schema for the obsIndexCat from fgcmMakeStars
1167 Returns
1168 -------
1169 schema: `afwTable.Schema`
1170 """
1172 obsSchema = afwTable.Schema()
1173 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
1175 return obsSchema
1177 def _makeFgcmRefSchema(self, nReferenceBands):
1178 """
1179 Make a schema for the referenceCat from fgcmMakeStars
1181 Parameters
1182 ----------
1183 nReferenceBands: `int`
1184 Number of reference bands
1186 Returns
1187 -------
1188 schema: `afwTable.Schema`
1189 """
1191 refSchema = afwTable.Schema()
1192 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
1193 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
1194 size=nReferenceBands)
1195 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
1196 size=nReferenceBands)
1198 return refSchema
1200 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
1201 """
1202 Get the reference filter names, in wavelength order, from the visitCat and
1203 information from the look-up-table.
1205 Parameters
1206 ----------
1207 visitCat: `afw.table.BaseCatalog`
1208 Catalog with visit data for FGCM
1209 stdFilterDict: `dict`
1210 Mapping of filterName to stdFilterName from LUT
1211 stdLambdaDict: `dict`
1212 Mapping of stdFilterName to stdLambda from LUT
1214 Returns
1215 -------
1216 referenceFilterNames: `list`
1217 Wavelength-ordered list of reference filter names
1218 """
1220 # Find the unique list of filter names in visitCat
1221 filterNames = np.unique(visitCat.asAstropy()['filtername'])
1223 # Find the unique list of "standard" filters
1224 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
1226 # And sort these by wavelength
1227 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
1229 return referenceFilterNames