Coverage for python/lsst/fgcmcal/fgcmBuildStarsBase.py : 17%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for BuildStars using src tables or sourceTable_visit tables.
22"""
24import os
25import sys
26import traceback
27import abc
29import numpy as np
31import lsst.daf.persistence as dafPersist
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34import lsst.afw.table as afwTable
35import lsst.geom as geom
36from lsst.daf.base import PropertyList
37from lsst.daf.base.dateTime import DateTime
38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
40from .utilities import computeApertureRadiusFromDataRef
41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
43import fgcm
45REFSTARS_FORMAT_VERSION = 1
47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
50class FgcmBuildStarsConfigBase(pexConfig.Config):
51 """Base config for FgcmBuildStars tasks"""
53 instFluxField = pexConfig.Field(
54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55 "The associated flag will be implicitly included in badFlags"),
56 dtype=str,
57 default='slot_CalibFlux_instFlux',
58 )
59 minPerBand = pexConfig.Field(
60 doc="Minimum observations per band",
61 dtype=int,
62 default=2,
63 )
64 matchRadius = pexConfig.Field(
65 doc="Match radius (arcseconds)",
66 dtype=float,
67 default=1.0,
68 )
69 isolationRadius = pexConfig.Field(
70 doc="Isolation radius (arcseconds)",
71 dtype=float,
72 default=2.0,
73 )
74 densityCutNside = pexConfig.Field(
75 doc="Density cut healpix nside",
76 dtype=int,
77 default=128,
78 )
79 densityCutMaxPerPixel = pexConfig.Field(
80 doc="Density cut number of stars per pixel",
81 dtype=int,
82 default=1000,
83 )
84 matchNside = pexConfig.Field(
85 doc="Healpix Nside for matching",
86 dtype=int,
87 default=4096,
88 )
89 coarseNside = pexConfig.Field(
90 doc="Healpix coarse Nside for partitioning matches",
91 dtype=int,
92 default=8,
93 )
94 # The following config will not be necessary after Gen2 retirement.
95 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
96 # which is easiest to access in the config file.
97 physicalFilterMap = pexConfig.DictField(
98 doc="Mapping from 'physicalFilter' to band.",
99 keytype=str,
100 itemtype=str,
101 default={},
102 )
103 requiredBands = pexConfig.ListField(
104 doc="Bands required for each star",
105 dtype=str,
106 default=(),
107 )
108 primaryBands = pexConfig.ListField(
109 doc=("Bands for 'primary' star matches. "
110 "A star must be observed in one of these bands to be considered "
111 "as a calibration star."),
112 dtype=str,
113 default=None
114 )
115 visitDataRefName = pexConfig.Field(
116 doc="dataRef name for the 'visit' field, usually 'visit'.",
117 dtype=str,
118 default="visit"
119 )
120 ccdDataRefName = pexConfig.Field(
121 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
122 dtype=str,
123 default="ccd"
124 )
125 doApplyWcsJacobian = pexConfig.Field(
126 doc="Apply the jacobian of the WCS to the star observations prior to fit?",
127 dtype=bool,
128 default=True
129 )
130 doModelErrorsWithBackground = pexConfig.Field(
131 doc="Model flux errors with background term?",
132 dtype=bool,
133 default=True
134 )
135 psfCandidateName = pexConfig.Field(
136 doc="Name of field with psf candidate flag for propagation",
137 dtype=str,
138 default="calib_psf_candidate"
139 )
140 doSubtractLocalBackground = pexConfig.Field(
141 doc=("Subtract the local background before performing calibration? "
142 "This is only supported for circular aperture calibration fluxes."),
143 dtype=bool,
144 default=False
145 )
146 localBackgroundFluxField = pexConfig.Field(
147 doc="Full name of the local background instFlux field to use.",
148 dtype=str,
149 default='base_LocalBackground_instFlux'
150 )
151 sourceSelector = sourceSelectorRegistry.makeField(
152 doc="How to select sources",
153 default="science"
154 )
155 apertureInnerInstFluxField = pexConfig.Field(
156 doc=("Full name of instFlux field that contains inner aperture "
157 "flux for aperture correction proxy"),
158 dtype=str,
159 default='base_CircularApertureFlux_12_0_instFlux'
160 )
161 apertureOuterInstFluxField = pexConfig.Field(
162 doc=("Full name of instFlux field that contains outer aperture "
163 "flux for aperture correction proxy"),
164 dtype=str,
165 default='base_CircularApertureFlux_17_0_instFlux'
166 )
167 doReferenceMatches = pexConfig.Field(
168 doc="Match reference catalog as additional constraint on calibration",
169 dtype=bool,
170 default=True,
171 )
172 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
173 target=FgcmLoadReferenceCatalogTask,
174 doc="FGCM reference object loader",
175 )
176 nVisitsPerCheckpoint = pexConfig.Field(
177 doc="Number of visits read between checkpoints",
178 dtype=int,
179 default=500,
180 )
182 def setDefaults(self):
183 sourceSelector = self.sourceSelector["science"]
184 sourceSelector.setDefaults()
186 sourceSelector.doFlags = True
187 sourceSelector.doUnresolved = True
188 sourceSelector.doSignalToNoise = True
189 sourceSelector.doIsolated = True
191 sourceSelector.signalToNoise.minimum = 10.0
192 sourceSelector.signalToNoise.maximum = 1000.0
194 # FGCM operates on unresolved sources, and this setting is
195 # appropriate for the current base_ClassificationExtendedness
196 sourceSelector.unresolved.maximum = 0.5
199class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
200 """Subclass of TaskRunner for FgcmBuildStars tasks
202 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
203 arguments, one of which is the butler (for persistence and mapper data),
204 and a list of dataRefs extracted from the command line. Note that FGCM
205 runs on a large set of dataRefs, and not on single dataRef/tract/patch.
206 This class transforms the process arguments generated by the ArgumentParser
207 into the arguments expected by FgcmBuildStarsTask.run(). This runner does
208 not use any parallelization.
209 """
210 @staticmethod
211 def getTargetList(parsedCmd):
212 """
213 Return a list with one element: a tuple with the butler and
214 list of dataRefs
215 """
216 # we want to combine the butler with any (or no!) dataRefs
217 return [(parsedCmd.butler, parsedCmd.id.refList)]
219 def __call__(self, args):
220 """
221 Parameters
222 ----------
223 args: `tuple` with (butler, dataRefList)
225 Returns
226 -------
227 exitStatus: `list` with `lsst.pipe.base.Struct`
228 exitStatus (0: success; 1: failure)
229 """
230 butler, dataRefList = args
232 task = self.TaskClass(config=self.config, log=self.log)
234 exitStatus = 0
235 if self.doRaise:
236 task.runDataRef(butler, dataRefList)
237 else:
238 try:
239 task.runDataRef(butler, dataRefList)
240 except Exception as e:
241 exitStatus = 1
242 task.log.fatal("Failed: %s" % e)
243 if not isinstance(e, pipeBase.TaskError):
244 traceback.print_exc(file=sys.stderr)
246 task.writeMetadata(butler)
248 # The task does not return any results:
249 return [pipeBase.Struct(exitStatus=exitStatus)]
251 def run(self, parsedCmd):
252 """
253 Run the task, with no multiprocessing
255 Parameters
256 ----------
257 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
258 """
260 resultList = []
262 if self.precall(parsedCmd):
263 targetList = self.getTargetList(parsedCmd)
264 resultList = self(targetList[0])
266 return resultList
269class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
270 """
271 Base task to build stars for FGCM global calibration
273 Parameters
274 ----------
275 butler : `lsst.daf.persistence.Butler`
276 """
277 def __init__(self, initInputs=None, butler=None, **kwargs):
278 super().__init__(**kwargs)
280 self.makeSubtask("sourceSelector")
281 # Only log warning and fatal errors from the sourceSelector
282 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
284 # no saving of metadata for now
285 def _getMetadataName(self):
286 return None
288 @pipeBase.timeMethod
289 def runDataRef(self, butler, dataRefs):
290 """
291 Cross-match and make star list for FGCM Input
293 Parameters
294 ----------
295 butler: `lsst.daf.persistence.Butler`
296 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
297 Source data references for the input visits.
299 Raises
300 ------
301 RuntimeErrror: Raised if `config.doReferenceMatches` is set and
302 an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
303 fails if the calibFlux is not a CircularAperture flux.
304 """
305 datasetType = dataRefs[0].butlerSubset.datasetType
306 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
308 if self.config.doReferenceMatches:
309 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
310 # Ensure that we have a LUT
311 if not butler.datasetExists('fgcmLookUpTable'):
312 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
313 # Compute aperture radius if necessary. This is useful to do now before
314 # any heavy lifting has happened (fail early).
315 calibFluxApertureRadius = None
316 if self.config.doSubtractLocalBackground:
317 try:
318 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
319 self.config.instFluxField)
320 except RuntimeError as e:
321 raise RuntimeError("Could not determine aperture radius from %s. "
322 "Cannot use doSubtractLocalBackground." %
323 (self.config.instFluxField)) from e
325 camera = butler.get('camera')
326 groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs)
328 # Make the visit catalog if necessary
329 # First check if the visit catalog is in the _current_ path
330 # We cannot use Gen2 datasetExists() because that checks all parent
331 # directories as well, which would make recovering from faults
332 # and fgcmcal reruns impossible.
333 visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
334 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
335 if os.path.exists(filename):
336 # This file exists and we should continue processing
337 inVisitCat = visitCatDataRef.get()
338 if len(inVisitCat) != len(groupedDataRefs):
339 raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
340 "number of visits. Cannot continue.")
341 else:
342 inVisitCat = None
344 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
345 visitCatDataRef=visitCatDataRef,
346 inVisitCat=inVisitCat)
348 # Persist the visitCat as a checkpoint file.
349 visitCatDataRef.put(visitCat)
351 starObsDataRef = butler.dataRef('fgcmStarObservations')
352 filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
353 if os.path.exists(filename):
354 inStarObsCat = starObsDataRef.get()
355 else:
356 inStarObsCat = None
358 rad = calibFluxApertureRadius
359 sourceSchemaDataRef = butler.dataRef('src_schema')
360 sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema
361 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
362 visitCat,
363 sourceSchema,
364 camera,
365 calibFluxApertureRadius=rad,
366 starObsDataRef=starObsDataRef,
367 visitCatDataRef=visitCatDataRef,
368 inStarObsCat=inStarObsCat)
369 visitCatDataRef.put(visitCat)
370 starObsDataRef.put(fgcmStarObservationCat)
372 # Always do the matching.
373 if self.config.doReferenceMatches:
374 lutDataRef = butler.dataRef('fgcmLookUpTable')
375 else:
376 lutDataRef = None
377 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
378 fgcmStarObservationCat,
379 lutDataRef=lutDataRef)
381 # Persist catalogs via the butler
382 butler.put(fgcmStarIdCat, 'fgcmStarIds')
383 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
384 if fgcmRefCat is not None:
385 butler.put(fgcmRefCat, 'fgcmReferenceStars')
387 @abc.abstractmethod
388 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
389 """
390 Find and group dataRefs (by visit); Gen2 only.
392 Parameters
393 ----------
394 butler : `lsst.daf.persistence.Butler`
395 Gen2 butler.
396 camera : `lsst.afw.cameraGeom.Camera`
397 Camera from the butler.
398 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
399 Data references for the input visits.
401 Returns
402 -------
403 groupedDataRefs : `dict` [`int`, `list`]
404 Dictionary with sorted visit keys, and `list`s of
405 `lsst.daf.persistence.ButlerDataRef`
406 """
407 raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.")
409 @abc.abstractmethod
410 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
411 sourceSchema,
412 camera,
413 calibFluxApertureRadius=None,
414 visitCatDataRef=None,
415 starObsDataRef=None,
416 inStarObsCat=None):
417 """
418 Compile all good star observations from visits in visitCat. Checkpoint files
419 will be stored if both visitCatDataRef and starObsDataRef are not None.
421 Parameters
422 ----------
423 groupedDataRefs : `dict` of `list`s
424 Lists of `~lsst.daf.persistence.ButlerDataRef` or
425 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
426 visitCat : `~afw.table.BaseCatalog`
427 Catalog with visit data for FGCM
428 sourceSchema : `~lsst.afw.table.Schema`
429 Schema for the input src catalogs.
430 camera : `~lsst.afw.cameraGeom.Camera`
431 calibFluxApertureRadius : `float`, optional
432 Aperture radius for calibration flux.
433 visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
434 Dataref to write visitCat for checkpoints
435 starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
436 Dataref to write the star observation catalog for checkpoints.
437 inStarObsCat : `~afw.table.BaseCatalog`
438 Input observation catalog. If this is incomplete, observations
439 will be appended from when it was cut off.
441 Returns
442 -------
443 fgcmStarObservations : `afw.table.BaseCatalog`
444 Full catalog of good observations.
446 Raises
447 ------
448 RuntimeError: Raised if doSubtractLocalBackground is True and
449 calibFluxApertureRadius is not set.
450 """
451 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
453 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
454 visitCatDataRef=None, inVisitCat=None):
455 """
456 Make a visit catalog with all the keys from each visit
458 Parameters
459 ----------
460 camera: `lsst.afw.cameraGeom.Camera`
461 Camera from the butler
462 groupedDataRefs: `dict`
463 Dictionary with visit keys, and `list`s of
464 `lsst.daf.persistence.ButlerDataRef`
465 bkgDataRefDict: `dict`, optional
466 Dictionary of gen3 dataRefHandles for background info.
467 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
468 Dataref to write visitCat for checkpoints
469 inVisitCat: `afw.table.BaseCatalog`, optional
470 Input (possibly incomplete) visit catalog
472 Returns
473 -------
474 visitCat: `afw.table.BaseCatalog`
475 """
477 self.log.info("Assembling visitCatalog from %d %ss" %
478 (len(groupedDataRefs), self.config.visitDataRefName))
480 nCcd = len(camera)
482 if inVisitCat is None:
483 schema = self._makeFgcmVisitSchema(nCcd)
485 visitCat = afwTable.BaseCatalog(schema)
486 visitCat.reserve(len(groupedDataRefs))
487 visitCat.resize(len(groupedDataRefs))
489 visitCat['visit'] = list(groupedDataRefs.keys())
490 visitCat['used'] = 0
491 visitCat['sources_read'] = False
492 else:
493 visitCat = inVisitCat
495 # No matter what, fill the catalog. This will check if it was
496 # already read.
497 self._fillVisitCatalog(visitCat, groupedDataRefs,
498 bkgDataRefDict=bkgDataRefDict,
499 visitCatDataRef=visitCatDataRef)
501 return visitCat
503 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
504 visitCatDataRef=None):
505 """
506 Fill the visit catalog with visit metadata
508 Parameters
509 ----------
510 visitCat : `afw.table.BaseCatalog`
511 Visit catalog. See _makeFgcmVisitSchema() for schema definition.
512 groupedDataRefs : `dict`
513 Dictionary with visit keys, and `list`s of
514 `lsst.daf.persistence.ButlerDataRef` or
515 `lsst.daf.butler.DeferredDatasetHandle`
516 visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional
517 Dataref to write ``visitCat`` for checkpoints. Gen2 only.
518 bkgDataRefDict : `dict`, optional
519 Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle`
520 for background info.
521 """
522 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
524 for i, visit in enumerate(groupedDataRefs):
525 # We don't use the bypasses since we need the psf info which does
526 # not have a bypass
527 # TODO: When DM-15500 is implemented in the Gen3 Butler, this
528 # can be fixed
530 # Do not read those that have already been read
531 if visitCat['used'][i]:
532 continue
534 if (i % self.config.nVisitsPerCheckpoint) == 0:
535 self.log.info("Retrieving metadata for %s %d (%d/%d)" %
536 (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
537 # Save checkpoint if desired
538 if visitCatDataRef is not None:
539 visitCatDataRef.put(visitCat)
541 dataRef = groupedDataRefs[visit][0]
542 if isinstance(dataRef, dafPersist.ButlerDataRef):
543 # Gen2: calexp dataRef
544 # The first dataRef in the group will be the reference ccd (if available)
545 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
546 visitInfo = exp.getInfo().getVisitInfo()
547 label = dataRef.get(datasetType='calexp_filterLabel')
548 physicalFilter = label.physicalLabel
549 psf = exp.getPsf()
550 psfSigma = psf.computeShape().getDeterminantRadius()
551 else:
552 # Gen3: use the visitSummary dataRef
553 summary = dataRef.get()
555 summaryRow = summary.find(self.config.referenceCCD)
556 if summaryRow is None:
557 # Take the first available ccd if reference isn't available
558 summaryRow = summary[0]
560 summaryDetector = summaryRow['id']
561 visitInfo = summaryRow.getVisitInfo()
562 physicalFilter = summaryRow['physical_filter']
563 # Compute the median psf sigma if possible
564 goodSigma, = np.where(summary['psfSigma'] > 0)
565 if goodSigma.size > 2:
566 psfSigma = np.median(summary['psfSigma'][goodSigma])
567 elif goodSigma > 0:
568 psfSigma = np.mean(summary['psfSigma'][goodSigma])
569 else:
570 psfSigma = 0.0
572 rec = visitCat[i]
573 rec['visit'] = visit
574 rec['physicalFilter'] = physicalFilter
575 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
576 # much easier to get the wcs's necessary to recompute the pointing
577 # ra/dec at the center of the camera.
578 radec = visitInfo.getBoresightRaDec()
579 rec['telra'] = radec.getRa().asDegrees()
580 rec['teldec'] = radec.getDec().asDegrees()
581 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
582 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
583 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
584 rec['exptime'] = visitInfo.getExposureTime()
585 # convert from Pa to millibar
586 # Note that I don't know if this unit will need to be per-camera config
587 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
588 # Flag to signify if this is a "deep" field. Not currently used
589 rec['deepFlag'] = 0
590 # Relative flat scaling (1.0 means no relative scaling)
591 rec['scaling'][:] = 1.0
592 # Median delta aperture, to be measured from stars
593 rec['deltaAper'] = 0.0
594 rec['psfSigma'] = psfSigma
596 if self.config.doModelErrorsWithBackground:
597 foundBkg = False
598 if isinstance(dataRef, dafPersist.ButlerDataRef):
599 # Gen2-style dataRef
600 det = dataRef.dataId[self.config.ccdDataRefName]
601 if dataRef.datasetExists(datasetType='calexpBackground'):
602 bgList = dataRef.get(datasetType='calexpBackground')
603 foundBkg = True
604 else:
605 # Gen3-style dataRef
606 try:
607 # Use the same detector used from the summary.
608 bkgRef = bkgDataRefDict[(visit, summaryDetector)]
609 bgList = bkgRef.get()
610 foundBkg = True
611 except KeyError:
612 pass
614 if foundBkg:
615 bgStats = (bg[0].getStatsImage().getImage().array
616 for bg in bgList)
617 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
618 else:
619 self.log.warn('Sky background not found for visit %d / ccd %d' %
620 (visit, det))
621 rec['skyBackground'] = -1.0
622 else:
623 rec['skyBackground'] = -1.0
625 rec['used'] = 1
627 def _makeSourceMapper(self, sourceSchema):
628 """
629 Make a schema mapper for fgcm sources
631 Parameters
632 ----------
633 sourceSchema: `afwTable.Schema`
634 Default source schema from the butler
636 Returns
637 -------
638 sourceMapper: `afwTable.schemaMapper`
639 Mapper to the FGCM source schema
640 """
642 # create a mapper to the preferred output
643 sourceMapper = afwTable.SchemaMapper(sourceSchema)
645 # map to ra/dec
646 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
647 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
648 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
649 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
650 # Add the mapping if the field exists in the input catalog.
651 # If the field does not exist, simply add it (set to False).
652 # This field is not required for calibration, but is useful
653 # to collate if available.
654 try:
655 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
656 'psf_candidate')
657 except LookupError:
658 sourceMapper.editOutputSchema().addField(
659 "psf_candidate", type='Flag',
660 doc=("Flag set if the source was a candidate for PSF determination, "
661 "as determined by the star selector."))
663 # and add the fields we want
664 sourceMapper.editOutputSchema().addField(
665 "visit", type=np.int32, doc="Visit number")
666 sourceMapper.editOutputSchema().addField(
667 "ccd", type=np.int32, doc="CCD number")
668 sourceMapper.editOutputSchema().addField(
669 "instMag", type=np.float32, doc="Instrumental magnitude")
670 sourceMapper.editOutputSchema().addField(
671 "instMagErr", type=np.float32, doc="Instrumental magnitude error")
672 sourceMapper.editOutputSchema().addField(
673 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
674 sourceMapper.editOutputSchema().addField(
675 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
677 return sourceMapper
679 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
680 """
681 Use FGCM code to match observations into unique stars.
683 Parameters
684 ----------
685 visitCat: `afw.table.BaseCatalog`
686 Catalog with visit data for fgcm
687 obsCat: `afw.table.BaseCatalog`
688 Full catalog of star observations for fgcm
689 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
690 `lsst.daf.butler.DeferredDatasetHandle`, optional
691 Data reference to fgcm look-up table (used if matching reference stars).
693 Returns
694 -------
695 fgcmStarIdCat: `afw.table.BaseCatalog`
696 Catalog of unique star identifiers and index keys
697 fgcmStarIndicesCat: `afwTable.BaseCatalog`
698 Catalog of unique star indices
699 fgcmRefCat: `afw.table.BaseCatalog`
700 Catalog of matched reference stars.
701 Will be None if `config.doReferenceMatches` is False.
702 """
703 # get filter names into a numpy array...
704 # This is the type that is expected by the fgcm code
705 visitFilterNames = np.zeros(len(visitCat), dtype='a30')
706 for i in range(len(visitCat)):
707 visitFilterNames[i] = visitCat[i]['physicalFilter']
709 # match to put filterNames with observations
710 visitIndex = np.searchsorted(visitCat['visit'],
711 obsCat['visit'])
713 obsFilterNames = visitFilterNames[visitIndex]
715 if self.config.doReferenceMatches:
716 # Get the reference filter names, using the LUT
717 lutCat = lutDataRef.get()
719 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
720 zip(lutCat[0]['physicalFilters'].split(','),
721 lutCat[0]['stdPhysicalFilters'].split(','))}
722 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
723 zip(lutCat[0]['stdPhysicalFilters'].split(','),
724 lutCat[0]['lambdaStdFilter'])}
726 del lutCat
728 referenceFilterNames = self._getReferenceFilterNames(visitCat,
729 stdFilterDict,
730 stdLambdaDict)
731 self.log.info("Using the following reference filters: %s" %
732 (', '.join(referenceFilterNames)))
734 else:
735 # This should be an empty list
736 referenceFilterNames = []
738 # make the fgcm starConfig dict
739 starConfig = {'logger': self.log,
740 'filterToBand': self.config.physicalFilterMap,
741 'requiredBands': self.config.requiredBands,
742 'minPerBand': self.config.minPerBand,
743 'matchRadius': self.config.matchRadius,
744 'isolationRadius': self.config.isolationRadius,
745 'matchNSide': self.config.matchNside,
746 'coarseNSide': self.config.coarseNside,
747 'densNSide': self.config.densityCutNside,
748 'densMaxPerPixel': self.config.densityCutMaxPerPixel,
749 'primaryBands': self.config.primaryBands,
750 'referenceFilterNames': referenceFilterNames}
752 # initialize the FgcmMakeStars object
753 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
755 # make the primary stars
756 # note that the ra/dec native Angle format is radians
757 # We determine the conversion from the native units (typically
758 # radians) to degrees for the first observation. This allows us
759 # to treate ra/dec as numpy arrays rather than Angles, which would
760 # be approximately 600x slower.
761 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
762 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
763 obsCat['dec'] * conv,
764 filterNameArray=obsFilterNames,
765 bandSelected=False)
767 # and match all the stars
768 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
769 obsCat['dec'] * conv,
770 obsFilterNames)
772 if self.config.doReferenceMatches:
773 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
775 # now persist
777 objSchema = self._makeFgcmObjSchema()
779 # make catalog and records
780 fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
781 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
782 for i in range(fgcmMakeStars.objIndexCat.size):
783 fgcmStarIdCat.addNew()
785 # fill the catalog
786 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
787 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
788 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
789 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
790 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
792 obsSchema = self._makeFgcmObsSchema()
794 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
795 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
796 for i in range(fgcmMakeStars.obsIndexCat.size):
797 fgcmStarIndicesCat.addNew()
799 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
801 if self.config.doReferenceMatches:
802 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
804 fgcmRefCat = afwTable.BaseCatalog(refSchema)
805 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
807 for i in range(fgcmMakeStars.referenceCat.size):
808 fgcmRefCat.addNew()
810 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
811 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
812 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
814 md = PropertyList()
815 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
816 md.set("FILTERNAMES", referenceFilterNames)
817 fgcmRefCat.setMetadata(md)
819 else:
820 fgcmRefCat = None
822 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
824 def _makeFgcmVisitSchema(self, nCcd):
825 """
826 Make a schema for an fgcmVisitCatalog
828 Parameters
829 ----------
830 nCcd: `int`
831 Number of CCDs in the camera
833 Returns
834 -------
835 schema: `afwTable.Schema`
836 """
838 schema = afwTable.Schema()
839 schema.addField('visit', type=np.int32, doc="Visit number")
840 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
841 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
842 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
843 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
844 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
845 schema.addField('mjd', type=np.float64, doc="MJD of visit")
846 schema.addField('exptime', type=np.float32, doc="Exposure time")
847 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
848 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
849 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
850 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
851 # the following field is not used yet
852 schema.addField('deepFlag', type=np.int32, doc="Deep observation")
853 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
854 size=nCcd)
855 schema.addField('used', type=np.int32, doc="This visit has been ingested.")
856 schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
858 return schema
860 def _makeFgcmObjSchema(self):
861 """
862 Make a schema for the objIndexCat from fgcmMakeStars
864 Returns
865 -------
866 schema: `afwTable.Schema`
867 """
869 objSchema = afwTable.Schema()
870 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
871 # Will investigate making these angles...
872 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
873 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
874 objSchema.addField('obsArrIndex', type=np.int32,
875 doc='Index in obsIndexTable for first observation')
876 objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
878 return objSchema
880 def _makeFgcmObsSchema(self):
881 """
882 Make a schema for the obsIndexCat from fgcmMakeStars
884 Returns
885 -------
886 schema: `afwTable.Schema`
887 """
889 obsSchema = afwTable.Schema()
890 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
892 return obsSchema
894 def _makeFgcmRefSchema(self, nReferenceBands):
895 """
896 Make a schema for the referenceCat from fgcmMakeStars
898 Parameters
899 ----------
900 nReferenceBands: `int`
901 Number of reference bands
903 Returns
904 -------
905 schema: `afwTable.Schema`
906 """
908 refSchema = afwTable.Schema()
909 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
910 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
911 size=nReferenceBands)
912 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
913 size=nReferenceBands)
915 return refSchema
917 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
918 """
919 Get the reference filter names, in wavelength order, from the visitCat and
920 information from the look-up-table.
922 Parameters
923 ----------
924 visitCat: `afw.table.BaseCatalog`
925 Catalog with visit data for FGCM
926 stdFilterDict: `dict`
927 Mapping of filterName to stdFilterName from LUT
928 stdLambdaDict: `dict`
929 Mapping of stdFilterName to stdLambda from LUT
931 Returns
932 -------
933 referenceFilterNames: `list`
934 Wavelength-ordered list of reference filter names
935 """
937 # Find the unique list of filter names in visitCat
938 filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
940 # Find the unique list of "standard" filters
941 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
943 # And sort these by wavelength
944 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
946 return referenceFilterNames