Coverage for python/lsst/fgcmcal/fgcmBuildStarsBase.py : 17%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for BuildStars using src tables or sourceTable_visit tables.
22"""
24import os
25import sys
26import traceback
27import abc
29import numpy as np
31import lsst.daf.persistence as dafPersist
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34import lsst.afw.table as afwTable
35import lsst.geom as geom
36from lsst.daf.base import PropertyList
37from lsst.daf.base.dateTime import DateTime
38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
40from .utilities import computeApertureRadiusFromDataRef
41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
43import fgcm
45REFSTARS_FORMAT_VERSION = 1
47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
50class FgcmBuildStarsConfigBase(pexConfig.Config):
51 """Base config for FgcmBuildStars tasks"""
53 instFluxField = pexConfig.Field(
54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55 "The associated flag will be implicitly included in badFlags"),
56 dtype=str,
57 default='slot_CalibFlux_instFlux',
58 )
59 minPerBand = pexConfig.Field(
60 doc="Minimum observations per band",
61 dtype=int,
62 default=2,
63 )
64 matchRadius = pexConfig.Field(
65 doc="Match radius (arcseconds)",
66 dtype=float,
67 default=1.0,
68 )
69 isolationRadius = pexConfig.Field(
70 doc="Isolation radius (arcseconds)",
71 dtype=float,
72 default=2.0,
73 )
74 densityCutNside = pexConfig.Field(
75 doc="Density cut healpix nside",
76 dtype=int,
77 default=128,
78 )
79 densityCutMaxPerPixel = pexConfig.Field(
80 doc="Density cut number of stars per pixel",
81 dtype=int,
82 default=1000,
83 )
84 matchNside = pexConfig.Field(
85 doc="Healpix Nside for matching",
86 dtype=int,
87 default=4096,
88 )
89 coarseNside = pexConfig.Field(
90 doc="Healpix coarse Nside for partitioning matches",
91 dtype=int,
92 default=8,
93 )
94 filterMap = pexConfig.DictField(
95 doc="Mapping from 'filterName' to band.",
96 keytype=str,
97 itemtype=str,
98 default={},
99 deprecated=("This field is no longer used, and has been deprecated by "
100 "DM-28088. It will be removed after v22. Use "
101 "physicalFilterMap instead.")
102 )
103 # The following config will not be necessary after Gen2 retirement.
104 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
105 # which is easiest to access in the config file.
106 physicalFilterMap = pexConfig.DictField(
107 doc="Mapping from 'physicalFilter' to band.",
108 keytype=str,
109 itemtype=str,
110 default={},
111 )
112 requiredBands = pexConfig.ListField(
113 doc="Bands required for each star",
114 dtype=str,
115 default=(),
116 )
117 primaryBands = pexConfig.ListField(
118 doc=("Bands for 'primary' star matches. "
119 "A star must be observed in one of these bands to be considered "
120 "as a calibration star."),
121 dtype=str,
122 default=None
123 )
124 visitDataRefName = pexConfig.Field(
125 doc="dataRef name for the 'visit' field, usually 'visit'.",
126 dtype=str,
127 default="visit"
128 )
129 ccdDataRefName = pexConfig.Field(
130 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
131 dtype=str,
132 default="ccd"
133 )
134 doApplyWcsJacobian = pexConfig.Field(
135 doc="Apply the jacobian of the WCS to the star observations prior to fit?",
136 dtype=bool,
137 default=True
138 )
139 doModelErrorsWithBackground = pexConfig.Field(
140 doc="Model flux errors with background term?",
141 dtype=bool,
142 default=True
143 )
144 psfCandidateName = pexConfig.Field(
145 doc="Name of field with psf candidate flag for propagation",
146 dtype=str,
147 default="calib_psf_candidate"
148 )
149 doSubtractLocalBackground = pexConfig.Field(
150 doc=("Subtract the local background before performing calibration? "
151 "This is only supported for circular aperture calibration fluxes."),
152 dtype=bool,
153 default=False
154 )
155 localBackgroundFluxField = pexConfig.Field(
156 doc="Full name of the local background instFlux field to use.",
157 dtype=str,
158 default='base_LocalBackground_instFlux'
159 )
160 sourceSelector = sourceSelectorRegistry.makeField(
161 doc="How to select sources",
162 default="science"
163 )
164 apertureInnerInstFluxField = pexConfig.Field(
165 doc=("Full name of instFlux field that contains inner aperture "
166 "flux for aperture correction proxy"),
167 dtype=str,
168 default='base_CircularApertureFlux_12_0_instFlux'
169 )
170 apertureOuterInstFluxField = pexConfig.Field(
171 doc=("Full name of instFlux field that contains outer aperture "
172 "flux for aperture correction proxy"),
173 dtype=str,
174 default='base_CircularApertureFlux_17_0_instFlux'
175 )
176 doReferenceMatches = pexConfig.Field(
177 doc="Match reference catalog as additional constraint on calibration",
178 dtype=bool,
179 default=True,
180 )
181 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
182 target=FgcmLoadReferenceCatalogTask,
183 doc="FGCM reference object loader",
184 )
185 nVisitsPerCheckpoint = pexConfig.Field(
186 doc="Number of visits read between checkpoints",
187 dtype=int,
188 default=500,
189 )
191 def setDefaults(self):
192 sourceSelector = self.sourceSelector["science"]
193 sourceSelector.setDefaults()
195 sourceSelector.doFlags = True
196 sourceSelector.doUnresolved = True
197 sourceSelector.doSignalToNoise = True
198 sourceSelector.doIsolated = True
200 sourceSelector.signalToNoise.minimum = 10.0
201 sourceSelector.signalToNoise.maximum = 1000.0
203 # FGCM operates on unresolved sources, and this setting is
204 # appropriate for the current base_ClassificationExtendedness
205 sourceSelector.unresolved.maximum = 0.5
208class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
209 """Subclass of TaskRunner for FgcmBuildStars tasks
211 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
212 arguments, one of which is the butler (for persistence and mapper data),
213 and a list of dataRefs extracted from the command line. Note that FGCM
214 runs on a large set of dataRefs, and not on single dataRef/tract/patch.
215 This class transforms the process arguments generated by the ArgumentParser
216 into the arguments expected by FgcmBuildStarsTask.run(). This runner does
217 not use any parallelization.
218 """
219 @staticmethod
220 def getTargetList(parsedCmd):
221 """
222 Return a list with one element: a tuple with the butler and
223 list of dataRefs
224 """
225 # we want to combine the butler with any (or no!) dataRefs
226 return [(parsedCmd.butler, parsedCmd.id.refList)]
228 def __call__(self, args):
229 """
230 Parameters
231 ----------
232 args: `tuple` with (butler, dataRefList)
234 Returns
235 -------
236 exitStatus: `list` with `lsst.pipe.base.Struct`
237 exitStatus (0: success; 1: failure)
238 """
239 butler, dataRefList = args
241 task = self.TaskClass(config=self.config, log=self.log)
243 exitStatus = 0
244 if self.doRaise:
245 task.runDataRef(butler, dataRefList)
246 else:
247 try:
248 task.runDataRef(butler, dataRefList)
249 except Exception as e:
250 exitStatus = 1
251 task.log.fatal("Failed: %s" % e)
252 if not isinstance(e, pipeBase.TaskError):
253 traceback.print_exc(file=sys.stderr)
255 task.writeMetadata(butler)
257 # The task does not return any results:
258 return [pipeBase.Struct(exitStatus=exitStatus)]
260 def run(self, parsedCmd):
261 """
262 Run the task, with no multiprocessing
264 Parameters
265 ----------
266 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
267 """
269 resultList = []
271 if self.precall(parsedCmd):
272 targetList = self.getTargetList(parsedCmd)
273 resultList = self(targetList[0])
275 return resultList
278class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
279 """
280 Base task to build stars for FGCM global calibration
282 Parameters
283 ----------
284 butler : `lsst.daf.persistence.Butler`
285 """
286 def __init__(self, butler=None, initInputs=None, **kwargs):
287 super().__init__(**kwargs)
289 self.makeSubtask("sourceSelector")
290 # Only log warning and fatal errors from the sourceSelector
291 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
293 # no saving of metadata for now
294 def _getMetadataName(self):
295 return None
297 @pipeBase.timeMethod
298 def runDataRef(self, butler, dataRefs):
299 """
300 Cross-match and make star list for FGCM Input
302 Parameters
303 ----------
304 butler: `lsst.daf.persistence.Butler`
305 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
306 Source data references for the input visits.
308 Raises
309 ------
310 RuntimeErrror: Raised if `config.doReferenceMatches` is set and
311 an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
312 fails if the calibFlux is not a CircularAperture flux.
313 """
314 datasetType = dataRefs[0].butlerSubset.datasetType
315 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
317 if self.config.doReferenceMatches:
318 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
319 # Ensure that we have a LUT
320 if not butler.datasetExists('fgcmLookUpTable'):
321 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
322 # Compute aperture radius if necessary. This is useful to do now before
323 # any heavy lifting has happened (fail early).
324 calibFluxApertureRadius = None
325 if self.config.doSubtractLocalBackground:
326 try:
327 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
328 self.config.instFluxField)
329 except RuntimeError as e:
330 raise RuntimeError("Could not determine aperture radius from %s. "
331 "Cannot use doSubtractLocalBackground." %
332 (self.config.instFluxField)) from e
334 camera = butler.get('camera')
335 groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs)
337 # Make the visit catalog if necessary
338 # First check if the visit catalog is in the _current_ path
339 # We cannot use Gen2 datasetExists() because that checks all parent
340 # directories as well, which would make recovering from faults
341 # and fgcmcal reruns impossible.
342 visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
343 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
344 if os.path.exists(filename):
345 # This file exists and we should continue processing
346 inVisitCat = visitCatDataRef.get()
347 if len(inVisitCat) != len(groupedDataRefs):
348 raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
349 "number of visits. Cannot continue.")
350 else:
351 inVisitCat = None
353 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
354 visitCatDataRef=visitCatDataRef,
355 inVisitCat=inVisitCat)
357 # Persist the visitCat as a checkpoint file.
358 visitCatDataRef.put(visitCat)
360 starObsDataRef = butler.dataRef('fgcmStarObservations')
361 filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
362 if os.path.exists(filename):
363 inStarObsCat = starObsDataRef.get()
364 else:
365 inStarObsCat = None
367 rad = calibFluxApertureRadius
368 sourceSchemaDataRef = butler.dataRef('src_schema')
369 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
370 visitCat,
371 sourceSchemaDataRef,
372 camera,
373 calibFluxApertureRadius=rad,
374 starObsDataRef=starObsDataRef,
375 visitCatDataRef=visitCatDataRef,
376 inStarObsCat=inStarObsCat)
377 visitCatDataRef.put(visitCat)
378 starObsDataRef.put(fgcmStarObservationCat)
380 # Always do the matching.
381 if self.config.doReferenceMatches:
382 lutDataRef = butler.dataRef('fgcmLookUpTable')
383 else:
384 lutDataRef = None
385 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
386 fgcmStarObservationCat,
387 lutDataRef=lutDataRef)
389 # Persist catalogs via the butler
390 butler.put(fgcmStarIdCat, 'fgcmStarIds')
391 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
392 if fgcmRefCat is not None:
393 butler.put(fgcmRefCat, 'fgcmReferenceStars')
395 @abc.abstractmethod
396 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
397 """
398 Find and group dataRefs (by visit); Gen2 only.
400 Parameters
401 ----------
402 butler : `lsst.daf.persistence.Butler`
403 Gen2 butler.
404 camera : `lsst.afw.cameraGeom.Camera`
405 Camera from the butler.
406 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
407 Data references for the input visits.
409 Returns
410 -------
411 groupedDataRefs : `dict` [`int`, `list`]
412 Dictionary with sorted visit keys, and `list`s of
413 `lsst.daf.persistence.ButlerDataRef`
414 """
415 raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.")
417 @abc.abstractmethod
418 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
419 sourceSchemaDataRef,
420 camera,
421 calibFluxApertureRadius=None,
422 visitCatDataRef=None,
423 starObsDataRef=None,
424 inStarObsCat=None):
425 """
426 Compile all good star observations from visits in visitCat. Checkpoint files
427 will be stored if both visitCatDataRef and starObsDataRef are not None.
429 Parameters
430 ----------
431 groupedDataRefs: `dict` of `list`s
432 Lists of `~lsst.daf.persistence.ButlerDataRef` or
433 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
434 visitCat: `~afw.table.BaseCatalog`
435 Catalog with visit data for FGCM
436 sourceSchemaDataRef: `~lsst.daf.persistence.ButlerDataRef` or
437 `~lsst.daf.butler.DeferredDatasetHandle`
438 DataRef for the schema of the src catalogs.
439 camera: `~lsst.afw.cameraGeom.Camera`
440 calibFluxApertureRadius: `float`, optional
441 Aperture radius for calibration flux.
442 visitCatDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
443 Dataref to write visitCat for checkpoints
444 starObsDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
445 Dataref to write the star observation catalog for checkpoints.
446 inStarObsCat: `~afw.table.BaseCatalog`
447 Input observation catalog. If this is incomplete, observations
448 will be appended from when it was cut off.
450 Returns
451 -------
452 fgcmStarObservations: `afw.table.BaseCatalog`
453 Full catalog of good observations.
455 Raises
456 ------
457 RuntimeError: Raised if doSubtractLocalBackground is True and
458 calibFluxApertureRadius is not set.
459 """
460 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
462 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
463 visitCatDataRef=None, inVisitCat=None):
464 """
465 Make a visit catalog with all the keys from each visit
467 Parameters
468 ----------
469 camera: `lsst.afw.cameraGeom.Camera`
470 Camera from the butler
471 groupedDataRefs: `dict`
472 Dictionary with visit keys, and `list`s of
473 `lsst.daf.persistence.ButlerDataRef`
474 bkgDataRefDict: `dict`, optional
475 Dictionary of gen3 dataRefHandles for background info.
476 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
477 Dataref to write visitCat for checkpoints
478 inVisitCat: `afw.table.BaseCatalog`, optional
479 Input (possibly incomplete) visit catalog
481 Returns
482 -------
483 visitCat: `afw.table.BaseCatalog`
484 """
486 self.log.info("Assembling visitCatalog from %d %ss" %
487 (len(groupedDataRefs), self.config.visitDataRefName))
489 nCcd = len(camera)
491 if inVisitCat is None:
492 schema = self._makeFgcmVisitSchema(nCcd)
494 visitCat = afwTable.BaseCatalog(schema)
495 visitCat.reserve(len(groupedDataRefs))
496 visitCat.resize(len(groupedDataRefs))
498 visitCat['visit'] = list(groupedDataRefs.keys())
499 visitCat['used'] = 0
500 visitCat['sources_read'] = False
501 else:
502 visitCat = inVisitCat
504 # No matter what, fill the catalog. This will check if it was
505 # already read.
506 self._fillVisitCatalog(visitCat, groupedDataRefs,
507 bkgDataRefDict=bkgDataRefDict,
508 visitCatDataRef=visitCatDataRef)
510 return visitCat
512 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
513 visitCatDataRef=None):
514 """
515 Fill the visit catalog with visit metadata
517 Parameters
518 ----------
519 visitCat: `afw.table.BaseCatalog`
520 Catalog with schema from _makeFgcmVisitSchema()
521 groupedDataRefs: `dict`
522 Dictionary with visit keys, and `list`s of
523 `lsst.daf.persistence.ButlerDataRef`
524 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
525 Dataref to write visitCat for checkpoints
526 bkgDataRefDict: `dict`, optional
527 Dictionary of gen3 dataRefHandles for background info. FIXME
528 """
529 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
531 for i, visit in enumerate(groupedDataRefs):
532 # We don't use the bypasses since we need the psf info which does
533 # not have a bypass
534 # TODO: When DM-15500 is implemented in the Gen3 Butler, this
535 # can be fixed
537 # Do not read those that have already been read
538 if visitCat['used'][i]:
539 continue
541 if (i % self.config.nVisitsPerCheckpoint) == 0:
542 self.log.info("Retrieving metadata for %s %d (%d/%d)" %
543 (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
544 # Save checkpoint if desired
545 if visitCatDataRef is not None:
546 visitCatDataRef.put(visitCat)
548 dataRef = groupedDataRefs[visit][0]
549 if isinstance(dataRef, dafPersist.ButlerDataRef):
550 # Gen2: calexp dataRef
551 # The first dataRef in the group will be the reference ccd (if available)
552 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
553 visitInfo = exp.getInfo().getVisitInfo()
554 label = dataRef.get(datasetType='calexp_filterLabel')
555 physicalFilter = label.physicalLabel
556 psf = exp.getPsf()
557 psfSigma = psf.computeShape().getDeterminantRadius()
558 else:
559 # Gen3: use the visitSummary dataRef
560 summary = dataRef.get()
562 summaryRow = summary.find(self.config.referenceCCD)
563 if summaryRow is None:
564 # Take the first available ccd if reference isn't available
565 summaryRow = summary[0]
567 visitInfo = summaryRow.getVisitInfo()
568 physicalFilter = summaryRow['physical_filter']
569 # Compute the median psf sigma if possible
570 goodSigma, = np.where(summary['psfSigma'] > 0)
571 if goodSigma.size > 2:
572 psfSigma = np.median(summary['psfSigma'][goodSigma])
573 elif goodSigma > 0:
574 psfSigma = np.mean(summary['psfSigma'][goodSigma])
575 else:
576 psfSigma = 0.0
578 rec = visitCat[i]
579 rec['visit'] = visit
580 rec['physicalFilter'] = physicalFilter
581 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
582 # much easier to get the wcs's necessary to recompute the pointing
583 # ra/dec at the center of the camera.
584 radec = visitInfo.getBoresightRaDec()
585 rec['telra'] = radec.getRa().asDegrees()
586 rec['teldec'] = radec.getDec().asDegrees()
587 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
588 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
589 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
590 rec['exptime'] = visitInfo.getExposureTime()
591 # convert from Pa to millibar
592 # Note that I don't know if this unit will need to be per-camera config
593 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
594 # Flag to signify if this is a "deep" field. Not currently used
595 rec['deepFlag'] = 0
596 # Relative flat scaling (1.0 means no relative scaling)
597 rec['scaling'][:] = 1.0
598 # Median delta aperture, to be measured from stars
599 rec['deltaAper'] = 0.0
600 rec['psfSigma'] = psfSigma
602 if self.config.doModelErrorsWithBackground:
603 foundBkg = False
604 if isinstance(dataRef, dafPersist.ButlerDataRef):
605 det = dataRef.dataId[self.config.ccdDataRefName]
606 if dataRef.datasetExists(datasetType='calexpBackground'):
607 bgList = dataRef.get(datasetType='calexpBackground')
608 foundBkg = True
609 else:
610 det = dataRef.dataId['detector']
611 try:
612 bkgRef = bkgDataRefDict[(visit, det)]
613 bgList = bkgRef.get()
614 foundBkg = True
615 except KeyError:
616 pass
618 if foundBkg:
619 bgStats = (bg[0].getStatsImage().getImage().array
620 for bg in bgList)
621 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
622 else:
623 self.log.warn('Sky background not found for visit %d / ccd %d' %
624 (visit, det))
625 rec['skyBackground'] = -1.0
626 else:
627 rec['skyBackground'] = -1.0
629 rec['used'] = 1
631 def _makeSourceMapper(self, sourceSchema):
632 """
633 Make a schema mapper for fgcm sources
635 Parameters
636 ----------
637 sourceSchema: `afwTable.Schema`
638 Default source schema from the butler
640 Returns
641 -------
642 sourceMapper: `afwTable.schemaMapper`
643 Mapper to the FGCM source schema
644 """
646 # create a mapper to the preferred output
647 sourceMapper = afwTable.SchemaMapper(sourceSchema)
649 # map to ra/dec
650 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
651 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
652 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
653 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
654 # Add the mapping if the field exists in the input catalog.
655 # If the field does not exist, simply add it (set to False).
656 # This field is not required for calibration, but is useful
657 # to collate if available.
658 try:
659 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
660 'psf_candidate')
661 except LookupError:
662 sourceMapper.editOutputSchema().addField(
663 "psf_candidate", type='Flag',
664 doc=("Flag set if the source was a candidate for PSF determination, "
665 "as determined by the star selector."))
667 # and add the fields we want
668 sourceMapper.editOutputSchema().addField(
669 "visit", type=np.int32, doc="Visit number")
670 sourceMapper.editOutputSchema().addField(
671 "ccd", type=np.int32, doc="CCD number")
672 sourceMapper.editOutputSchema().addField(
673 "instMag", type=np.float32, doc="Instrumental magnitude")
674 sourceMapper.editOutputSchema().addField(
675 "instMagErr", type=np.float32, doc="Instrumental magnitude error")
676 sourceMapper.editOutputSchema().addField(
677 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
678 sourceMapper.editOutputSchema().addField(
679 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
681 return sourceMapper
683 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
684 """
685 Use FGCM code to match observations into unique stars.
687 Parameters
688 ----------
689 visitCat: `afw.table.BaseCatalog`
690 Catalog with visit data for fgcm
691 obsCat: `afw.table.BaseCatalog`
692 Full catalog of star observations for fgcm
693 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
694 `lsst.daf.butler.DeferredDatasetHandle`, optional
695 Data reference to fgcm look-up table (used if matching reference stars).
697 Returns
698 -------
699 fgcmStarIdCat: `afw.table.BaseCatalog`
700 Catalog of unique star identifiers and index keys
701 fgcmStarIndicesCat: `afwTable.BaseCatalog`
702 Catalog of unique star indices
703 fgcmRefCat: `afw.table.BaseCatalog`
704 Catalog of matched reference stars.
705 Will be None if `config.doReferenceMatches` is False.
706 """
707 # get filter names into a numpy array...
708 # This is the type that is expected by the fgcm code
709 visitFilterNames = np.zeros(len(visitCat), dtype='a30')
710 for i in range(len(visitCat)):
711 visitFilterNames[i] = visitCat[i]['physicalFilter']
713 # match to put filterNames with observations
714 visitIndex = np.searchsorted(visitCat['visit'],
715 obsCat['visit'])
717 obsFilterNames = visitFilterNames[visitIndex]
719 if self.config.doReferenceMatches:
720 # Get the reference filter names, using the LUT
721 lutCat = lutDataRef.get()
723 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
724 zip(lutCat[0]['physicalFilters'].split(','),
725 lutCat[0]['stdPhysicalFilters'].split(','))}
726 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
727 zip(lutCat[0]['stdPhysicalFilters'].split(','),
728 lutCat[0]['lambdaStdFilter'])}
730 del lutCat
732 referenceFilterNames = self._getReferenceFilterNames(visitCat,
733 stdFilterDict,
734 stdLambdaDict)
735 self.log.info("Using the following reference filters: %s" %
736 (', '.join(referenceFilterNames)))
738 else:
739 # This should be an empty list
740 referenceFilterNames = []
742 # make the fgcm starConfig dict
743 starConfig = {'logger': self.log,
744 'filterToBand': self.config.physicalFilterMap,
745 'requiredBands': self.config.requiredBands,
746 'minPerBand': self.config.minPerBand,
747 'matchRadius': self.config.matchRadius,
748 'isolationRadius': self.config.isolationRadius,
749 'matchNSide': self.config.matchNside,
750 'coarseNSide': self.config.coarseNside,
751 'densNSide': self.config.densityCutNside,
752 'densMaxPerPixel': self.config.densityCutMaxPerPixel,
753 'primaryBands': self.config.primaryBands,
754 'referenceFilterNames': referenceFilterNames}
756 # initialize the FgcmMakeStars object
757 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
759 # make the primary stars
760 # note that the ra/dec native Angle format is radians
761 # We determine the conversion from the native units (typically
762 # radians) to degrees for the first observation. This allows us
763 # to treate ra/dec as numpy arrays rather than Angles, which would
764 # be approximately 600x slower.
765 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
766 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
767 obsCat['dec'] * conv,
768 filterNameArray=obsFilterNames,
769 bandSelected=False)
771 # and match all the stars
772 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
773 obsCat['dec'] * conv,
774 obsFilterNames)
776 if self.config.doReferenceMatches:
777 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
779 # now persist
781 objSchema = self._makeFgcmObjSchema()
783 # make catalog and records
784 fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
785 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
786 for i in range(fgcmMakeStars.objIndexCat.size):
787 fgcmStarIdCat.addNew()
789 # fill the catalog
790 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
791 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
792 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
793 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
794 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
796 obsSchema = self._makeFgcmObsSchema()
798 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
799 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
800 for i in range(fgcmMakeStars.obsIndexCat.size):
801 fgcmStarIndicesCat.addNew()
803 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
805 if self.config.doReferenceMatches:
806 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
808 fgcmRefCat = afwTable.BaseCatalog(refSchema)
809 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
811 for i in range(fgcmMakeStars.referenceCat.size):
812 fgcmRefCat.addNew()
814 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
815 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
816 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
818 md = PropertyList()
819 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
820 md.set("FILTERNAMES", referenceFilterNames)
821 fgcmRefCat.setMetadata(md)
823 else:
824 fgcmRefCat = None
826 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
828 def _makeFgcmVisitSchema(self, nCcd):
829 """
830 Make a schema for an fgcmVisitCatalog
832 Parameters
833 ----------
834 nCcd: `int`
835 Number of CCDs in the camera
837 Returns
838 -------
839 schema: `afwTable.Schema`
840 """
842 schema = afwTable.Schema()
843 schema.addField('visit', type=np.int32, doc="Visit number")
844 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
845 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
846 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
847 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
848 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
849 schema.addField('mjd', type=np.float64, doc="MJD of visit")
850 schema.addField('exptime', type=np.float32, doc="Exposure time")
851 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
852 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
853 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
854 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
855 # the following field is not used yet
856 schema.addField('deepFlag', type=np.int32, doc="Deep observation")
857 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
858 size=nCcd)
859 schema.addField('used', type=np.int32, doc="This visit has been ingested.")
860 schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
862 return schema
864 def _makeFgcmObjSchema(self):
865 """
866 Make a schema for the objIndexCat from fgcmMakeStars
868 Returns
869 -------
870 schema: `afwTable.Schema`
871 """
873 objSchema = afwTable.Schema()
874 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
875 # Will investigate making these angles...
876 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
877 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
878 objSchema.addField('obsArrIndex', type=np.int32,
879 doc='Index in obsIndexTable for first observation')
880 objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
882 return objSchema
884 def _makeFgcmObsSchema(self):
885 """
886 Make a schema for the obsIndexCat from fgcmMakeStars
888 Returns
889 -------
890 schema: `afwTable.Schema`
891 """
893 obsSchema = afwTable.Schema()
894 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
896 return obsSchema
898 def _makeFgcmRefSchema(self, nReferenceBands):
899 """
900 Make a schema for the referenceCat from fgcmMakeStars
902 Parameters
903 ----------
904 nReferenceBands: `int`
905 Number of reference bands
907 Returns
908 -------
909 schema: `afwTable.Schema`
910 """
912 refSchema = afwTable.Schema()
913 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
914 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
915 size=nReferenceBands)
916 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
917 size=nReferenceBands)
919 return refSchema
921 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
922 """
923 Get the reference filter names, in wavelength order, from the visitCat and
924 information from the look-up-table.
926 Parameters
927 ----------
928 visitCat: `afw.table.BaseCatalog`
929 Catalog with visit data for FGCM
930 stdFilterDict: `dict`
931 Mapping of filterName to stdFilterName from LUT
932 stdLambdaDict: `dict`
933 Mapping of stdFilterName to stdLambda from LUT
935 Returns
936 -------
937 referenceFilterNames: `list`
938 Wavelength-ordered list of reference filter names
939 """
941 # Find the unique list of filter names in visitCat
942 filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
944 # Find the unique list of "standard" filters
945 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
947 # And sort these by wavelength
948 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
950 return referenceFilterNames