Coverage for python/lsst/fgcmcal/fgcmBuildStarsBase.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for BuildStars using src tables or sourceTable_visit tables.
22"""
24import os
25import sys
26import traceback
27import abc
29import numpy as np
31import lsst.daf.persistence as dafPersist
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34import lsst.afw.table as afwTable
35import lsst.geom as geom
36from lsst.daf.base import PropertyList
37from lsst.daf.base.dateTime import DateTime
38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
40from .utilities import computeApertureRadiusFromDataRef
41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
43import fgcm
45REFSTARS_FORMAT_VERSION = 1
47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
50class FgcmBuildStarsConfigBase(pexConfig.Config):
51 """Base config for FgcmBuildStars tasks"""
53 instFluxField = pexConfig.Field(
54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55 "The associated flag will be implicitly included in badFlags"),
56 dtype=str,
57 default='slot_CalibFlux_instFlux',
58 )
59 minPerBand = pexConfig.Field(
60 doc="Minimum observations per band",
61 dtype=int,
62 default=2,
63 )
64 matchRadius = pexConfig.Field(
65 doc="Match radius (arcseconds)",
66 dtype=float,
67 default=1.0,
68 )
69 isolationRadius = pexConfig.Field(
70 doc="Isolation radius (arcseconds)",
71 dtype=float,
72 default=2.0,
73 )
74 densityCutNside = pexConfig.Field(
75 doc="Density cut healpix nside",
76 dtype=int,
77 default=128,
78 )
79 densityCutMaxPerPixel = pexConfig.Field(
80 doc="Density cut number of stars per pixel",
81 dtype=int,
82 default=1000,
83 )
84 randomSeed = pexConfig.Field(
85 doc="Random seed for high density down-sampling.",
86 dtype=int,
87 default=None,
88 optional=True,
89 )
90 matchNside = pexConfig.Field(
91 doc="Healpix Nside for matching",
92 dtype=int,
93 default=4096,
94 )
95 coarseNside = pexConfig.Field(
96 doc="Healpix coarse Nside for partitioning matches",
97 dtype=int,
98 default=8,
99 )
100 # The following config will not be necessary after Gen2 retirement.
101 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
102 # which is easiest to access in the config file.
103 physicalFilterMap = pexConfig.DictField(
104 doc="Mapping from 'physicalFilter' to band.",
105 keytype=str,
106 itemtype=str,
107 default={},
108 )
109 requiredBands = pexConfig.ListField(
110 doc="Bands required for each star",
111 dtype=str,
112 default=(),
113 )
114 primaryBands = pexConfig.ListField(
115 doc=("Bands for 'primary' star matches. "
116 "A star must be observed in one of these bands to be considered "
117 "as a calibration star."),
118 dtype=str,
119 default=None
120 )
121 visitDataRefName = pexConfig.Field(
122 doc="dataRef name for the 'visit' field, usually 'visit'.",
123 dtype=str,
124 default="visit"
125 )
126 ccdDataRefName = pexConfig.Field(
127 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
128 dtype=str,
129 default="ccd"
130 )
131 doApplyWcsJacobian = pexConfig.Field(
132 doc="Apply the jacobian of the WCS to the star observations prior to fit?",
133 dtype=bool,
134 default=True
135 )
136 doModelErrorsWithBackground = pexConfig.Field(
137 doc="Model flux errors with background term?",
138 dtype=bool,
139 default=True
140 )
141 psfCandidateName = pexConfig.Field(
142 doc="Name of field with psf candidate flag for propagation",
143 dtype=str,
144 default="calib_psf_candidate"
145 )
146 doSubtractLocalBackground = pexConfig.Field(
147 doc=("Subtract the local background before performing calibration? "
148 "This is only supported for circular aperture calibration fluxes."),
149 dtype=bool,
150 default=False
151 )
152 localBackgroundFluxField = pexConfig.Field(
153 doc="Full name of the local background instFlux field to use.",
154 dtype=str,
155 default='base_LocalBackground_instFlux'
156 )
157 sourceSelector = sourceSelectorRegistry.makeField(
158 doc="How to select sources",
159 default="science"
160 )
161 apertureInnerInstFluxField = pexConfig.Field(
162 doc=("Full name of instFlux field that contains inner aperture "
163 "flux for aperture correction proxy"),
164 dtype=str,
165 default='base_CircularApertureFlux_12_0_instFlux'
166 )
167 apertureOuterInstFluxField = pexConfig.Field(
168 doc=("Full name of instFlux field that contains outer aperture "
169 "flux for aperture correction proxy"),
170 dtype=str,
171 default='base_CircularApertureFlux_17_0_instFlux'
172 )
173 doReferenceMatches = pexConfig.Field(
174 doc="Match reference catalog as additional constraint on calibration",
175 dtype=bool,
176 default=True,
177 )
178 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
179 target=FgcmLoadReferenceCatalogTask,
180 doc="FGCM reference object loader",
181 )
182 nVisitsPerCheckpoint = pexConfig.Field(
183 doc="Number of visits read between checkpoints",
184 dtype=int,
185 default=500,
186 )
188 def setDefaults(self):
189 sourceSelector = self.sourceSelector["science"]
190 sourceSelector.setDefaults()
192 sourceSelector.doFlags = True
193 sourceSelector.doUnresolved = True
194 sourceSelector.doSignalToNoise = True
195 sourceSelector.doIsolated = True
197 sourceSelector.signalToNoise.minimum = 10.0
198 sourceSelector.signalToNoise.maximum = 1000.0
200 # FGCM operates on unresolved sources, and this setting is
201 # appropriate for the current base_ClassificationExtendedness
202 sourceSelector.unresolved.maximum = 0.5
205class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
206 """Subclass of TaskRunner for FgcmBuildStars tasks
208 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
209 arguments, one of which is the butler (for persistence and mapper data),
210 and a list of dataRefs extracted from the command line. Note that FGCM
211 runs on a large set of dataRefs, and not on single dataRef/tract/patch.
212 This class transforms the process arguments generated by the ArgumentParser
213 into the arguments expected by FgcmBuildStarsTask.run(). This runner does
214 not use any parallelization.
215 """
216 @staticmethod
217 def getTargetList(parsedCmd):
218 """
219 Return a list with one element: a tuple with the butler and
220 list of dataRefs
221 """
222 # we want to combine the butler with any (or no!) dataRefs
223 return [(parsedCmd.butler, parsedCmd.id.refList)]
225 def __call__(self, args):
226 """
227 Parameters
228 ----------
229 args: `tuple` with (butler, dataRefList)
231 Returns
232 -------
233 exitStatus: `list` with `lsst.pipe.base.Struct`
234 exitStatus (0: success; 1: failure)
235 """
236 butler, dataRefList = args
238 task = self.TaskClass(config=self.config, log=self.log)
240 exitStatus = 0
241 if self.doRaise:
242 task.runDataRef(butler, dataRefList)
243 else:
244 try:
245 task.runDataRef(butler, dataRefList)
246 except Exception as e:
247 exitStatus = 1
248 task.log.fatal("Failed: %s" % e)
249 if not isinstance(e, pipeBase.TaskError):
250 traceback.print_exc(file=sys.stderr)
252 task.writeMetadata(butler)
254 # The task does not return any results:
255 return [pipeBase.Struct(exitStatus=exitStatus)]
257 def run(self, parsedCmd):
258 """
259 Run the task, with no multiprocessing
261 Parameters
262 ----------
263 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
264 """
266 resultList = []
268 if self.precall(parsedCmd):
269 targetList = self.getTargetList(parsedCmd)
270 resultList = self(targetList[0])
272 return resultList
275class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
276 """
277 Base task to build stars for FGCM global calibration
279 Parameters
280 ----------
281 butler : `lsst.daf.persistence.Butler`
282 """
283 def __init__(self, initInputs=None, butler=None, **kwargs):
284 super().__init__(**kwargs)
286 self.makeSubtask("sourceSelector")
287 # Only log warning and fatal errors from the sourceSelector
288 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
290 # no saving of metadata for now
291 def _getMetadataName(self):
292 return None
294 @pipeBase.timeMethod
295 def runDataRef(self, butler, dataRefs):
296 """
297 Cross-match and make star list for FGCM Input
299 Parameters
300 ----------
301 butler: `lsst.daf.persistence.Butler`
302 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
303 Source data references for the input visits.
305 Raises
306 ------
307 RuntimeErrror: Raised if `config.doReferenceMatches` is set and
308 an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
309 fails if the calibFlux is not a CircularAperture flux.
310 """
311 datasetType = dataRefs[0].butlerSubset.datasetType
312 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
314 if self.config.doReferenceMatches:
315 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
316 # Ensure that we have a LUT
317 if not butler.datasetExists('fgcmLookUpTable'):
318 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
319 # Compute aperture radius if necessary. This is useful to do now before
320 # any heavy lifting has happened (fail early).
321 calibFluxApertureRadius = None
322 if self.config.doSubtractLocalBackground:
323 try:
324 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
325 self.config.instFluxField)
326 except RuntimeError as e:
327 raise RuntimeError("Could not determine aperture radius from %s. "
328 "Cannot use doSubtractLocalBackground." %
329 (self.config.instFluxField)) from e
331 camera = butler.get('camera')
332 groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs)
334 # Make the visit catalog if necessary
335 # First check if the visit catalog is in the _current_ path
336 # We cannot use Gen2 datasetExists() because that checks all parent
337 # directories as well, which would make recovering from faults
338 # and fgcmcal reruns impossible.
339 visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
340 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
341 if os.path.exists(filename):
342 # This file exists and we should continue processing
343 inVisitCat = visitCatDataRef.get()
344 if len(inVisitCat) != len(groupedDataRefs):
345 raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
346 "number of visits. Cannot continue.")
347 else:
348 inVisitCat = None
350 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
351 visitCatDataRef=visitCatDataRef,
352 inVisitCat=inVisitCat)
354 # Persist the visitCat as a checkpoint file.
355 visitCatDataRef.put(visitCat)
357 starObsDataRef = butler.dataRef('fgcmStarObservations')
358 filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
359 if os.path.exists(filename):
360 inStarObsCat = starObsDataRef.get()
361 else:
362 inStarObsCat = None
364 rad = calibFluxApertureRadius
365 sourceSchemaDataRef = butler.dataRef('src_schema')
366 sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema
367 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
368 visitCat,
369 sourceSchema,
370 camera,
371 calibFluxApertureRadius=rad,
372 starObsDataRef=starObsDataRef,
373 visitCatDataRef=visitCatDataRef,
374 inStarObsCat=inStarObsCat)
375 visitCatDataRef.put(visitCat)
376 starObsDataRef.put(fgcmStarObservationCat)
378 # Always do the matching.
379 if self.config.doReferenceMatches:
380 lutDataRef = butler.dataRef('fgcmLookUpTable')
381 else:
382 lutDataRef = None
383 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
384 fgcmStarObservationCat,
385 lutDataRef=lutDataRef)
387 # Persist catalogs via the butler
388 butler.put(fgcmStarIdCat, 'fgcmStarIds')
389 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
390 if fgcmRefCat is not None:
391 butler.put(fgcmRefCat, 'fgcmReferenceStars')
393 @abc.abstractmethod
394 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
395 """
396 Find and group dataRefs (by visit); Gen2 only.
398 Parameters
399 ----------
400 butler : `lsst.daf.persistence.Butler`
401 Gen2 butler.
402 camera : `lsst.afw.cameraGeom.Camera`
403 Camera from the butler.
404 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
405 Data references for the input visits.
407 Returns
408 -------
409 groupedDataRefs : `dict` [`int`, `list`]
410 Dictionary with sorted visit keys, and `list`s of
411 `lsst.daf.persistence.ButlerDataRef`
412 """
413 raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.")
415 @abc.abstractmethod
416 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
417 sourceSchema,
418 camera,
419 calibFluxApertureRadius=None,
420 visitCatDataRef=None,
421 starObsDataRef=None,
422 inStarObsCat=None):
423 """
424 Compile all good star observations from visits in visitCat. Checkpoint files
425 will be stored if both visitCatDataRef and starObsDataRef are not None.
427 Parameters
428 ----------
429 groupedDataRefs : `dict` of `list`s
430 Lists of `~lsst.daf.persistence.ButlerDataRef` or
431 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
432 visitCat : `~afw.table.BaseCatalog`
433 Catalog with visit data for FGCM
434 sourceSchema : `~lsst.afw.table.Schema`
435 Schema for the input src catalogs.
436 camera : `~lsst.afw.cameraGeom.Camera`
437 calibFluxApertureRadius : `float`, optional
438 Aperture radius for calibration flux.
439 visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
440 Dataref to write visitCat for checkpoints
441 starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
442 Dataref to write the star observation catalog for checkpoints.
443 inStarObsCat : `~afw.table.BaseCatalog`
444 Input observation catalog. If this is incomplete, observations
445 will be appended from when it was cut off.
447 Returns
448 -------
449 fgcmStarObservations : `afw.table.BaseCatalog`
450 Full catalog of good observations.
452 Raises
453 ------
454 RuntimeError: Raised if doSubtractLocalBackground is True and
455 calibFluxApertureRadius is not set.
456 """
457 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
459 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
460 visitCatDataRef=None, inVisitCat=None):
461 """
462 Make a visit catalog with all the keys from each visit
464 Parameters
465 ----------
466 camera: `lsst.afw.cameraGeom.Camera`
467 Camera from the butler
468 groupedDataRefs: `dict`
469 Dictionary with visit keys, and `list`s of
470 `lsst.daf.persistence.ButlerDataRef`
471 bkgDataRefDict: `dict`, optional
472 Dictionary of gen3 dataRefHandles for background info.
473 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
474 Dataref to write visitCat for checkpoints
475 inVisitCat: `afw.table.BaseCatalog`, optional
476 Input (possibly incomplete) visit catalog
478 Returns
479 -------
480 visitCat: `afw.table.BaseCatalog`
481 """
483 self.log.info("Assembling visitCatalog from %d %ss" %
484 (len(groupedDataRefs), self.config.visitDataRefName))
486 nCcd = len(camera)
488 if inVisitCat is None:
489 schema = self._makeFgcmVisitSchema(nCcd)
491 visitCat = afwTable.BaseCatalog(schema)
492 visitCat.reserve(len(groupedDataRefs))
493 visitCat.resize(len(groupedDataRefs))
495 visitCat['visit'] = list(groupedDataRefs.keys())
496 visitCat['used'] = 0
497 visitCat['sources_read'] = False
498 else:
499 visitCat = inVisitCat
501 # No matter what, fill the catalog. This will check if it was
502 # already read.
503 self._fillVisitCatalog(visitCat, groupedDataRefs,
504 bkgDataRefDict=bkgDataRefDict,
505 visitCatDataRef=visitCatDataRef)
507 return visitCat
509 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
510 visitCatDataRef=None):
511 """
512 Fill the visit catalog with visit metadata
514 Parameters
515 ----------
516 visitCat : `afw.table.BaseCatalog`
517 Visit catalog. See _makeFgcmVisitSchema() for schema definition.
518 groupedDataRefs : `dict`
519 Dictionary with visit keys, and `list`s of
520 `lsst.daf.persistence.ButlerDataRef` or
521 `lsst.daf.butler.DeferredDatasetHandle`
522 visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional
523 Dataref to write ``visitCat`` for checkpoints. Gen2 only.
524 bkgDataRefDict : `dict`, optional
525 Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle`
526 for background info.
527 """
528 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
530 for i, visit in enumerate(groupedDataRefs):
531 # We don't use the bypasses since we need the psf info which does
532 # not have a bypass
533 # TODO: When DM-15500 is implemented in the Gen3 Butler, this
534 # can be fixed
536 # Do not read those that have already been read
537 if visitCat['used'][i]:
538 continue
540 if (i % self.config.nVisitsPerCheckpoint) == 0:
541 self.log.info("Retrieving metadata for %s %d (%d/%d)" %
542 (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
543 # Save checkpoint if desired
544 if visitCatDataRef is not None:
545 visitCatDataRef.put(visitCat)
547 dataRef = groupedDataRefs[visit][0]
548 if isinstance(dataRef, dafPersist.ButlerDataRef):
549 # Gen2: calexp dataRef
550 # The first dataRef in the group will be the reference ccd (if available)
551 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
552 visitInfo = exp.getInfo().getVisitInfo()
553 label = dataRef.get(datasetType='calexp_filterLabel')
554 physicalFilter = label.physicalLabel
555 psf = exp.getPsf()
556 psfSigma = psf.computeShape().getDeterminantRadius()
557 else:
558 # Gen3: use the visitSummary dataRef
559 summary = dataRef.get()
561 summaryRow = summary.find(self.config.referenceCCD)
562 if summaryRow is None:
563 # Take the first available ccd if reference isn't available
564 summaryRow = summary[0]
566 summaryDetector = summaryRow['id']
567 visitInfo = summaryRow.getVisitInfo()
568 physicalFilter = summaryRow['physical_filter']
569 # Compute the median psf sigma if possible
570 goodSigma, = np.where(summary['psfSigma'] > 0)
571 if goodSigma.size > 2:
572 psfSigma = np.median(summary['psfSigma'][goodSigma])
573 elif goodSigma > 0:
574 psfSigma = np.mean(summary['psfSigma'][goodSigma])
575 else:
576 psfSigma = 0.0
578 rec = visitCat[i]
579 rec['visit'] = visit
580 rec['physicalFilter'] = physicalFilter
581 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
582 # much easier to get the wcs's necessary to recompute the pointing
583 # ra/dec at the center of the camera.
584 radec = visitInfo.getBoresightRaDec()
585 rec['telra'] = radec.getRa().asDegrees()
586 rec['teldec'] = radec.getDec().asDegrees()
587 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
588 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
589 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
590 rec['exptime'] = visitInfo.getExposureTime()
591 # convert from Pa to millibar
592 # Note that I don't know if this unit will need to be per-camera config
593 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
594 # Flag to signify if this is a "deep" field. Not currently used
595 rec['deepFlag'] = 0
596 # Relative flat scaling (1.0 means no relative scaling)
597 rec['scaling'][:] = 1.0
598 # Median delta aperture, to be measured from stars
599 rec['deltaAper'] = 0.0
600 rec['psfSigma'] = psfSigma
602 if self.config.doModelErrorsWithBackground:
603 foundBkg = False
604 if isinstance(dataRef, dafPersist.ButlerDataRef):
605 # Gen2-style dataRef
606 det = dataRef.dataId[self.config.ccdDataRefName]
607 if dataRef.datasetExists(datasetType='calexpBackground'):
608 bgList = dataRef.get(datasetType='calexpBackground')
609 foundBkg = True
610 else:
611 # Gen3-style dataRef
612 try:
613 # Use the same detector used from the summary.
614 bkgRef = bkgDataRefDict[(visit, summaryDetector)]
615 bgList = bkgRef.get()
616 foundBkg = True
617 except KeyError:
618 pass
620 if foundBkg:
621 bgStats = (bg[0].getStatsImage().getImage().array
622 for bg in bgList)
623 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
624 else:
625 self.log.warning('Sky background not found for visit %d / ccd %d' %
626 (visit, det))
627 rec['skyBackground'] = -1.0
628 else:
629 rec['skyBackground'] = -1.0
631 rec['used'] = 1
633 def _makeSourceMapper(self, sourceSchema):
634 """
635 Make a schema mapper for fgcm sources
637 Parameters
638 ----------
639 sourceSchema: `afwTable.Schema`
640 Default source schema from the butler
642 Returns
643 -------
644 sourceMapper: `afwTable.schemaMapper`
645 Mapper to the FGCM source schema
646 """
648 # create a mapper to the preferred output
649 sourceMapper = afwTable.SchemaMapper(sourceSchema)
651 # map to ra/dec
652 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
653 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
654 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
655 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
656 # Add the mapping if the field exists in the input catalog.
657 # If the field does not exist, simply add it (set to False).
658 # This field is not required for calibration, but is useful
659 # to collate if available.
660 try:
661 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
662 'psf_candidate')
663 except LookupError:
664 sourceMapper.editOutputSchema().addField(
665 "psf_candidate", type='Flag',
666 doc=("Flag set if the source was a candidate for PSF determination, "
667 "as determined by the star selector."))
669 # and add the fields we want
670 sourceMapper.editOutputSchema().addField(
671 "visit", type=np.int32, doc="Visit number")
672 sourceMapper.editOutputSchema().addField(
673 "ccd", type=np.int32, doc="CCD number")
674 sourceMapper.editOutputSchema().addField(
675 "instMag", type=np.float32, doc="Instrumental magnitude")
676 sourceMapper.editOutputSchema().addField(
677 "instMagErr", type=np.float32, doc="Instrumental magnitude error")
678 sourceMapper.editOutputSchema().addField(
679 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
680 sourceMapper.editOutputSchema().addField(
681 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
683 return sourceMapper
685 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
686 """
687 Use FGCM code to match observations into unique stars.
689 Parameters
690 ----------
691 visitCat: `afw.table.BaseCatalog`
692 Catalog with visit data for fgcm
693 obsCat: `afw.table.BaseCatalog`
694 Full catalog of star observations for fgcm
695 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
696 `lsst.daf.butler.DeferredDatasetHandle`, optional
697 Data reference to fgcm look-up table (used if matching reference stars).
699 Returns
700 -------
701 fgcmStarIdCat: `afw.table.BaseCatalog`
702 Catalog of unique star identifiers and index keys
703 fgcmStarIndicesCat: `afwTable.BaseCatalog`
704 Catalog of unique star indices
705 fgcmRefCat: `afw.table.BaseCatalog`
706 Catalog of matched reference stars.
707 Will be None if `config.doReferenceMatches` is False.
708 """
709 # get filter names into a numpy array...
710 # This is the type that is expected by the fgcm code
711 visitFilterNames = np.zeros(len(visitCat), dtype='a30')
712 for i in range(len(visitCat)):
713 visitFilterNames[i] = visitCat[i]['physicalFilter']
715 # match to put filterNames with observations
716 visitIndex = np.searchsorted(visitCat['visit'],
717 obsCat['visit'])
719 obsFilterNames = visitFilterNames[visitIndex]
721 if self.config.doReferenceMatches:
722 # Get the reference filter names, using the LUT
723 lutCat = lutDataRef.get()
725 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
726 zip(lutCat[0]['physicalFilters'].split(','),
727 lutCat[0]['stdPhysicalFilters'].split(','))}
728 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
729 zip(lutCat[0]['stdPhysicalFilters'].split(','),
730 lutCat[0]['lambdaStdFilter'])}
732 del lutCat
734 referenceFilterNames = self._getReferenceFilterNames(visitCat,
735 stdFilterDict,
736 stdLambdaDict)
737 self.log.info("Using the following reference filters: %s" %
738 (', '.join(referenceFilterNames)))
740 else:
741 # This should be an empty list
742 referenceFilterNames = []
744 # make the fgcm starConfig dict
745 starConfig = {'logger': self.log,
746 'filterToBand': self.config.physicalFilterMap,
747 'requiredBands': self.config.requiredBands,
748 'minPerBand': self.config.minPerBand,
749 'matchRadius': self.config.matchRadius,
750 'isolationRadius': self.config.isolationRadius,
751 'matchNSide': self.config.matchNside,
752 'coarseNSide': self.config.coarseNside,
753 'densNSide': self.config.densityCutNside,
754 'densMaxPerPixel': self.config.densityCutMaxPerPixel,
755 'randomSeed': self.config.randomSeed,
756 'primaryBands': self.config.primaryBands,
757 'referenceFilterNames': referenceFilterNames}
759 # initialize the FgcmMakeStars object
760 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
762 # make the primary stars
763 # note that the ra/dec native Angle format is radians
764 # We determine the conversion from the native units (typically
765 # radians) to degrees for the first observation. This allows us
766 # to treate ra/dec as numpy arrays rather than Angles, which would
767 # be approximately 600x slower.
768 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
769 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
770 obsCat['dec'] * conv,
771 filterNameArray=obsFilterNames,
772 bandSelected=False)
774 # and match all the stars
775 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
776 obsCat['dec'] * conv,
777 obsFilterNames)
779 if self.config.doReferenceMatches:
780 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
782 # now persist
784 objSchema = self._makeFgcmObjSchema()
786 # make catalog and records
787 fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
788 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
789 for i in range(fgcmMakeStars.objIndexCat.size):
790 fgcmStarIdCat.addNew()
792 # fill the catalog
793 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
794 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
795 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
796 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
797 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
799 obsSchema = self._makeFgcmObsSchema()
801 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
802 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
803 for i in range(fgcmMakeStars.obsIndexCat.size):
804 fgcmStarIndicesCat.addNew()
806 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
808 if self.config.doReferenceMatches:
809 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
811 fgcmRefCat = afwTable.BaseCatalog(refSchema)
812 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
814 for i in range(fgcmMakeStars.referenceCat.size):
815 fgcmRefCat.addNew()
817 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
818 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
819 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
821 md = PropertyList()
822 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
823 md.set("FILTERNAMES", referenceFilterNames)
824 fgcmRefCat.setMetadata(md)
826 else:
827 fgcmRefCat = None
829 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
831 def _makeFgcmVisitSchema(self, nCcd):
832 """
833 Make a schema for an fgcmVisitCatalog
835 Parameters
836 ----------
837 nCcd: `int`
838 Number of CCDs in the camera
840 Returns
841 -------
842 schema: `afwTable.Schema`
843 """
845 schema = afwTable.Schema()
846 schema.addField('visit', type=np.int32, doc="Visit number")
847 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
848 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
849 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
850 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
851 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
852 schema.addField('mjd', type=np.float64, doc="MJD of visit")
853 schema.addField('exptime', type=np.float32, doc="Exposure time")
854 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
855 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
856 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
857 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
858 # the following field is not used yet
859 schema.addField('deepFlag', type=np.int32, doc="Deep observation")
860 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
861 size=nCcd)
862 schema.addField('used', type=np.int32, doc="This visit has been ingested.")
863 schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
865 return schema
867 def _makeFgcmObjSchema(self):
868 """
869 Make a schema for the objIndexCat from fgcmMakeStars
871 Returns
872 -------
873 schema: `afwTable.Schema`
874 """
876 objSchema = afwTable.Schema()
877 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
878 # Will investigate making these angles...
879 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
880 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
881 objSchema.addField('obsArrIndex', type=np.int32,
882 doc='Index in obsIndexTable for first observation')
883 objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
885 return objSchema
887 def _makeFgcmObsSchema(self):
888 """
889 Make a schema for the obsIndexCat from fgcmMakeStars
891 Returns
892 -------
893 schema: `afwTable.Schema`
894 """
896 obsSchema = afwTable.Schema()
897 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
899 return obsSchema
901 def _makeFgcmRefSchema(self, nReferenceBands):
902 """
903 Make a schema for the referenceCat from fgcmMakeStars
905 Parameters
906 ----------
907 nReferenceBands: `int`
908 Number of reference bands
910 Returns
911 -------
912 schema: `afwTable.Schema`
913 """
915 refSchema = afwTable.Schema()
916 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
917 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
918 size=nReferenceBands)
919 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
920 size=nReferenceBands)
922 return refSchema
924 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
925 """
926 Get the reference filter names, in wavelength order, from the visitCat and
927 information from the look-up-table.
929 Parameters
930 ----------
931 visitCat: `afw.table.BaseCatalog`
932 Catalog with visit data for FGCM
933 stdFilterDict: `dict`
934 Mapping of filterName to stdFilterName from LUT
935 stdLambdaDict: `dict`
936 Mapping of stdFilterName to stdLambda from LUT
938 Returns
939 -------
940 referenceFilterNames: `list`
941 Wavelength-ordered list of reference filter names
942 """
944 # Find the unique list of filter names in visitCat
945 filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
947 # Find the unique list of "standard" filters
948 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
950 # And sort these by wavelength
951 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
953 return referenceFilterNames