Coverage for python/lsst/fgcmcal/fgcmBuildStarsBase.py : 18%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for BuildStars using src tables or sourceTable_visit tables.
22"""
24import os
25import sys
26import traceback
27import abc
29import numpy as np
31import lsst.pex.config as pexConfig
32import lsst.pipe.base as pipeBase
33import lsst.afw.table as afwTable
34import lsst.geom as geom
35from lsst.daf.base import PropertyList
36from lsst.daf.base.dateTime import DateTime
37from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
39from .utilities import computeApertureRadiusFromDataRef
40from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
42import fgcm
44REFSTARS_FORMAT_VERSION = 1
46__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
49class FgcmBuildStarsConfigBase(pexConfig.Config):
50 """Base config for FgcmBuildStars tasks"""
52 instFluxField = pexConfig.Field(
53 doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
54 "The associated flag will be implicitly included in badFlags"),
55 dtype=str,
56 default='slot_CalibFlux_instFlux',
57 )
58 minPerBand = pexConfig.Field(
59 doc="Minimum observations per band",
60 dtype=int,
61 default=2,
62 )
63 matchRadius = pexConfig.Field(
64 doc="Match radius (arcseconds)",
65 dtype=float,
66 default=1.0,
67 )
68 isolationRadius = pexConfig.Field(
69 doc="Isolation radius (arcseconds)",
70 dtype=float,
71 default=2.0,
72 )
73 densityCutNside = pexConfig.Field(
74 doc="Density cut healpix nside",
75 dtype=int,
76 default=128,
77 )
78 densityCutMaxPerPixel = pexConfig.Field(
79 doc="Density cut number of stars per pixel",
80 dtype=int,
81 default=1000,
82 )
83 matchNside = pexConfig.Field(
84 doc="Healpix Nside for matching",
85 dtype=int,
86 default=4096,
87 )
88 coarseNside = pexConfig.Field(
89 doc="Healpix coarse Nside for partitioning matches",
90 dtype=int,
91 default=8,
92 )
93 filterMap = pexConfig.DictField(
94 doc="Mapping from 'filterName' to band.",
95 keytype=str,
96 itemtype=str,
97 default={},
98 )
99 requiredBands = pexConfig.ListField(
100 doc="Bands required for each star",
101 dtype=str,
102 default=(),
103 )
104 primaryBands = pexConfig.ListField(
105 doc=("Bands for 'primary' star matches. "
106 "A star must be observed in one of these bands to be considered "
107 "as a calibration star."),
108 dtype=str,
109 default=None
110 )
111 visitDataRefName = pexConfig.Field(
112 doc="dataRef name for the 'visit' field, usually 'visit'.",
113 dtype=str,
114 default="visit"
115 )
116 ccdDataRefName = pexConfig.Field(
117 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
118 dtype=str,
119 default="ccd"
120 )
121 doApplyWcsJacobian = pexConfig.Field(
122 doc="Apply the jacobian of the WCS to the star observations prior to fit?",
123 dtype=bool,
124 default=True
125 )
126 psfCandidateName = pexConfig.Field(
127 doc="Name of field with psf candidate flag for propagation",
128 dtype=str,
129 default="calib_psf_candidate"
130 )
131 doSubtractLocalBackground = pexConfig.Field(
132 doc=("Subtract the local background before performing calibration? "
133 "This is only supported for circular aperture calibration fluxes."),
134 dtype=bool,
135 default=False
136 )
137 localBackgroundFluxField = pexConfig.Field(
138 doc="Full name of the local background instFlux field to use.",
139 dtype=str,
140 default='base_LocalBackground_instFlux'
141 )
142 sourceSelector = sourceSelectorRegistry.makeField(
143 doc="How to select sources",
144 default="science"
145 )
146 apertureInnerInstFluxField = pexConfig.Field(
147 doc=("Full name of instFlux field that contains inner aperture "
148 "flux for aperture correction proxy"),
149 dtype=str,
150 default='base_CircularApertureFlux_12_0_instFlux'
151 )
152 apertureOuterInstFluxField = pexConfig.Field(
153 doc=("Full name of instFlux field that contains outer aperture "
154 "flux for aperture correction proxy"),
155 dtype=str,
156 default='base_CircularApertureFlux_17_0_instFlux'
157 )
158 doReferenceMatches = pexConfig.Field(
159 doc="Match reference catalog as additional constraint on calibration",
160 dtype=bool,
161 default=True,
162 )
163 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
164 target=FgcmLoadReferenceCatalogTask,
165 doc="FGCM reference object loader",
166 )
167 nVisitsPerCheckpoint = pexConfig.Field(
168 doc="Number of visits read between checkpoints",
169 dtype=int,
170 default=500,
171 )
173 def setDefaults(self):
174 sourceSelector = self.sourceSelector["science"]
175 sourceSelector.setDefaults()
177 sourceSelector.doFlags = True
178 sourceSelector.doUnresolved = True
179 sourceSelector.doSignalToNoise = True
180 sourceSelector.doIsolated = True
182 sourceSelector.signalToNoise.minimum = 10.0
183 sourceSelector.signalToNoise.maximum = 1000.0
185 # FGCM operates on unresolved sources, and this setting is
186 # appropriate for the current base_ClassificationExtendedness
187 sourceSelector.unresolved.maximum = 0.5
190class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
191 """Subclass of TaskRunner for FgcmBuildStars tasks
193 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
194 arguments, one of which is the butler (for persistence and mapper data),
195 and a list of dataRefs extracted from the command line. Note that FGCM
196 runs on a large set of dataRefs, and not on single dataRef/tract/patch.
197 This class transforms the process arguments generated by the ArgumentParser
198 into the arguments expected by FgcmBuildStarsTask.run(). This runner does
199 not use any parallelization.
200 """
201 @staticmethod
202 def getTargetList(parsedCmd):
203 """
204 Return a list with one element: a tuple with the butler and
205 list of dataRefs
206 """
207 # we want to combine the butler with any (or no!) dataRefs
208 return [(parsedCmd.butler, parsedCmd.id.refList)]
210 def __call__(self, args):
211 """
212 Parameters
213 ----------
214 args: `tuple` with (butler, dataRefList)
216 Returns
217 -------
218 exitStatus: `list` with `lsst.pipe.base.Struct`
219 exitStatus (0: success; 1: failure)
220 """
221 butler, dataRefList = args
223 task = self.TaskClass(config=self.config, log=self.log)
225 exitStatus = 0
226 if self.doRaise:
227 task.runDataRef(butler, dataRefList)
228 else:
229 try:
230 task.runDataRef(butler, dataRefList)
231 except Exception as e:
232 exitStatus = 1
233 task.log.fatal("Failed: %s" % e)
234 if not isinstance(e, pipeBase.TaskError):
235 traceback.print_exc(file=sys.stderr)
237 task.writeMetadata(butler)
239 # The task does not return any results:
240 return [pipeBase.Struct(exitStatus=exitStatus)]
242 def run(self, parsedCmd):
243 """
244 Run the task, with no multiprocessing
246 Parameters
247 ----------
248 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
249 """
251 resultList = []
253 if self.precall(parsedCmd):
254 targetList = self.getTargetList(parsedCmd)
255 resultList = self(targetList[0])
257 return resultList
260class FgcmBuildStarsBaseTask(pipeBase.CmdLineTask, abc.ABC):
261 """
262 Base task to build stars for FGCM global calibration
264 Parameters
265 ----------
266 butler : `lsst.daf.persistence.Butler`
267 """
268 def __init__(self, butler=None, **kwargs):
269 pipeBase.CmdLineTask.__init__(self, **kwargs)
270 self.makeSubtask("sourceSelector")
271 # Only log warning and fatal errors from the sourceSelector
272 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
274 # no saving of metadata for now
275 def _getMetadataName(self):
276 return None
278 @pipeBase.timeMethod
279 def runDataRef(self, butler, dataRefs):
280 """
281 Cross-match and make star list for FGCM Input
283 Parameters
284 ----------
285 butler: `lsst.daf.persistence.Butler`
286 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
287 Source data references for the input visits.
289 Raises
290 ------
291 RuntimeErrror: Raised if `config.doReferenceMatches` is set and
292 an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
293 fails if the calibFlux is not a CircularAperture flux.
294 """
295 datasetType = dataRefs[0].butlerSubset.datasetType
296 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
298 if self.config.doReferenceMatches:
299 # Ensure that we have a LUT
300 if not butler.datasetExists('fgcmLookUpTable'):
301 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
302 # Compute aperture radius if necessary. This is useful to do now before
303 # any heavy lifting has happened (fail early).
304 calibFluxApertureRadius = None
305 if self.config.doSubtractLocalBackground:
306 try:
307 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
308 self.config.instFluxField)
309 except RuntimeError as e:
310 raise RuntimeError("Could not determine aperture radius from %s. "
311 "Cannot use doSubtractLocalBackground." %
312 (self.config.instFluxField)) from e
314 groupedDataRefs = self.findAndGroupDataRefs(butler, dataRefs)
316 camera = butler.get('camera')
318 # Make the visit catalog if necessary
319 # First check if the visit catalog is in the _current_ path
320 # We cannot use Gen2 datasetExists() because that checks all parent
321 # directories as well, which would make recovering from faults
322 # and fgcmcal reruns impossible.
323 visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
324 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
325 if os.path.exists(filename):
326 # This file exists and we should continue processing
327 inVisitCat = visitCatDataRef.get()
328 if len(inVisitCat) != len(groupedDataRefs):
329 raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
330 "number of visits. Cannot continue.")
331 else:
332 inVisitCat = None
334 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
335 visitCatDataRef=visitCatDataRef,
336 inVisitCat=inVisitCat)
338 # Persist the visitCat as a checkpoint file.
339 visitCatDataRef.put(visitCat)
341 starObsDataRef = butler.dataRef('fgcmStarObservations')
342 filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
343 if os.path.exists(filename):
344 inStarObsCat = starObsDataRef.get()
345 else:
346 inStarObsCat = None
348 rad = calibFluxApertureRadius
349 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
350 visitCat,
351 calibFluxApertureRadius=rad,
352 starObsDataRef=starObsDataRef,
353 visitCatDataRef=visitCatDataRef,
354 inStarObsCat=inStarObsCat)
355 visitCatDataRef.put(visitCat)
356 starObsDataRef.put(fgcmStarObservationCat)
358 # Always do the matching.
359 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(butler,
360 visitCat,
361 fgcmStarObservationCat)
363 # Persist catalogs via the butler
364 butler.put(fgcmStarIdCat, 'fgcmStarIds')
365 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
366 if fgcmRefCat is not None:
367 butler.put(fgcmRefCat, 'fgcmReferenceStars')
369 @abc.abstractmethod
370 def findAndGroupDataRefs(self, butler, dataRefs):
371 """
372 Find and group dataRefs (by visit).
374 Parameters
375 ----------
376 butler: `lsst.daf.persistence.Butler`
377 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
378 Data references for the input visits.
380 Returns
381 -------
382 groupedDataRefs: `dict` [`int`, `list`]
383 Dictionary with visit keys, and `list`s of `lsst.daf.persistence.ButlerDataRef`
384 """
385 raise NotImplementedError("findAndGroupDataRefs not implemented.")
387 @abc.abstractmethod
388 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
389 calibFluxApertureRadius=None,
390 visitCatDataRef=None,
391 starObsDataRef=None,
392 inStarObsCat=None):
393 """
394 Compile all good star observations from visits in visitCat. Checkpoint files
395 will be stored if both visitCatDataRef and starObsDataRef are not None.
397 Parameters
398 ----------
399 groupedDataRefs: `dict` of `list`s
400 Lists of `lsst.daf.persistence.ButlerDataRef`, grouped by visit.
401 visitCat: `afw.table.BaseCatalog`
402 Catalog with visit data for FGCM
403 calibFluxApertureRadius: `float`, optional
404 Aperture radius for calibration flux.
405 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
406 Dataref to write visitCat for checkpoints
407 starObsDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
408 Dataref to write the star observation catalog for checkpoints.
409 inStarObsCat: `afw.table.BaseCatalog`
410 Input observation catalog. If this is incomplete, observations
411 will be appended from when it was cut off.
413 Returns
414 -------
415 fgcmStarObservations: `afw.table.BaseCatalog`
416 Full catalog of good observations.
418 Raises
419 ------
420 RuntimeError: Raised if doSubtractLocalBackground is True and
421 calibFluxApertureRadius is not set.
422 """
423 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
425 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs,
426 visitCatDataRef=None, inVisitCat=None):
427 """
428 Make a visit catalog with all the keys from each visit
430 Parameters
431 ----------
432 camera: `lsst.afw.cameraGeom.Camera`
433 Camera from the butler
434 groupedDataRefs: `dict`
435 Dictionary with visit keys, and `list`s of
436 `lsst.daf.persistence.ButlerDataRef`
437 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
438 Dataref to write visitCat for checkpoints
439 inVisitCat: `afw.table.BaseCatalog`
440 Input (possibly incomplete) visit catalog
442 Returns
443 -------
444 visitCat: `afw.table.BaseCatalog`
445 """
447 self.log.info("Assembling visitCatalog from %d %ss" %
448 (len(groupedDataRefs), self.config.visitDataRefName))
450 nCcd = len(camera)
452 if inVisitCat is None:
453 schema = self._makeFgcmVisitSchema(nCcd)
455 visitCat = afwTable.BaseCatalog(schema)
456 visitCat.reserve(len(groupedDataRefs))
458 for i, visit in enumerate(sorted(groupedDataRefs)):
459 rec = visitCat.addNew()
460 rec['visit'] = visit
461 rec['used'] = 0
462 rec['sources_read'] = 0
463 else:
464 visitCat = inVisitCat
466 # No matter what, fill the catalog. This will check if it was
467 # already read.
468 self._fillVisitCatalog(visitCat, groupedDataRefs,
469 visitCatDataRef=visitCatDataRef)
471 return visitCat
473 def _fillVisitCatalog(self, visitCat, groupedDataRefs,
474 visitCatDataRef=None):
475 """
476 Fill the visit catalog with visit metadata
478 Parameters
479 ----------
480 visitCat: `afw.table.BaseCatalog`
481 Catalog with schema from _makeFgcmVisitSchema()
482 groupedDataRefs: `dict`
483 Dictionary with visit keys, and `list`s of
484 `lsst.daf.persistence.ButlerDataRef
485 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
486 Dataref to write visitCat for checkpoints
487 """
489 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
491 for i, visit in enumerate(sorted(groupedDataRefs)):
492 # We don't use the bypasses since we need the psf info which does
493 # not have a bypass
494 # TODO: When DM-15500 is implemented in the Gen3 Butler, this
495 # can be fixed
497 # Do not read those that have already been read
498 if visitCat['used'][i]:
499 continue
501 if (i % self.config.nVisitsPerCheckpoint) == 0:
502 self.log.info("Retrieving metadata for %s %d (%d/%d)" %
503 (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
504 # Save checkpoint if desired
505 if visitCatDataRef is not None:
506 visitCatDataRef.put(visitCat)
508 # Note that the reference ccd is first in the list (if available).
510 # The first dataRef in the group will be the reference ccd (if available)
511 dataRef = groupedDataRefs[visit][0]
513 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox,
514 flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
516 visitInfo = exp.getInfo().getVisitInfo()
517 f = exp.getFilter()
518 psf = exp.getPsf()
520 rec = visitCat[i]
521 rec['visit'] = visit
522 rec['filtername'] = f.getName()
523 radec = visitInfo.getBoresightRaDec()
524 rec['telra'] = radec.getRa().asDegrees()
525 rec['teldec'] = radec.getDec().asDegrees()
526 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
527 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
528 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
529 rec['exptime'] = visitInfo.getExposureTime()
530 # convert from Pa to millibar
531 # Note that I don't know if this unit will need to be per-camera config
532 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
533 # Flag to signify if this is a "deep" field. Not currently used
534 rec['deepFlag'] = 0
535 # Relative flat scaling (1.0 means no relative scaling)
536 rec['scaling'][:] = 1.0
537 # Median delta aperture, to be measured from stars
538 rec['deltaAper'] = 0.0
540 rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
542 if dataRef.datasetExists(datasetType='calexpBackground'):
543 # Get background for reference CCD
544 # This approximation is good enough for now
545 bgStats = (bg[0].getStatsImage().getImage().array
546 for bg in dataRef.get(datasetType='calexpBackground'))
547 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
548 else:
549 self.log.warn('Sky background not found for visit %d / ccd %d' %
550 (visit, dataRef.dataId[self.config.ccdDataRefName]))
551 rec['skyBackground'] = -1.0
553 rec['used'] = 1
555 def _makeSourceMapper(self, sourceSchema):
556 """
557 Make a schema mapper for fgcm sources
559 Parameters
560 ----------
561 sourceSchema: `afwTable.Schema`
562 Default source schema from the butler
564 Returns
565 -------
566 sourceMapper: `afwTable.schemaMapper`
567 Mapper to the FGCM source schema
568 """
570 # create a mapper to the preferred output
571 sourceMapper = afwTable.SchemaMapper(sourceSchema)
573 # map to ra/dec
574 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
575 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
576 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
577 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
578 # Add the mapping if the field exists in the input catalog.
579 # If the field does not exist, simply add it (set to False).
580 # This field is not required for calibration, but is useful
581 # to collate if available.
582 try:
583 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
584 'psf_candidate')
585 except LookupError:
586 sourceMapper.editOutputSchema().addField(
587 "psf_candidate", type='Flag',
588 doc=("Flag set if the source was a candidate for PSF determination, "
589 "as determined by the star selector."))
591 # and add the fields we want
592 sourceMapper.editOutputSchema().addField(
593 "visit", type=np.int32, doc="Visit number")
594 sourceMapper.editOutputSchema().addField(
595 "ccd", type=np.int32, doc="CCD number")
596 sourceMapper.editOutputSchema().addField(
597 "instMag", type=np.float32, doc="Instrumental magnitude")
598 sourceMapper.editOutputSchema().addField(
599 "instMagErr", type=np.float32, doc="Instrumental magnitude error")
600 sourceMapper.editOutputSchema().addField(
601 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
602 sourceMapper.editOutputSchema().addField(
603 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
605 return sourceMapper
607 def fgcmMatchStars(self, butler, visitCat, obsCat):
608 """
609 Use FGCM code to match observations into unique stars.
611 Parameters
612 ----------
613 butler: `lsst.daf.persistence.Butler`
614 visitCat: `afw.table.BaseCatalog`
615 Catalog with visit data for fgcm
616 obsCat: `afw.table.BaseCatalog`
617 Full catalog of star observations for fgcm
619 Returns
620 -------
621 fgcmStarIdCat: `afw.table.BaseCatalog`
622 Catalog of unique star identifiers and index keys
623 fgcmStarIndicesCat: `afwTable.BaseCatalog`
624 Catalog of unique star indices
625 fgcmRefCat: `afw.table.BaseCatalog`
626 Catalog of matched reference stars.
627 Will be None if `config.doReferenceMatches` is False.
628 """
630 if self.config.doReferenceMatches:
631 # Make a subtask for reference loading
632 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
634 # get filter names into a numpy array...
635 # This is the type that is expected by the fgcm code
636 visitFilterNames = np.zeros(len(visitCat), dtype='a10')
637 for i in range(len(visitCat)):
638 visitFilterNames[i] = visitCat[i]['filtername']
640 # match to put filterNames with observations
641 visitIndex = np.searchsorted(visitCat['visit'],
642 obsCat['visit'])
644 obsFilterNames = visitFilterNames[visitIndex]
646 if self.config.doReferenceMatches:
647 # Get the reference filter names, using the LUT
648 lutCat = butler.get('fgcmLookUpTable')
650 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
651 zip(lutCat[0]['filterNames'].split(','),
652 lutCat[0]['stdFilterNames'].split(','))}
653 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
654 zip(lutCat[0]['stdFilterNames'].split(','),
655 lutCat[0]['lambdaStdFilter'])}
657 del lutCat
659 referenceFilterNames = self._getReferenceFilterNames(visitCat,
660 stdFilterDict,
661 stdLambdaDict)
662 self.log.info("Using the following reference filters: %s" %
663 (', '.join(referenceFilterNames)))
665 else:
666 # This should be an empty list
667 referenceFilterNames = []
669 # make the fgcm starConfig dict
671 starConfig = {'logger': self.log,
672 'filterToBand': self.config.filterMap,
673 'requiredBands': self.config.requiredBands,
674 'minPerBand': self.config.minPerBand,
675 'matchRadius': self.config.matchRadius,
676 'isolationRadius': self.config.isolationRadius,
677 'matchNSide': self.config.matchNside,
678 'coarseNSide': self.config.coarseNside,
679 'densNSide': self.config.densityCutNside,
680 'densMaxPerPixel': self.config.densityCutMaxPerPixel,
681 'primaryBands': self.config.primaryBands,
682 'referenceFilterNames': referenceFilterNames}
684 # initialize the FgcmMakeStars object
685 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
687 # make the primary stars
688 # note that the ra/dec native Angle format is radians
689 # We determine the conversion from the native units (typically
690 # radians) to degrees for the first observation. This allows us
691 # to treate ra/dec as numpy arrays rather than Angles, which would
692 # be approximately 600x slower.
693 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
694 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
695 obsCat['dec'] * conv,
696 filterNameArray=obsFilterNames,
697 bandSelected=False)
699 # and match all the stars
700 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
701 obsCat['dec'] * conv,
702 obsFilterNames)
704 if self.config.doReferenceMatches:
705 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
707 # now persist
709 objSchema = self._makeFgcmObjSchema()
711 # make catalog and records
712 fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
713 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
714 for i in range(fgcmMakeStars.objIndexCat.size):
715 fgcmStarIdCat.addNew()
717 # fill the catalog
718 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
719 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
720 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
721 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
722 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
724 obsSchema = self._makeFgcmObsSchema()
726 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
727 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
728 for i in range(fgcmMakeStars.obsIndexCat.size):
729 fgcmStarIndicesCat.addNew()
731 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
733 if self.config.doReferenceMatches:
734 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
736 fgcmRefCat = afwTable.BaseCatalog(refSchema)
737 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
739 for i in range(fgcmMakeStars.referenceCat.size):
740 fgcmRefCat.addNew()
742 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
743 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
744 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
746 md = PropertyList()
747 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
748 md.set("FILTERNAMES", referenceFilterNames)
749 fgcmRefCat.setMetadata(md)
751 else:
752 fgcmRefCat = None
754 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
756 def _makeFgcmVisitSchema(self, nCcd):
757 """
758 Make a schema for an fgcmVisitCatalog
760 Parameters
761 ----------
762 nCcd: `int`
763 Number of CCDs in the camera
765 Returns
766 -------
767 schema: `afwTable.Schema`
768 """
770 schema = afwTable.Schema()
771 schema.addField('visit', type=np.int32, doc="Visit number")
772 # Note that the FGCM code currently handles filternames up to 2 characters long
773 schema.addField('filtername', type=str, size=10, doc="Filter name")
774 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
775 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
776 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
777 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
778 schema.addField('mjd', type=np.float64, doc="MJD of visit")
779 schema.addField('exptime', type=np.float32, doc="Exposure time")
780 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
781 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
782 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
783 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
784 # the following field is not used yet
785 schema.addField('deepFlag', type=np.int32, doc="Deep observation")
786 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
787 size=nCcd)
788 schema.addField('used', type=np.int32, doc="This visit has been ingested.")
789 schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
791 return schema
793 def _makeFgcmObjSchema(self):
794 """
795 Make a schema for the objIndexCat from fgcmMakeStars
797 Returns
798 -------
799 schema: `afwTable.Schema`
800 """
802 objSchema = afwTable.Schema()
803 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
804 # Will investigate making these angles...
805 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
806 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
807 objSchema.addField('obsArrIndex', type=np.int32,
808 doc='Index in obsIndexTable for first observation')
809 objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
811 return objSchema
813 def _makeFgcmObsSchema(self):
814 """
815 Make a schema for the obsIndexCat from fgcmMakeStars
817 Returns
818 -------
819 schema: `afwTable.Schema`
820 """
822 obsSchema = afwTable.Schema()
823 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
825 return obsSchema
827 def _makeFgcmRefSchema(self, nReferenceBands):
828 """
829 Make a schema for the referenceCat from fgcmMakeStars
831 Parameters
832 ----------
833 nReferenceBands: `int`
834 Number of reference bands
836 Returns
837 -------
838 schema: `afwTable.Schema`
839 """
841 refSchema = afwTable.Schema()
842 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
843 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
844 size=nReferenceBands)
845 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
846 size=nReferenceBands)
848 return refSchema
850 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
851 """
852 Get the reference filter names, in wavelength order, from the visitCat and
853 information from the look-up-table.
855 Parameters
856 ----------
857 visitCat: `afw.table.BaseCatalog`
858 Catalog with visit data for FGCM
859 stdFilterDict: `dict`
860 Mapping of filterName to stdFilterName from LUT
861 stdLambdaDict: `dict`
862 Mapping of stdFilterName to stdLambda from LUT
864 Returns
865 -------
866 referenceFilterNames: `list`
867 Wavelength-ordered list of reference filter names
868 """
870 # Find the unique list of filter names in visitCat
871 filterNames = np.unique(visitCat.asAstropy()['filtername'])
873 # Find the unique list of "standard" filters
874 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
876 # And sort these by wavelength
877 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
879 return referenceFilterNames