Coverage for python/lsst/fgcmcal/fgcmBuildStarsBase.py : 18%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for BuildStars using src tables or sourceTable_visit tables.
22"""
24import os
25import sys
26import traceback
27import abc
29import numpy as np
31import lsst.daf.persistence as dafPersist
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34import lsst.afw.table as afwTable
35import lsst.geom as geom
36from lsst.daf.base import PropertyList
37from lsst.daf.base.dateTime import DateTime
38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
40from .utilities import computeApertureRadiusFromDataRef
41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
43import fgcm
45REFSTARS_FORMAT_VERSION = 1
47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
50class FgcmBuildStarsConfigBase(pexConfig.Config):
51 """Base config for FgcmBuildStars tasks"""
53 instFluxField = pexConfig.Field(
54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55 "The associated flag will be implicitly included in badFlags"),
56 dtype=str,
57 default='slot_CalibFlux_instFlux',
58 )
59 minPerBand = pexConfig.Field(
60 doc="Minimum observations per band",
61 dtype=int,
62 default=2,
63 )
64 matchRadius = pexConfig.Field(
65 doc="Match radius (arcseconds)",
66 dtype=float,
67 default=1.0,
68 )
69 isolationRadius = pexConfig.Field(
70 doc="Isolation radius (arcseconds)",
71 dtype=float,
72 default=2.0,
73 )
74 densityCutNside = pexConfig.Field(
75 doc="Density cut healpix nside",
76 dtype=int,
77 default=128,
78 )
79 densityCutMaxPerPixel = pexConfig.Field(
80 doc="Density cut number of stars per pixel",
81 dtype=int,
82 default=1000,
83 )
84 matchNside = pexConfig.Field(
85 doc="Healpix Nside for matching",
86 dtype=int,
87 default=4096,
88 )
89 coarseNside = pexConfig.Field(
90 doc="Healpix coarse Nside for partitioning matches",
91 dtype=int,
92 default=8,
93 )
94 filterMap = pexConfig.DictField(
95 doc="Mapping from 'filterName' to band.",
96 keytype=str,
97 itemtype=str,
98 default={},
99 )
100 requiredBands = pexConfig.ListField(
101 doc="Bands required for each star",
102 dtype=str,
103 default=(),
104 )
105 primaryBands = pexConfig.ListField(
106 doc=("Bands for 'primary' star matches. "
107 "A star must be observed in one of these bands to be considered "
108 "as a calibration star."),
109 dtype=str,
110 default=None
111 )
112 visitDataRefName = pexConfig.Field(
113 doc="dataRef name for the 'visit' field, usually 'visit'.",
114 dtype=str,
115 default="visit"
116 )
117 ccdDataRefName = pexConfig.Field(
118 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
119 dtype=str,
120 default="ccd"
121 )
122 doApplyWcsJacobian = pexConfig.Field(
123 doc="Apply the jacobian of the WCS to the star observations prior to fit?",
124 dtype=bool,
125 default=True
126 )
127 doModelErrorsWithBackground = pexConfig.Field(
128 doc="Model flux errors with background term?",
129 dtype=bool,
130 default=True
131 )
132 psfCandidateName = pexConfig.Field(
133 doc="Name of field with psf candidate flag for propagation",
134 dtype=str,
135 default="calib_psf_candidate"
136 )
137 doSubtractLocalBackground = pexConfig.Field(
138 doc=("Subtract the local background before performing calibration? "
139 "This is only supported for circular aperture calibration fluxes."),
140 dtype=bool,
141 default=False
142 )
143 localBackgroundFluxField = pexConfig.Field(
144 doc="Full name of the local background instFlux field to use.",
145 dtype=str,
146 default='base_LocalBackground_instFlux'
147 )
148 sourceSelector = sourceSelectorRegistry.makeField(
149 doc="How to select sources",
150 default="science"
151 )
152 apertureInnerInstFluxField = pexConfig.Field(
153 doc=("Full name of instFlux field that contains inner aperture "
154 "flux for aperture correction proxy"),
155 dtype=str,
156 default='base_CircularApertureFlux_12_0_instFlux'
157 )
158 apertureOuterInstFluxField = pexConfig.Field(
159 doc=("Full name of instFlux field that contains outer aperture "
160 "flux for aperture correction proxy"),
161 dtype=str,
162 default='base_CircularApertureFlux_17_0_instFlux'
163 )
164 doReferenceMatches = pexConfig.Field(
165 doc="Match reference catalog as additional constraint on calibration",
166 dtype=bool,
167 default=True,
168 )
169 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
170 target=FgcmLoadReferenceCatalogTask,
171 doc="FGCM reference object loader",
172 )
173 nVisitsPerCheckpoint = pexConfig.Field(
174 doc="Number of visits read between checkpoints",
175 dtype=int,
176 default=500,
177 )
179 def setDefaults(self):
180 sourceSelector = self.sourceSelector["science"]
181 sourceSelector.setDefaults()
183 sourceSelector.doFlags = True
184 sourceSelector.doUnresolved = True
185 sourceSelector.doSignalToNoise = True
186 sourceSelector.doIsolated = True
188 sourceSelector.signalToNoise.minimum = 10.0
189 sourceSelector.signalToNoise.maximum = 1000.0
191 # FGCM operates on unresolved sources, and this setting is
192 # appropriate for the current base_ClassificationExtendedness
193 sourceSelector.unresolved.maximum = 0.5
196class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
197 """Subclass of TaskRunner for FgcmBuildStars tasks
199 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
200 arguments, one of which is the butler (for persistence and mapper data),
201 and a list of dataRefs extracted from the command line. Note that FGCM
202 runs on a large set of dataRefs, and not on single dataRef/tract/patch.
203 This class transforms the process arguments generated by the ArgumentParser
204 into the arguments expected by FgcmBuildStarsTask.run(). This runner does
205 not use any parallelization.
206 """
207 @staticmethod
208 def getTargetList(parsedCmd):
209 """
210 Return a list with one element: a tuple with the butler and
211 list of dataRefs
212 """
213 # we want to combine the butler with any (or no!) dataRefs
214 return [(parsedCmd.butler, parsedCmd.id.refList)]
216 def __call__(self, args):
217 """
218 Parameters
219 ----------
220 args: `tuple` with (butler, dataRefList)
222 Returns
223 -------
224 exitStatus: `list` with `lsst.pipe.base.Struct`
225 exitStatus (0: success; 1: failure)
226 """
227 butler, dataRefList = args
229 task = self.TaskClass(config=self.config, log=self.log)
231 exitStatus = 0
232 if self.doRaise:
233 task.runDataRef(butler, dataRefList)
234 else:
235 try:
236 task.runDataRef(butler, dataRefList)
237 except Exception as e:
238 exitStatus = 1
239 task.log.fatal("Failed: %s" % e)
240 if not isinstance(e, pipeBase.TaskError):
241 traceback.print_exc(file=sys.stderr)
243 task.writeMetadata(butler)
245 # The task does not return any results:
246 return [pipeBase.Struct(exitStatus=exitStatus)]
248 def run(self, parsedCmd):
249 """
250 Run the task, with no multiprocessing
252 Parameters
253 ----------
254 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
255 """
257 resultList = []
259 if self.precall(parsedCmd):
260 targetList = self.getTargetList(parsedCmd)
261 resultList = self(targetList[0])
263 return resultList
266class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
267 """
268 Base task to build stars for FGCM global calibration
270 Parameters
271 ----------
272 butler : `lsst.daf.persistence.Butler`
273 """
274 def __init__(self, butler=None, initInputs=None, **kwargs):
275 super().__init__(**kwargs)
277 self.makeSubtask("sourceSelector")
278 # Only log warning and fatal errors from the sourceSelector
279 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
281 # no saving of metadata for now
282 def _getMetadataName(self):
283 return None
285 @pipeBase.timeMethod
286 def runDataRef(self, butler, dataRefs):
287 """
288 Cross-match and make star list for FGCM Input
290 Parameters
291 ----------
292 butler: `lsst.daf.persistence.Butler`
293 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
294 Source data references for the input visits.
296 Raises
297 ------
298 RuntimeErrror: Raised if `config.doReferenceMatches` is set and
299 an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
300 fails if the calibFlux is not a CircularAperture flux.
301 """
302 datasetType = dataRefs[0].butlerSubset.datasetType
303 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
305 if self.config.doReferenceMatches:
306 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
307 # Ensure that we have a LUT
308 if not butler.datasetExists('fgcmLookUpTable'):
309 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
310 # Compute aperture radius if necessary. This is useful to do now before
311 # any heavy lifting has happened (fail early).
312 calibFluxApertureRadius = None
313 if self.config.doSubtractLocalBackground:
314 try:
315 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
316 self.config.instFluxField)
317 except RuntimeError as e:
318 raise RuntimeError("Could not determine aperture radius from %s. "
319 "Cannot use doSubtractLocalBackground." %
320 (self.config.instFluxField)) from e
322 camera = butler.get('camera')
323 groupedDataRefs = self._findAndGroupDataRefs(camera, dataRefs, butler=butler)
325 # Make the visit catalog if necessary
326 # First check if the visit catalog is in the _current_ path
327 # We cannot use Gen2 datasetExists() because that checks all parent
328 # directories as well, which would make recovering from faults
329 # and fgcmcal reruns impossible.
330 visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
331 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
332 if os.path.exists(filename):
333 # This file exists and we should continue processing
334 inVisitCat = visitCatDataRef.get()
335 if len(inVisitCat) != len(groupedDataRefs):
336 raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
337 "number of visits. Cannot continue.")
338 else:
339 inVisitCat = None
341 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
342 visitCatDataRef=visitCatDataRef,
343 inVisitCat=inVisitCat)
345 # Persist the visitCat as a checkpoint file.
346 visitCatDataRef.put(visitCat)
348 starObsDataRef = butler.dataRef('fgcmStarObservations')
349 filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
350 if os.path.exists(filename):
351 inStarObsCat = starObsDataRef.get()
352 else:
353 inStarObsCat = None
355 rad = calibFluxApertureRadius
356 sourceSchemaDataRef = butler.dataRef('src_schema')
357 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
358 visitCat,
359 sourceSchemaDataRef,
360 camera,
361 calibFluxApertureRadius=rad,
362 starObsDataRef=starObsDataRef,
363 visitCatDataRef=visitCatDataRef,
364 inStarObsCat=inStarObsCat)
365 visitCatDataRef.put(visitCat)
366 starObsDataRef.put(fgcmStarObservationCat)
368 # Always do the matching.
369 if self.config.doReferenceMatches:
370 lutDataRef = butler.dataRef('fgcmLookUpTable')
371 else:
372 lutDataRef = None
373 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat,
374 fgcmStarObservationCat,
375 lutDataRef=lutDataRef)
377 # Persist catalogs via the butler
378 butler.put(fgcmStarIdCat, 'fgcmStarIds')
379 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
380 if fgcmRefCat is not None:
381 butler.put(fgcmRefCat, 'fgcmReferenceStars')
383 @abc.abstractmethod
384 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
385 """
386 Find and group dataRefs (by visit). For Gen2 usage, set butler, and for
387 Gen3, use calexpDataRefDict
389 Parameters
390 ----------
391 camera : `lsst.afw.cameraGeom.Camera`
392 Camera from the butler.
393 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` or
394 `lsst.daf.butler.DeferredDatasetHandle`
395 Data references for the input visits.
396 butler : `lsst.daf.persistence.Butler`, optional
397 Gen2 butler when used as CommandLineTask
398 calexpDataRefDict : `dict`, optional
399 Dictionary of Gen3 deferred data refs for calexps
401 Returns
402 -------
403 groupedDataRefs : `OrderedDict` [`int`, `list`]
404 Dictionary with sorted visit keys, and `list`s of
405 `lsst.daf.persistence.ButlerDataRef` or
406 `lsst.daf.butler.DeferredDatasetHandle`
408 Raises
409 ------
410 RuntimeError : Raised if neither or both of butler and dataRefDict are set.
411 """
412 raise NotImplementedError("_findAndGroupDataRefs not implemented.")
414 @abc.abstractmethod
415 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
416 sourceSchemaDataRef,
417 camera,
418 calibFluxApertureRadius=None,
419 visitCatDataRef=None,
420 starObsDataRef=None,
421 inStarObsCat=None):
422 """
423 Compile all good star observations from visits in visitCat. Checkpoint files
424 will be stored if both visitCatDataRef and starObsDataRef are not None.
426 Parameters
427 ----------
428 groupedDataRefs: `dict` of `list`s
429 Lists of `~lsst.daf.persistence.ButlerDataRef` or
430 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
431 visitCat: `~afw.table.BaseCatalog`
432 Catalog with visit data for FGCM
433 sourceSchemaDataRef: `~lsst.daf.persistence.ButlerDataRef` or
434 `~lsst.daf.butler.DeferredDatasetHandle`
435 DataRef for the schema of the src catalogs.
436 camera: `~lsst.afw.cameraGeom.Camera`
437 calibFluxApertureRadius: `float`, optional
438 Aperture radius for calibration flux.
439 visitCatDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
440 Dataref to write visitCat for checkpoints
441 starObsDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
442 Dataref to write the star observation catalog for checkpoints.
443 inStarObsCat: `~afw.table.BaseCatalog`
444 Input observation catalog. If this is incomplete, observations
445 will be appended from when it was cut off.
447 Returns
448 -------
449 fgcmStarObservations: `afw.table.BaseCatalog`
450 Full catalog of good observations.
452 Raises
453 ------
454 RuntimeError: Raised if doSubtractLocalBackground is True and
455 calibFluxApertureRadius is not set.
456 """
457 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
459 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
460 visitCatDataRef=None, inVisitCat=None):
461 """
462 Make a visit catalog with all the keys from each visit
464 Parameters
465 ----------
466 camera: `lsst.afw.cameraGeom.Camera`
467 Camera from the butler
468 groupedDataRefs: `dict`
469 Dictionary with visit keys, and `list`s of
470 `lsst.daf.persistence.ButlerDataRef`
471 bkgDataRefDict: `dict`, optional
472 Dictionary of gen3 dataRefHandles for background info.
473 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
474 Dataref to write visitCat for checkpoints
475 inVisitCat: `afw.table.BaseCatalog`, optional
476 Input (possibly incomplete) visit catalog
478 Returns
479 -------
480 visitCat: `afw.table.BaseCatalog`
481 """
483 self.log.info("Assembling visitCatalog from %d %ss" %
484 (len(groupedDataRefs), self.config.visitDataRefName))
486 nCcd = len(camera)
488 if inVisitCat is None:
489 schema = self._makeFgcmVisitSchema(nCcd)
491 visitCat = afwTable.BaseCatalog(schema)
492 visitCat.reserve(len(groupedDataRefs))
493 visitCat.resize(len(groupedDataRefs))
495 visitCat['visit'] = list(groupedDataRefs.keys())
496 visitCat['used'] = 0
497 visitCat['sources_read'] = False
498 else:
499 visitCat = inVisitCat
501 # No matter what, fill the catalog. This will check if it was
502 # already read.
503 self._fillVisitCatalog(visitCat, groupedDataRefs,
504 bkgDataRefDict=bkgDataRefDict,
505 visitCatDataRef=visitCatDataRef)
507 return visitCat
509 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
510 visitCatDataRef=None):
511 """
512 Fill the visit catalog with visit metadata
514 Parameters
515 ----------
516 visitCat: `afw.table.BaseCatalog`
517 Catalog with schema from _makeFgcmVisitSchema()
518 groupedDataRefs: `dict`
519 Dictionary with visit keys, and `list`s of
520 `lsst.daf.persistence.ButlerDataRef`
521 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
522 Dataref to write visitCat for checkpoints
523 bkgDataRefDict: `dict`, optional
524 Dictionary of gen3 dataRefHandles for background info. FIXME
525 """
526 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
528 for i, visit in enumerate(groupedDataRefs):
529 # We don't use the bypasses since we need the psf info which does
530 # not have a bypass
531 # TODO: When DM-15500 is implemented in the Gen3 Butler, this
532 # can be fixed
534 # Do not read those that have already been read
535 if visitCat['used'][i]:
536 continue
538 if (i % self.config.nVisitsPerCheckpoint) == 0:
539 self.log.info("Retrieving metadata for %s %d (%d/%d)" %
540 (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
541 # Save checkpoint if desired
542 if visitCatDataRef is not None:
543 visitCatDataRef.put(visitCat)
545 # Note that the reference ccd is first in the list (if available).
547 # The first dataRef in the group will be the reference ccd (if available)
548 dataRef = groupedDataRefs[visit][0]
549 if isinstance(dataRef, dafPersist.ButlerDataRef):
550 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
551 visitInfo = exp.getInfo().getVisitInfo()
552 f = exp.getFilter()
553 psf = exp.getPsf()
554 else:
555 visitInfo = dataRef.get(component='visitInfo')
556 f = dataRef.get(component='filter')
557 psf = dataRef.get(component='psf')
559 rec = visitCat[i]
560 rec['visit'] = visit
561 rec['filtername'] = f.getName()
562 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
563 # much easier to get the wcs's necessary to recompute the pointing
564 # ra/dec at the center of the camera.
565 radec = visitInfo.getBoresightRaDec()
566 rec['telra'] = radec.getRa().asDegrees()
567 rec['teldec'] = radec.getDec().asDegrees()
568 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
569 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
570 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
571 rec['exptime'] = visitInfo.getExposureTime()
572 # convert from Pa to millibar
573 # Note that I don't know if this unit will need to be per-camera config
574 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
575 # Flag to signify if this is a "deep" field. Not currently used
576 rec['deepFlag'] = 0
577 # Relative flat scaling (1.0 means no relative scaling)
578 rec['scaling'][:] = 1.0
579 # Median delta aperture, to be measured from stars
580 rec['deltaAper'] = 0.0
582 rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
584 if self.config.doModelErrorsWithBackground:
585 foundBkg = False
586 if isinstance(dataRef, dafPersist.ButlerDataRef):
587 det = dataRef.dataId[self.config.ccdDataRefName]
588 if dataRef.datasetExists(datasetType='calexpBackground'):
589 bgList = dataRef.get(datasetType='calexpBackground')
590 foundBkg = True
591 else:
592 det = dataRef.dataId.byName()['detector']
593 try:
594 bkgRef = bkgDataRefDict[(visit, det)]
595 bgList = bkgRef.get()
596 foundBkg = True
597 except KeyError:
598 pass
600 if foundBkg:
601 bgStats = (bg[0].getStatsImage().getImage().array
602 for bg in bgList)
603 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
604 else:
605 self.log.warn('Sky background not found for visit %d / ccd %d' %
606 (visit, det))
607 rec['skyBackground'] = -1.0
608 else:
609 rec['skyBackground'] = -1.0
611 rec['used'] = 1
613 def _makeSourceMapper(self, sourceSchema):
614 """
615 Make a schema mapper for fgcm sources
617 Parameters
618 ----------
619 sourceSchema: `afwTable.Schema`
620 Default source schema from the butler
622 Returns
623 -------
624 sourceMapper: `afwTable.schemaMapper`
625 Mapper to the FGCM source schema
626 """
628 # create a mapper to the preferred output
629 sourceMapper = afwTable.SchemaMapper(sourceSchema)
631 # map to ra/dec
632 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
633 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
634 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
635 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
636 # Add the mapping if the field exists in the input catalog.
637 # If the field does not exist, simply add it (set to False).
638 # This field is not required for calibration, but is useful
639 # to collate if available.
640 try:
641 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
642 'psf_candidate')
643 except LookupError:
644 sourceMapper.editOutputSchema().addField(
645 "psf_candidate", type='Flag',
646 doc=("Flag set if the source was a candidate for PSF determination, "
647 "as determined by the star selector."))
649 # and add the fields we want
650 sourceMapper.editOutputSchema().addField(
651 "visit", type=np.int32, doc="Visit number")
652 sourceMapper.editOutputSchema().addField(
653 "ccd", type=np.int32, doc="CCD number")
654 sourceMapper.editOutputSchema().addField(
655 "instMag", type=np.float32, doc="Instrumental magnitude")
656 sourceMapper.editOutputSchema().addField(
657 "instMagErr", type=np.float32, doc="Instrumental magnitude error")
658 sourceMapper.editOutputSchema().addField(
659 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
660 sourceMapper.editOutputSchema().addField(
661 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
663 return sourceMapper
665 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
666 """
667 Use FGCM code to match observations into unique stars.
669 Parameters
670 ----------
671 visitCat: `afw.table.BaseCatalog`
672 Catalog with visit data for fgcm
673 obsCat: `afw.table.BaseCatalog`
674 Full catalog of star observations for fgcm
675 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
676 `lsst.daf.butler.DeferredDatasetHandle`, optional
677 Data reference to fgcm look-up table (used if matching reference stars).
679 Returns
680 -------
681 fgcmStarIdCat: `afw.table.BaseCatalog`
682 Catalog of unique star identifiers and index keys
683 fgcmStarIndicesCat: `afwTable.BaseCatalog`
684 Catalog of unique star indices
685 fgcmRefCat: `afw.table.BaseCatalog`
686 Catalog of matched reference stars.
687 Will be None if `config.doReferenceMatches` is False.
688 """
689 # get filter names into a numpy array...
690 # This is the type that is expected by the fgcm code
691 visitFilterNames = np.zeros(len(visitCat), dtype='a10')
692 for i in range(len(visitCat)):
693 visitFilterNames[i] = visitCat[i]['filtername']
695 # match to put filterNames with observations
696 visitIndex = np.searchsorted(visitCat['visit'],
697 obsCat['visit'])
699 obsFilterNames = visitFilterNames[visitIndex]
701 if self.config.doReferenceMatches:
702 # Get the reference filter names, using the LUT
703 lutCat = lutDataRef.get()
705 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
706 zip(lutCat[0]['filterNames'].split(','),
707 lutCat[0]['stdFilterNames'].split(','))}
708 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
709 zip(lutCat[0]['stdFilterNames'].split(','),
710 lutCat[0]['lambdaStdFilter'])}
712 del lutCat
714 referenceFilterNames = self._getReferenceFilterNames(visitCat,
715 stdFilterDict,
716 stdLambdaDict)
717 self.log.info("Using the following reference filters: %s" %
718 (', '.join(referenceFilterNames)))
720 else:
721 # This should be an empty list
722 referenceFilterNames = []
724 # make the fgcm starConfig dict
726 starConfig = {'logger': self.log,
727 'filterToBand': self.config.filterMap,
728 'requiredBands': self.config.requiredBands,
729 'minPerBand': self.config.minPerBand,
730 'matchRadius': self.config.matchRadius,
731 'isolationRadius': self.config.isolationRadius,
732 'matchNSide': self.config.matchNside,
733 'coarseNSide': self.config.coarseNside,
734 'densNSide': self.config.densityCutNside,
735 'densMaxPerPixel': self.config.densityCutMaxPerPixel,
736 'primaryBands': self.config.primaryBands,
737 'referenceFilterNames': referenceFilterNames}
739 # initialize the FgcmMakeStars object
740 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
742 # make the primary stars
743 # note that the ra/dec native Angle format is radians
744 # We determine the conversion from the native units (typically
745 # radians) to degrees for the first observation. This allows us
746 # to treate ra/dec as numpy arrays rather than Angles, which would
747 # be approximately 600x slower.
748 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
749 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
750 obsCat['dec'] * conv,
751 filterNameArray=obsFilterNames,
752 bandSelected=False)
754 # and match all the stars
755 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
756 obsCat['dec'] * conv,
757 obsFilterNames)
759 if self.config.doReferenceMatches:
760 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
762 # now persist
764 objSchema = self._makeFgcmObjSchema()
766 # make catalog and records
767 fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
768 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
769 for i in range(fgcmMakeStars.objIndexCat.size):
770 fgcmStarIdCat.addNew()
772 # fill the catalog
773 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
774 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
775 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
776 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
777 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
779 obsSchema = self._makeFgcmObsSchema()
781 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
782 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
783 for i in range(fgcmMakeStars.obsIndexCat.size):
784 fgcmStarIndicesCat.addNew()
786 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
788 if self.config.doReferenceMatches:
789 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
791 fgcmRefCat = afwTable.BaseCatalog(refSchema)
792 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
794 for i in range(fgcmMakeStars.referenceCat.size):
795 fgcmRefCat.addNew()
797 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
798 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
799 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
801 md = PropertyList()
802 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
803 md.set("FILTERNAMES", referenceFilterNames)
804 fgcmRefCat.setMetadata(md)
806 else:
807 fgcmRefCat = None
809 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
811 def _makeFgcmVisitSchema(self, nCcd):
812 """
813 Make a schema for an fgcmVisitCatalog
815 Parameters
816 ----------
817 nCcd: `int`
818 Number of CCDs in the camera
820 Returns
821 -------
822 schema: `afwTable.Schema`
823 """
825 schema = afwTable.Schema()
826 schema.addField('visit', type=np.int32, doc="Visit number")
827 # Note that the FGCM code currently handles filternames up to 2 characters long
828 schema.addField('filtername', type=str, size=10, doc="Filter name")
829 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
830 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
831 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
832 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
833 schema.addField('mjd', type=np.float64, doc="MJD of visit")
834 schema.addField('exptime', type=np.float32, doc="Exposure time")
835 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
836 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
837 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
838 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
839 # the following field is not used yet
840 schema.addField('deepFlag', type=np.int32, doc="Deep observation")
841 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
842 size=nCcd)
843 schema.addField('used', type=np.int32, doc="This visit has been ingested.")
844 schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
846 return schema
848 def _makeFgcmObjSchema(self):
849 """
850 Make a schema for the objIndexCat from fgcmMakeStars
852 Returns
853 -------
854 schema: `afwTable.Schema`
855 """
857 objSchema = afwTable.Schema()
858 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
859 # Will investigate making these angles...
860 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
861 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
862 objSchema.addField('obsArrIndex', type=np.int32,
863 doc='Index in obsIndexTable for first observation')
864 objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
866 return objSchema
868 def _makeFgcmObsSchema(self):
869 """
870 Make a schema for the obsIndexCat from fgcmMakeStars
872 Returns
873 -------
874 schema: `afwTable.Schema`
875 """
877 obsSchema = afwTable.Schema()
878 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
880 return obsSchema
882 def _makeFgcmRefSchema(self, nReferenceBands):
883 """
884 Make a schema for the referenceCat from fgcmMakeStars
886 Parameters
887 ----------
888 nReferenceBands: `int`
889 Number of reference bands
891 Returns
892 -------
893 schema: `afwTable.Schema`
894 """
896 refSchema = afwTable.Schema()
897 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
898 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
899 size=nReferenceBands)
900 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
901 size=nReferenceBands)
903 return refSchema
905 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
906 """
907 Get the reference filter names, in wavelength order, from the visitCat and
908 information from the look-up-table.
910 Parameters
911 ----------
912 visitCat: `afw.table.BaseCatalog`
913 Catalog with visit data for FGCM
914 stdFilterDict: `dict`
915 Mapping of filterName to stdFilterName from LUT
916 stdLambdaDict: `dict`
917 Mapping of stdFilterName to stdLambda from LUT
919 Returns
920 -------
921 referenceFilterNames: `list`
922 Wavelength-ordered list of reference filter names
923 """
925 # Find the unique list of filter names in visitCat
926 filterNames = np.unique(visitCat.asAstropy()['filtername'])
928 # Find the unique list of "standard" filters
929 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
931 # And sort these by wavelength
932 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
934 return referenceFilterNames