lsst.fgcmcal g9c3108eb99+ae3892c209
fgcmBuildStarsBase.py
Go to the documentation of this file.
1# This file is part of fgcmcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Base class for BuildStars using src tables or sourceTable_visit tables.
22"""
23
24import os
25import sys
26import traceback
27import abc
28
29import numpy as np
30
31import lsst.daf.persistence as dafPersist
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34import lsst.afw.table as afwTable
35import lsst.geom as geom
36from lsst.daf.base import PropertyList
37from lsst.daf.base.dateTime import DateTime
38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
39
40from .utilities import computeApertureRadiusFromDataRef
41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
42
43import fgcm
44
45REFSTARS_FORMAT_VERSION = 1
46
47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
48
49
50class FgcmBuildStarsConfigBase(pexConfig.Config):
51 """Base config for FgcmBuildStars tasks"""
52
53 instFluxField = pexConfig.Field(
54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55 "The associated flag will be implicitly included in badFlags"),
56 dtype=str,
57 default='slot_CalibFlux_instFlux',
58 )
59 minPerBand = pexConfig.Field(
60 doc="Minimum observations per band",
61 dtype=int,
62 default=2,
63 )
64 matchRadius = pexConfig.Field(
65 doc="Match radius (arcseconds)",
66 dtype=float,
67 default=1.0,
68 )
69 isolationRadius = pexConfig.Field(
70 doc="Isolation radius (arcseconds)",
71 dtype=float,
72 default=2.0,
73 )
74 densityCutNside = pexConfig.Field(
75 doc="Density cut healpix nside",
76 dtype=int,
77 default=128,
78 )
79 densityCutMaxPerPixel = pexConfig.Field(
80 doc="Density cut number of stars per pixel",
81 dtype=int,
82 default=1000,
83 )
84 randomSeed = pexConfig.Field(
85 doc="Random seed for high density down-sampling.",
86 dtype=int,
87 default=None,
88 optional=True,
89 )
90 matchNside = pexConfig.Field(
91 doc="Healpix Nside for matching",
92 dtype=int,
93 default=4096,
94 )
95 coarseNside = pexConfig.Field(
96 doc="Healpix coarse Nside for partitioning matches",
97 dtype=int,
98 default=8,
99 )
100 # The following config will not be necessary after Gen2 retirement.
101 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
102 # which is easiest to access in the config file.
103 physicalFilterMap = pexConfig.DictField(
104 doc="Mapping from 'physicalFilter' to band.",
105 keytype=str,
106 itemtype=str,
107 default={},
108 )
109 requiredBands = pexConfig.ListField(
110 doc="Bands required for each star",
111 dtype=str,
112 default=(),
113 )
114 primaryBands = pexConfig.ListField(
115 doc=("Bands for 'primary' star matches. "
116 "A star must be observed in one of these bands to be considered "
117 "as a calibration star."),
118 dtype=str,
119 default=None
120 )
121 visitDataRefName = pexConfig.Field(
122 doc="dataRef name for the 'visit' field, usually 'visit'.",
123 dtype=str,
124 default="visit"
125 )
126 ccdDataRefName = pexConfig.Field(
127 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
128 dtype=str,
129 default="ccd"
130 )
131 doApplyWcsJacobian = pexConfig.Field(
132 doc="Apply the jacobian of the WCS to the star observations prior to fit?",
133 dtype=bool,
134 default=True
135 )
136 doModelErrorsWithBackground = pexConfig.Field(
137 doc="Model flux errors with background term?",
138 dtype=bool,
139 default=True
140 )
141 psfCandidateName = pexConfig.Field(
142 doc="Name of field with psf candidate flag for propagation",
143 dtype=str,
144 default="calib_psf_candidate"
145 )
146 doSubtractLocalBackground = pexConfig.Field(
147 doc=("Subtract the local background before performing calibration? "
148 "This is only supported for circular aperture calibration fluxes."),
149 dtype=bool,
150 default=False
151 )
152 localBackgroundFluxField = pexConfig.Field(
153 doc="Full name of the local background instFlux field to use.",
154 dtype=str,
155 default='base_LocalBackground_instFlux'
156 )
157 sourceSelector = sourceSelectorRegistry.makeField(
158 doc="How to select sources",
159 default="science"
160 )
161 apertureInnerInstFluxField = pexConfig.Field(
162 doc=("Full name of instFlux field that contains inner aperture "
163 "flux for aperture correction proxy"),
164 dtype=str,
165 default='base_CircularApertureFlux_12_0_instFlux'
166 )
167 apertureOuterInstFluxField = pexConfig.Field(
168 doc=("Full name of instFlux field that contains outer aperture "
169 "flux for aperture correction proxy"),
170 dtype=str,
171 default='base_CircularApertureFlux_17_0_instFlux'
172 )
173 doReferenceMatches = pexConfig.Field(
174 doc="Match reference catalog as additional constraint on calibration",
175 dtype=bool,
176 default=True,
177 )
178 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
179 target=FgcmLoadReferenceCatalogTask,
180 doc="FGCM reference object loader",
181 )
182 nVisitsPerCheckpoint = pexConfig.Field(
183 doc="Number of visits read between checkpoints",
184 dtype=int,
185 default=500,
186 )
187
188 def setDefaults(self):
189 sourceSelector = self.sourceSelectorsourceSelector["science"]
190 sourceSelector.setDefaults()
191
192 sourceSelector.doFlags = True
193 sourceSelector.doUnresolved = True
194 sourceSelector.doSignalToNoise = True
195 sourceSelector.doIsolated = True
196
197 sourceSelector.signalToNoise.minimum = 10.0
198 sourceSelector.signalToNoise.maximum = 1000.0
199
200 # FGCM operates on unresolved sources, and this setting is
201 # appropriate for the current base_ClassificationExtendedness
202 sourceSelector.unresolved.maximum = 0.5
203
204
205class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
206 """Subclass of TaskRunner for FgcmBuildStars tasks
207
208 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
209 arguments, one of which is the butler (for persistence and mapper data),
210 and a list of dataRefs extracted from the command line. Note that FGCM
211 runs on a large set of dataRefs, and not on single dataRef/tract/patch.
212 This class transforms the process arguments generated by the ArgumentParser
213 into the arguments expected by FgcmBuildStarsTask.run(). This runner does
214 not use any parallelization.
215 """
216 @staticmethod
217 def getTargetList(parsedCmd):
218 """
219 Return a list with one element: a tuple with the butler and
220 list of dataRefs
221 """
222 # we want to combine the butler with any (or no!) dataRefs
223 return [(parsedCmd.butler, parsedCmd.id.refList)]
224
225 def __call__(self, args):
226 """
227 Parameters
228 ----------
229 args: `tuple` with (butler, dataRefList)
230
231 Returns
232 -------
233 exitStatus: `list` with `lsst.pipe.base.Struct`
234 exitStatus (0: success; 1: failure)
235 """
236 butler, dataRefList = args
237
238 task = self.TaskClass(config=self.config, log=self.log)
239
240 exitStatus = 0
241 if self.doRaise:
242 task.runDataRef(butler, dataRefList)
243 else:
244 try:
245 task.runDataRef(butler, dataRefList)
246 except Exception as e:
247 exitStatus = 1
248 task.log.fatal("Failed: %s" % e)
249 if not isinstance(e, pipeBase.TaskError):
250 traceback.print_exc(file=sys.stderr)
251
252 task.writeMetadata(butler)
253
254 # The task does not return any results:
255 return [pipeBase.Struct(exitStatus=exitStatus)]
256
257 def run(self, parsedCmd):
258 """
259 Run the task, with no multiprocessing
260
261 Parameters
262 ----------
263 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
264 """
265
266 resultList = []
267
268 if self.precall(parsedCmd):
269 targetList = self.getTargetListgetTargetList(parsedCmd)
270 resultList = self(targetList[0])
271
272 return resultList
273
274
275class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
276 """
277 Base task to build stars for FGCM global calibration
278
279 Parameters
280 ----------
281 butler : `lsst.daf.persistence.Butler`
282 """
283 def __init__(self, initInputs=None, butler=None, **kwargs):
284 super().__init__(**kwargs)
285
286 self.makeSubtask("sourceSelector")
287 # Only log warning and fatal errors from the sourceSelector
288 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
289
290 # no saving of metadata for now
291 def _getMetadataName(self):
292 return None
293
294 @pipeBase.timeMethod
295 def runDataRef(self, butler, dataRefs):
296 """
297 Cross-match and make star list for FGCM Input
298
299 Parameters
300 ----------
301 butler: `lsst.daf.persistence.Butler`
302 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
303 Source data references for the input visits.
304
305 Raises
306 ------
307 RuntimeErrror: Raised if `config.doReferenceMatches` is set and
308 an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
309 fails if the calibFlux is not a CircularAperture flux.
310 """
311 datasetType = dataRefs[0].butlerSubset.datasetType
312 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
313
314 if self.config.doReferenceMatches:
315 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
316 # Ensure that we have a LUT
317 if not butler.datasetExists('fgcmLookUpTable'):
318 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
319 # Compute aperture radius if necessary. This is useful to do now before
320 # any heavy lifting has happened (fail early).
321 calibFluxApertureRadius = None
322 if self.config.doSubtractLocalBackground:
323 try:
324 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
325 self.config.instFluxField)
326 except RuntimeError as e:
327 raise RuntimeError("Could not determine aperture radius from %s. "
328 "Cannot use doSubtractLocalBackground." %
329 (self.config.instFluxField)) from e
330
331 camera = butler.get('camera')
332 groupedDataRefs = self._findAndGroupDataRefsGen2_findAndGroupDataRefsGen2(butler, camera, dataRefs)
333
334 # Make the visit catalog if necessary
335 # First check if the visit catalog is in the _current_ path
336 # We cannot use Gen2 datasetExists() because that checks all parent
337 # directories as well, which would make recovering from faults
338 # and fgcmcal reruns impossible.
339 visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
340 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
341 if os.path.exists(filename):
342 # This file exists and we should continue processing
343 inVisitCat = visitCatDataRef.get()
344 if len(inVisitCat) != len(groupedDataRefs):
345 raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
346 "number of visits. Cannot continue.")
347 else:
348 inVisitCat = None
349
350 visitCat = self.fgcmMakeVisitCatalogfgcmMakeVisitCatalog(camera, groupedDataRefs,
351 visitCatDataRef=visitCatDataRef,
352 inVisitCat=inVisitCat)
353
354 # Persist the visitCat as a checkpoint file.
355 visitCatDataRef.put(visitCat)
356
357 starObsDataRef = butler.dataRef('fgcmStarObservations')
358 filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
359 if os.path.exists(filename):
360 inStarObsCat = starObsDataRef.get()
361 else:
362 inStarObsCat = None
363
364 rad = calibFluxApertureRadius
365 sourceSchemaDataRef = butler.dataRef('src_schema')
366 sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema
367 fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedDataRefs,
368 visitCat,
369 sourceSchema,
370 camera,
371 calibFluxApertureRadius=rad,
372 starObsDataRef=starObsDataRef,
373 visitCatDataRef=visitCatDataRef,
374 inStarObsCat=inStarObsCat)
375 visitCatDataRef.put(visitCat)
376 starObsDataRef.put(fgcmStarObservationCat)
377
378 # Always do the matching.
379 if self.config.doReferenceMatches:
380 lutDataRef = butler.dataRef('fgcmLookUpTable')
381 else:
382 lutDataRef = None
383 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStarsfgcmMatchStars(visitCat,
384 fgcmStarObservationCat,
385 lutDataRef=lutDataRef)
386
387 # Persist catalogs via the butler
388 butler.put(fgcmStarIdCat, 'fgcmStarIds')
389 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
390 if fgcmRefCat is not None:
391 butler.put(fgcmRefCat, 'fgcmReferenceStars')
392
393 @abc.abstractmethod
394 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
395 """
396 Find and group dataRefs (by visit); Gen2 only.
397
398 Parameters
399 ----------
400 butler : `lsst.daf.persistence.Butler`
401 Gen2 butler.
402 camera : `lsst.afw.cameraGeom.Camera`
403 Camera from the butler.
404 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
405 Data references for the input visits.
406
407 Returns
408 -------
409 groupedDataRefs : `dict` [`int`, `list`]
410 Dictionary with sorted visit keys, and `list`s of
411 `lsst.daf.persistence.ButlerDataRef`
412 """
413 raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.")
414
415 @abc.abstractmethod
416 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
417 sourceSchema,
418 camera,
419 calibFluxApertureRadius=None,
420 visitCatDataRef=None,
421 starObsDataRef=None,
422 inStarObsCat=None):
423 """
424 Compile all good star observations from visits in visitCat. Checkpoint files
425 will be stored if both visitCatDataRef and starObsDataRef are not None.
426
427 Parameters
428 ----------
429 groupedDataRefs : `dict` of `list`s
430 Lists of `~lsst.daf.persistence.ButlerDataRef` or
431 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
432 visitCat : `~afw.table.BaseCatalog`
433 Catalog with visit data for FGCM
434 sourceSchema : `~lsst.afw.table.Schema`
435 Schema for the input src catalogs.
436 camera : `~lsst.afw.cameraGeom.Camera`
437 calibFluxApertureRadius : `float`, optional
438 Aperture radius for calibration flux.
439 visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
440 Dataref to write visitCat for checkpoints
441 starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
442 Dataref to write the star observation catalog for checkpoints.
443 inStarObsCat : `~afw.table.BaseCatalog`
444 Input observation catalog. If this is incomplete, observations
445 will be appended from when it was cut off.
446
447 Returns
448 -------
449 fgcmStarObservations : `afw.table.BaseCatalog`
450 Full catalog of good observations.
451
452 Raises
453 ------
454 RuntimeError: Raised if doSubtractLocalBackground is True and
455 calibFluxApertureRadius is not set.
456 """
457 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
458
459 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
460 visitCatDataRef=None, inVisitCat=None):
461 """
462 Make a visit catalog with all the keys from each visit
463
464 Parameters
465 ----------
466 camera: `lsst.afw.cameraGeom.Camera`
467 Camera from the butler
468 groupedDataRefs: `dict`
469 Dictionary with visit keys, and `list`s of
470 `lsst.daf.persistence.ButlerDataRef`
471 bkgDataRefDict: `dict`, optional
472 Dictionary of gen3 dataRefHandles for background info.
473 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
474 Dataref to write visitCat for checkpoints
475 inVisitCat: `afw.table.BaseCatalog`, optional
476 Input (possibly incomplete) visit catalog
477
478 Returns
479 -------
480 visitCat: `afw.table.BaseCatalog`
481 """
482
483 self.log.info("Assembling visitCatalog from %d %ss" %
484 (len(groupedDataRefs), self.config.visitDataRefName))
485
486 nCcd = len(camera)
487
488 if inVisitCat is None:
489 schema = self._makeFgcmVisitSchema_makeFgcmVisitSchema(nCcd)
490
491 visitCat = afwTable.BaseCatalog(schema)
492 visitCat.reserve(len(groupedDataRefs))
493 visitCat.resize(len(groupedDataRefs))
494
495 visitCat['visit'] = list(groupedDataRefs.keys())
496 visitCat['used'] = 0
497 visitCat['sources_read'] = False
498 else:
499 visitCat = inVisitCat
500
501 # No matter what, fill the catalog. This will check if it was
502 # already read.
503 self._fillVisitCatalog_fillVisitCatalog(visitCat, groupedDataRefs,
504 bkgDataRefDict=bkgDataRefDict,
505 visitCatDataRef=visitCatDataRef)
506
507 return visitCat
508
509 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
510 visitCatDataRef=None):
511 """
512 Fill the visit catalog with visit metadata
513
514 Parameters
515 ----------
516 visitCat : `afw.table.BaseCatalog`
517 Visit catalog. See _makeFgcmVisitSchema() for schema definition.
518 groupedDataRefs : `dict`
519 Dictionary with visit keys, and `list`s of
520 `lsst.daf.persistence.ButlerDataRef` or
521 `lsst.daf.butler.DeferredDatasetHandle`
522 visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional
523 Dataref to write ``visitCat`` for checkpoints. Gen2 only.
524 bkgDataRefDict : `dict`, optional
525 Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle`
526 for background info.
527 """
528 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
529
530 for i, visit in enumerate(groupedDataRefs):
531 # We don't use the bypasses since we need the psf info which does
532 # not have a bypass
533 # TODO: When DM-15500 is implemented in the Gen3 Butler, this
534 # can be fixed
535
536 # Do not read those that have already been read
537 if visitCat['used'][i]:
538 continue
539
540 if (i % self.config.nVisitsPerCheckpoint) == 0:
541 self.log.info("Retrieving metadata for %s %d (%d/%d)" %
542 (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
543 # Save checkpoint if desired
544 if visitCatDataRef is not None:
545 visitCatDataRef.put(visitCat)
546
547 dataRef = groupedDataRefs[visit][0]
548 if isinstance(dataRef, dafPersist.ButlerDataRef):
549 # Gen2: calexp dataRef
550 # The first dataRef in the group will be the reference ccd (if available)
551 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
552 visitInfo = exp.getInfo().getVisitInfo()
553 label = dataRef.get(datasetType='calexp_filterLabel')
554 physicalFilter = label.physicalLabel
555 psf = exp.getPsf()
556 bbox = dataRef.get(datasetType='calexp_bbox')
557 psfSigma = psf.computeShape(bbox.getCenter()).getDeterminantRadius()
558 else:
559 # Gen3: use the visitSummary dataRef
560 summary = dataRef.get()
561
562 summaryRow = summary.find(self.config.referenceCCD)
563 if summaryRow is None:
564 # Take the first available ccd if reference isn't available
565 summaryRow = summary[0]
566
567 summaryDetector = summaryRow['id']
568 visitInfo = summaryRow.getVisitInfo()
569 physicalFilter = summaryRow['physical_filter']
570 # Compute the median psf sigma if possible
571 goodSigma, = np.where(summary['psfSigma'] > 0)
572 if goodSigma.size > 2:
573 psfSigma = np.median(summary['psfSigma'][goodSigma])
574 elif goodSigma > 0:
575 psfSigma = np.mean(summary['psfSigma'][goodSigma])
576 else:
577 psfSigma = 0.0
578
579 rec = visitCat[i]
580 rec['visit'] = visit
581 rec['physicalFilter'] = physicalFilter
582 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
583 # much easier to get the wcs's necessary to recompute the pointing
584 # ra/dec at the center of the camera.
585 radec = visitInfo.getBoresightRaDec()
586 rec['telra'] = radec.getRa().asDegrees()
587 rec['teldec'] = radec.getDec().asDegrees()
588 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
589 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
590 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
591 rec['exptime'] = visitInfo.getExposureTime()
592 # convert from Pa to millibar
593 # Note that I don't know if this unit will need to be per-camera config
594 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
595 # Flag to signify if this is a "deep" field. Not currently used
596 rec['deepFlag'] = 0
597 # Relative flat scaling (1.0 means no relative scaling)
598 rec['scaling'][:] = 1.0
599 # Median delta aperture, to be measured from stars
600 rec['deltaAper'] = 0.0
601 rec['psfSigma'] = psfSigma
602
603 if self.config.doModelErrorsWithBackground:
604 foundBkg = False
605 if isinstance(dataRef, dafPersist.ButlerDataRef):
606 # Gen2-style dataRef
607 det = dataRef.dataId[self.config.ccdDataRefName]
608 if dataRef.datasetExists(datasetType='calexpBackground'):
609 bgList = dataRef.get(datasetType='calexpBackground')
610 foundBkg = True
611 else:
612 # Gen3-style dataRef
613 try:
614 # Use the same detector used from the summary.
615 bkgRef = bkgDataRefDict[(visit, summaryDetector)]
616 bgList = bkgRef.get()
617 foundBkg = True
618 except KeyError:
619 pass
620
621 if foundBkg:
622 bgStats = (bg[0].getStatsImage().getImage().array
623 for bg in bgList)
624 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
625 else:
626 self.log.warning('Sky background not found for visit %d / ccd %d' %
627 (visit, det))
628 rec['skyBackground'] = -1.0
629 else:
630 rec['skyBackground'] = -1.0
631
632 rec['used'] = 1
633
634 def _makeSourceMapper(self, sourceSchema):
635 """
636 Make a schema mapper for fgcm sources
637
638 Parameters
639 ----------
640 sourceSchema: `afwTable.Schema`
641 Default source schema from the butler
642
643 Returns
644 -------
645 sourceMapper: `afwTable.schemaMapper`
646 Mapper to the FGCM source schema
647 """
648
649 # create a mapper to the preferred output
650 sourceMapper = afwTable.SchemaMapper(sourceSchema)
651
652 # map to ra/dec
653 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
654 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
655 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
656 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
657 # Add the mapping if the field exists in the input catalog.
658 # If the field does not exist, simply add it (set to False).
659 # This field is not required for calibration, but is useful
660 # to collate if available.
661 try:
662 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
663 'psf_candidate')
664 except LookupError:
665 sourceMapper.editOutputSchema().addField(
666 "psf_candidate", type='Flag',
667 doc=("Flag set if the source was a candidate for PSF determination, "
668 "as determined by the star selector."))
669
670 # and add the fields we want
671 sourceMapper.editOutputSchema().addField(
672 "visit", type=np.int32, doc="Visit number")
673 sourceMapper.editOutputSchema().addField(
674 "ccd", type=np.int32, doc="CCD number")
675 sourceMapper.editOutputSchema().addField(
676 "instMag", type=np.float32, doc="Instrumental magnitude")
677 sourceMapper.editOutputSchema().addField(
678 "instMagErr", type=np.float32, doc="Instrumental magnitude error")
679 sourceMapper.editOutputSchema().addField(
680 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
681 sourceMapper.editOutputSchema().addField(
682 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
683
684 return sourceMapper
685
686 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
687 """
688 Use FGCM code to match observations into unique stars.
689
690 Parameters
691 ----------
692 visitCat: `afw.table.BaseCatalog`
693 Catalog with visit data for fgcm
694 obsCat: `afw.table.BaseCatalog`
695 Full catalog of star observations for fgcm
696 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
697 `lsst.daf.butler.DeferredDatasetHandle`, optional
698 Data reference to fgcm look-up table (used if matching reference stars).
699
700 Returns
701 -------
702 fgcmStarIdCat: `afw.table.BaseCatalog`
703 Catalog of unique star identifiers and index keys
704 fgcmStarIndicesCat: `afwTable.BaseCatalog`
705 Catalog of unique star indices
706 fgcmRefCat: `afw.table.BaseCatalog`
707 Catalog of matched reference stars.
708 Will be None if `config.doReferenceMatches` is False.
709 """
710 # get filter names into a numpy array...
711 # This is the type that is expected by the fgcm code
712 visitFilterNames = np.zeros(len(visitCat), dtype='a30')
713 for i in range(len(visitCat)):
714 visitFilterNames[i] = visitCat[i]['physicalFilter']
715
716 # match to put filterNames with observations
717 visitIndex = np.searchsorted(visitCat['visit'],
718 obsCat['visit'])
719
720 obsFilterNames = visitFilterNames[visitIndex]
721
722 if self.config.doReferenceMatches:
723 # Get the reference filter names, using the LUT
724 lutCat = lutDataRef.get()
725
726 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
727 zip(lutCat[0]['physicalFilters'].split(','),
728 lutCat[0]['stdPhysicalFilters'].split(','))}
729 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
730 zip(lutCat[0]['stdPhysicalFilters'].split(','),
731 lutCat[0]['lambdaStdFilter'])}
732
733 del lutCat
734
735 referenceFilterNames = self._getReferenceFilterNames_getReferenceFilterNames(visitCat,
736 stdFilterDict,
737 stdLambdaDict)
738 self.log.info("Using the following reference filters: %s" %
739 (', '.join(referenceFilterNames)))
740
741 else:
742 # This should be an empty list
743 referenceFilterNames = []
744
745 # make the fgcm starConfig dict
746 starConfig = {'logger': self.log,
747 'filterToBand': self.config.physicalFilterMap,
748 'requiredBands': self.config.requiredBands,
749 'minPerBand': self.config.minPerBand,
750 'matchRadius': self.config.matchRadius,
751 'isolationRadius': self.config.isolationRadius,
752 'matchNSide': self.config.matchNside,
753 'coarseNSide': self.config.coarseNside,
754 'densNSide': self.config.densityCutNside,
755 'densMaxPerPixel': self.config.densityCutMaxPerPixel,
756 'randomSeed': self.config.randomSeed,
757 'primaryBands': self.config.primaryBands,
758 'referenceFilterNames': referenceFilterNames}
759
760 # initialize the FgcmMakeStars object
761 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
762
763 # make the primary stars
764 # note that the ra/dec native Angle format is radians
765 # We determine the conversion from the native units (typically
766 # radians) to degrees for the first observation. This allows us
767 # to treate ra/dec as numpy arrays rather than Angles, which would
768 # be approximately 600x slower.
769 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
770 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
771 obsCat['dec'] * conv,
772 filterNameArray=obsFilterNames,
773 bandSelected=False)
774
775 # and match all the stars
776 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
777 obsCat['dec'] * conv,
778 obsFilterNames)
779
780 if self.config.doReferenceMatches:
781 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
782
783 # now persist
784
785 objSchema = self._makeFgcmObjSchema_makeFgcmObjSchema()
786
787 # make catalog and records
788 fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
789 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
790 for i in range(fgcmMakeStars.objIndexCat.size):
791 fgcmStarIdCat.addNew()
792
793 # fill the catalog
794 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
795 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
796 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
797 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
798 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
799
800 obsSchema = self._makeFgcmObsSchema_makeFgcmObsSchema()
801
802 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
803 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
804 for i in range(fgcmMakeStars.obsIndexCat.size):
805 fgcmStarIndicesCat.addNew()
806
807 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
808
809 if self.config.doReferenceMatches:
810 refSchema = self._makeFgcmRefSchema_makeFgcmRefSchema(len(referenceFilterNames))
811
812 fgcmRefCat = afwTable.BaseCatalog(refSchema)
813 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
814
815 for i in range(fgcmMakeStars.referenceCat.size):
816 fgcmRefCat.addNew()
817
818 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
819 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
820 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
821
822 md = PropertyList()
823 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
824 md.set("FILTERNAMES", referenceFilterNames)
825 fgcmRefCat.setMetadata(md)
826
827 else:
828 fgcmRefCat = None
829
830 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
831
832 def _makeFgcmVisitSchema(self, nCcd):
833 """
834 Make a schema for an fgcmVisitCatalog
835
836 Parameters
837 ----------
838 nCcd: `int`
839 Number of CCDs in the camera
840
841 Returns
842 -------
843 schema: `afwTable.Schema`
844 """
845
846 schema = afwTable.Schema()
847 schema.addField('visit', type=np.int32, doc="Visit number")
848 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
849 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
850 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
851 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
852 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
853 schema.addField('mjd', type=np.float64, doc="MJD of visit")
854 schema.addField('exptime', type=np.float32, doc="Exposure time")
855 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
856 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
857 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
858 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
859 # the following field is not used yet
860 schema.addField('deepFlag', type=np.int32, doc="Deep observation")
861 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
862 size=nCcd)
863 schema.addField('used', type=np.int32, doc="This visit has been ingested.")
864 schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
865
866 return schema
867
868 def _makeFgcmObjSchema(self):
869 """
870 Make a schema for the objIndexCat from fgcmMakeStars
871
872 Returns
873 -------
874 schema: `afwTable.Schema`
875 """
876
877 objSchema = afwTable.Schema()
878 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
879 # Will investigate making these angles...
880 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
881 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
882 objSchema.addField('obsArrIndex', type=np.int32,
883 doc='Index in obsIndexTable for first observation')
884 objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
885
886 return objSchema
887
888 def _makeFgcmObsSchema(self):
889 """
890 Make a schema for the obsIndexCat from fgcmMakeStars
891
892 Returns
893 -------
894 schema: `afwTable.Schema`
895 """
896
897 obsSchema = afwTable.Schema()
898 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
899
900 return obsSchema
901
902 def _makeFgcmRefSchema(self, nReferenceBands):
903 """
904 Make a schema for the referenceCat from fgcmMakeStars
905
906 Parameters
907 ----------
908 nReferenceBands: `int`
909 Number of reference bands
910
911 Returns
912 -------
913 schema: `afwTable.Schema`
914 """
915
916 refSchema = afwTable.Schema()
917 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
918 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
919 size=nReferenceBands)
920 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
921 size=nReferenceBands)
922
923 return refSchema
924
925 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
926 """
927 Get the reference filter names, in wavelength order, from the visitCat and
928 information from the look-up-table.
929
930 Parameters
931 ----------
932 visitCat: `afw.table.BaseCatalog`
933 Catalog with visit data for FGCM
934 stdFilterDict: `dict`
935 Mapping of filterName to stdFilterName from LUT
936 stdLambdaDict: `dict`
937 Mapping of stdFilterName to stdLambda from LUT
938
939 Returns
940 -------
941 referenceFilterNames: `list`
942 Wavelength-ordered list of reference filter names
943 """
944
945 # Find the unique list of filter names in visitCat
946 filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
947
948 # Find the unique list of "standard" filters
949 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
950
951 # And sort these by wavelength
952 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
953
954 return referenceFilterNames
def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None)
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, sourceSchema, camera, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def __init__(self, initInputs=None, butler=None, **kwargs)
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None, inVisitCat=None)
def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:808