21"""Base class for BuildStars using src tables or sourceTable_visit tables.
28import lsst.pex.config
as pexConfig
29import lsst.pipe.base
as pipeBase
30import lsst.afw.table
as afwTable
31from lsst.daf.base
import PropertyList
32from lsst.daf.base.dateTime
import DateTime
33from lsst.meas.algorithms.sourceSelector
import sourceSelectorRegistry
35from .fgcmLoadReferenceCatalog
import FgcmLoadReferenceCatalogTask
39REFSTARS_FORMAT_VERSION = 1
41__all__ = [
'FgcmBuildStarsConfigBase',
'FgcmBuildStarsBaseTask']
45 """Base config for FgcmBuildStars tasks"""
47 instFluxField = pexConfig.Field(
48 doc=(
"Faull name of the source instFlux field to use, including 'instFlux'. "
49 "The associated flag will be implicitly included in badFlags"),
51 default=
'slot_CalibFlux_instFlux',
53 minPerBand = pexConfig.Field(
54 doc=
"Minimum observations per band",
58 matchRadius = pexConfig.Field(
59 doc=
"Match radius (arcseconds)",
63 isolationRadius = pexConfig.Field(
64 doc=
"Isolation radius (arcseconds)",
68 densityCutNside = pexConfig.Field(
69 doc=
"Density cut healpix nside",
73 densityCutMaxPerPixel = pexConfig.Field(
74 doc=
"Density cut number of stars per pixel",
78 randomSeed = pexConfig.Field(
79 doc=
"Random seed for high density down-sampling.",
84 matchNside = pexConfig.Field(
85 doc=
"Healpix Nside for matching",
89 coarseNside = pexConfig.Field(
90 doc=
"Healpix coarse Nside for partitioning matches",
94 physicalFilterMap = pexConfig.DictField(
95 doc=
"Mapping from 'physicalFilter' to band.",
100 requiredBands = pexConfig.ListField(
101 doc=
"Bands required for each star",
105 primaryBands = pexConfig.ListField(
106 doc=(
"Bands for 'primary' star matches. "
107 "A star must be observed in one of these bands to be considered "
108 "as a calibration star."),
112 visitDataRefName = pexConfig.Field(
113 doc=
"dataRef name for the 'visit' field, usually 'visit'.",
116 deprecated=
"The visitDataRefname was only used for gen2; this config will be removed after v24."
118 ccdDataRefName = pexConfig.Field(
119 doc=
"dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
122 deprecated=
"The ccdDataRefname was only used for gen2; this config will be removed after v24."
124 doApplyWcsJacobian = pexConfig.Field(
125 doc=
"Apply the jacobian of the WCS to the star observations prior to fit?",
129 doModelErrorsWithBackground = pexConfig.Field(
130 doc=
"Model flux errors with background term?",
134 psfCandidateName = pexConfig.Field(
135 doc=
"Name of field with psf candidate flag for propagation",
137 default=
"calib_psf_candidate"
139 doSubtractLocalBackground = pexConfig.Field(
140 doc=(
"Subtract the local background before performing calibration? "
141 "This is only supported for circular aperture calibration fluxes."),
145 localBackgroundFluxField = pexConfig.Field(
146 doc=
"Full name of the local background instFlux field to use.",
148 default=
'base_LocalBackground_instFlux'
150 sourceSelector = sourceSelectorRegistry.makeField(
151 doc=
"How to select sources",
154 apertureInnerInstFluxField = pexConfig.Field(
155 doc=(
"Full name of instFlux field that contains inner aperture "
156 "flux for aperture correction proxy"),
158 default=
'base_CircularApertureFlux_12_0_instFlux'
160 apertureOuterInstFluxField = pexConfig.Field(
161 doc=(
"Full name of instFlux field that contains outer aperture "
162 "flux for aperture correction proxy"),
164 default=
'base_CircularApertureFlux_17_0_instFlux'
166 doReferenceMatches = pexConfig.Field(
167 doc=
"Match reference catalog as additional constraint on calibration",
171 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
172 target=FgcmLoadReferenceCatalogTask,
173 doc=
"FGCM reference object loader",
175 nVisitsPerCheckpoint = pexConfig.Field(
176 doc=
"Number of visits read between checkpoints",
183 sourceSelector.setDefaults()
185 sourceSelector.doFlags =
True
186 sourceSelector.doUnresolved =
True
187 sourceSelector.doSignalToNoise =
True
188 sourceSelector.doIsolated =
True
189 sourceSelector.doRequireFiniteRaDec =
True
191 sourceSelector.signalToNoise.minimum = 10.0
192 sourceSelector.signalToNoise.maximum = 1000.0
196 sourceSelector.unresolved.maximum = 0.5
201 Base task to build stars for FGCM
global calibration
206 self.makeSubtask(
"sourceSelector")
208 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
214 calibFluxApertureRadius=None):
216 Compile all good star observations from visits
in visitCat.
220 groupedHandles : `dict` [`list` [`lsst.daf.butler.DeferredDatasetHandle`]]
221 Dataset handles, grouped by visit.
222 visitCat : `afw.table.BaseCatalog`
223 Catalog
with visit data
for FGCM
224 sourceSchema : `lsst.afw.table.Schema`
225 Schema
for the input src catalogs.
226 camera : `lsst.afw.cameraGeom.Camera`
227 calibFluxApertureRadius : `float`, optional
228 Aperture radius
for calibration flux.
229 inStarObsCat : `afw.table.BaseCatalog`
230 Input observation catalog. If this
is incomplete, observations
231 will be appended
from when it was cut off.
235 fgcmStarObservations : `afw.table.BaseCatalog`
236 Full catalog of good observations.
240 RuntimeError: Raised
if doSubtractLocalBackground
is True and
241 calibFluxApertureRadius
is not set.
243 raise NotImplementedError(
"fgcmMakeAllStarObservations not implemented.")
247 Make a visit catalog with all the keys
from each visit
251 camera: `lsst.afw.cameraGeom.Camera`
252 Camera
from the butler
253 groupedHandles: `dict` [`list` [`lsst.daf.butler.DeferredDatasetHandle`]]
254 Dataset handles, grouped by visit.
255 bkgHandleDict: `dict`, optional
256 Dictionary of `lsst.daf.butler.DeferredDatasetHandle`
for background info.
260 visitCat: `afw.table.BaseCatalog`
263 self.log.info("Assembling visitCatalog from %d visits", len(groupedHandles))
269 visitCat = afwTable.BaseCatalog(schema)
270 visitCat.reserve(len(groupedHandles))
271 visitCat.resize(len(groupedHandles))
273 visitCat[
'visit'] = list(groupedHandles.keys())
275 visitCat[
'sources_read'] =
False
280 bkgHandleDict=bkgHandleDict)
284 def _fillVisitCatalog(self, visitCat, groupedHandles, bkgHandleDict=None):
286 Fill the visit catalog with visit metadata
290 visitCat : `afw.table.BaseCatalog`
291 Visit catalog. See _makeFgcmVisitSchema()
for schema definition.
292 groupedHandles : `dict` [`list` [`lsst.daf.butler.DeferredDatasetHandle`]]
293 Dataset handles, grouped by visit.
294 bkgHandleDict : `dict`, optional
295 Dictionary of `lsst.daf.butler.DeferredDatasetHandle`
298 for i, visit
in enumerate(groupedHandles):
299 if (i % self.config.nVisitsPerCheckpoint) == 0:
300 self.log.info(
"Retrieving metadata for visit %d (%d/%d)", visit, i, len(groupedHandles))
302 handle = groupedHandles[visit][0]
303 summary = handle.get()
305 summaryRow = summary.find(self.config.referenceCCD)
306 if summaryRow
is None:
308 summaryRow = summary[0]
310 summaryDetector = summaryRow[
'id']
311 visitInfo = summaryRow.getVisitInfo()
312 physicalFilter = summaryRow[
'physical_filter']
314 goodSigma, = np.where(summary[
'psfSigma'] > 0)
315 if goodSigma.size > 2:
316 psfSigma = np.median(summary[
'psfSigma'][goodSigma])
317 elif goodSigma.size > 0:
318 psfSigma = np.mean(summary[
'psfSigma'][goodSigma])
320 self.log.warning(
"Could not find any good summary psfSigma for visit %d", visit)
325 rec[
'physicalFilter'] = physicalFilter
327 radec = visitInfo.getBoresightRaDec()
328 rec[
'telra'] = radec.getRa().asDegrees()
329 rec[
'teldec'] = radec.getDec().asDegrees()
330 rec[
'telha'] = visitInfo.getBoresightHourAngle().asDegrees()
331 rec[
'telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
332 rec[
'mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
333 rec[
'exptime'] = visitInfo.getExposureTime()
336 rec[
'pmb'] = visitInfo.getWeather().getAirPressure() / 100
340 rec[
'scaling'][:] = 1.0
342 rec[
'deltaAper'] = 0.0
343 rec[
'psfSigma'] = psfSigma
345 if self.config.doModelErrorsWithBackground:
347 bkgHandle = bkgHandleDict[(visit, summaryDetector)]
348 bgList = bkgHandle.get()
350 bgStats = (bg[0].getStatsImage().getImage().array
352 rec[
'skyBackground'] = sum(np.median(bg[np.isfinite(bg)])
for bg
in bgStats)
354 rec[
'skyBackground'] = -1.0
358 def _makeSourceMapper(self, sourceSchema):
360 Make a schema mapper for fgcm sources
364 sourceSchema: `afwTable.Schema`
365 Default source schema
from the butler
369 sourceMapper: `afwTable.schemaMapper`
370 Mapper to the FGCM source schema
374 sourceMapper = afwTable.SchemaMapper(sourceSchema)
377 sourceMapper.addMapping(sourceSchema[
'coord_ra'].asKey(),
'ra')
378 sourceMapper.addMapping(sourceSchema[
'coord_dec'].asKey(),
'dec')
379 sourceMapper.addMapping(sourceSchema[
'slot_Centroid_x'].asKey(),
'x')
380 sourceMapper.addMapping(sourceSchema[
'slot_Centroid_y'].asKey(),
'y')
386 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
389 sourceMapper.editOutputSchema().addField(
390 "psf_candidate", type=
'Flag',
391 doc=(
"Flag set if the source was a candidate for PSF determination, "
392 "as determined by the star selector."))
395 sourceMapper.editOutputSchema().addField(
396 "visit", type=np.int64, doc=
"Visit number")
397 sourceMapper.editOutputSchema().addField(
398 "ccd", type=np.int32, doc=
"CCD number")
399 sourceMapper.editOutputSchema().addField(
400 "instMag", type=np.float32, doc=
"Instrumental magnitude")
401 sourceMapper.editOutputSchema().addField(
402 "instMagErr", type=np.float32, doc=
"Instrumental magnitude error")
403 sourceMapper.editOutputSchema().addField(
404 "jacobian", type=np.float32, doc=
"Relative pixel scale from wcs jacobian")
405 sourceMapper.editOutputSchema().addField(
406 "deltaMagBkg", type=np.float32, doc=
"Change in magnitude due to local background offset")
407 sourceMapper.editOutputSchema().addField(
408 "deltaMagAper", type=np.float32, doc=
"Change in magnitude from larger to smaller aperture")
414 Use FGCM code to match observations into unique stars.
418 visitCat: `afw.table.BaseCatalog`
419 Catalog with visit data
for fgcm
420 obsCat: `afw.table.BaseCatalog`
421 Full catalog of star observations
for fgcm
422 lutHandle: `lsst.daf.butler.DeferredDatasetHandle`, optional
423 Data reference to fgcm look-up table (used
if matching reference stars).
427 fgcmStarIdCat: `afw.table.BaseCatalog`
428 Catalog of unique star identifiers
and index keys
429 fgcmStarIndicesCat: `afwTable.BaseCatalog`
430 Catalog of unique star indices
431 fgcmRefCat: `afw.table.BaseCatalog`
432 Catalog of matched reference stars.
433 Will be
None if `config.doReferenceMatches`
is False.
437 visitFilterNames = np.zeros(len(visitCat), dtype=
'a30')
438 for i
in range(len(visitCat)):
439 visitFilterNames[i] = visitCat[i][
'physicalFilter']
442 visitIndex = np.searchsorted(visitCat[
'visit'],
445 obsFilterNames = visitFilterNames[visitIndex]
447 if self.config.doReferenceMatches:
449 lutCat = lutHandle.get()
451 stdFilterDict = {filterName: stdFilter
for (filterName, stdFilter)
in
452 zip(lutCat[0][
'physicalFilters'].split(
','),
453 lutCat[0][
'stdPhysicalFilters'].split(
','))}
454 stdLambdaDict = {stdFilter: stdLambda
for (stdFilter, stdLambda)
in
455 zip(lutCat[0][
'stdPhysicalFilters'].split(
','),
456 lutCat[0][
'lambdaStdFilter'])}
463 self.log.info(
"Using the following reference filters: %s" %
464 (
', '.join(referenceFilterNames)))
468 referenceFilterNames = []
471 starConfig = {
'logger': self.log,
473 'filterToBand': self.config.physicalFilterMap,
474 'requiredBands': self.config.requiredBands,
475 'minPerBand': self.config.minPerBand,
476 'matchRadius': self.config.matchRadius,
477 'isolationRadius': self.config.isolationRadius,
478 'matchNSide': self.config.matchNside,
479 'coarseNSide': self.config.coarseNside,
480 'densNSide': self.config.densityCutNside,
481 'densMaxPerPixel': self.config.densityCutMaxPerPixel,
482 'randomSeed': self.config.randomSeed,
483 'primaryBands': self.config.primaryBands,
484 'referenceFilterNames': referenceFilterNames}
487 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
495 conv = obsCat[0][
'ra'].asDegrees() / float(obsCat[0][
'ra'])
496 fgcmMakeStars.makePrimaryStars(obsCat[
'ra'] * conv,
497 obsCat[
'dec'] * conv,
498 filterNameArray=obsFilterNames,
502 fgcmMakeStars.makeMatchedStars(obsCat[
'ra'] * conv,
503 obsCat[
'dec'] * conv,
506 if self.config.doReferenceMatches:
507 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
514 fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
515 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
516 for i
in range(fgcmMakeStars.objIndexCat.size):
517 fgcmStarIdCat.addNew()
520 fgcmStarIdCat[
'fgcm_id'][:] = fgcmMakeStars.objIndexCat[
'fgcm_id']
521 fgcmStarIdCat[
'ra'][:] = fgcmMakeStars.objIndexCat[
'ra']
522 fgcmStarIdCat[
'dec'][:] = fgcmMakeStars.objIndexCat[
'dec']
523 fgcmStarIdCat[
'obsArrIndex'][:] = fgcmMakeStars.objIndexCat[
'obsarrindex']
524 fgcmStarIdCat[
'nObs'][:] = fgcmMakeStars.objIndexCat[
'nobs']
528 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
529 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
530 for i
in range(fgcmMakeStars.obsIndexCat.size):
531 fgcmStarIndicesCat.addNew()
533 fgcmStarIndicesCat[
'obsIndex'][:] = fgcmMakeStars.obsIndexCat[
'obsindex']
535 if self.config.doReferenceMatches:
538 fgcmRefCat = afwTable.BaseCatalog(refSchema)
539 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
541 for i
in range(fgcmMakeStars.referenceCat.size):
544 fgcmRefCat[
'fgcm_id'][:] = fgcmMakeStars.referenceCat[
'fgcm_id']
545 fgcmRefCat[
'refMag'][:, :] = fgcmMakeStars.referenceCat[
'refMag']
546 fgcmRefCat[
'refMagErr'][:, :] = fgcmMakeStars.referenceCat[
'refMagErr']
549 md.set(
"REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
550 md.set(
"FILTERNAMES", referenceFilterNames)
551 fgcmRefCat.setMetadata(md)
556 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
558 def _makeFgcmVisitSchema(self, nCcd):
560 Make a schema for an fgcmVisitCatalog
565 Number of CCDs
in the camera
569 schema: `afwTable.Schema`
572 schema = afwTable.Schema()
573 schema.addField('visit', type=np.int64, doc=
"Visit number")
574 schema.addField(
'physicalFilter', type=str, size=30, doc=
"Physical filter")
575 schema.addField(
'telra', type=np.float64, doc=
"Pointing RA (deg)")
576 schema.addField(
'teldec', type=np.float64, doc=
"Pointing Dec (deg)")
577 schema.addField(
'telha', type=np.float64, doc=
"Pointing Hour Angle (deg)")
578 schema.addField(
'telrot', type=np.float64, doc=
"Camera rotation (deg)")
579 schema.addField(
'mjd', type=np.float64, doc=
"MJD of visit")
580 schema.addField(
'exptime', type=np.float32, doc=
"Exposure time")
581 schema.addField(
'pmb', type=np.float32, doc=
"Pressure (millibar)")
582 schema.addField(
'psfSigma', type=np.float32, doc=
"PSF sigma (reference CCD)")
583 schema.addField(
'deltaAper', type=np.float32, doc=
"Delta-aperture")
584 schema.addField(
'skyBackground', type=np.float32, doc=
"Sky background (ADU) (reference CCD)")
586 schema.addField(
'deepFlag', type=np.int32, doc=
"Deep observation")
587 schema.addField(
'scaling', type=
'ArrayD', doc=
"Scaling applied due to flat adjustment",
589 schema.addField(
'used', type=np.int32, doc=
"This visit has been ingested.")
590 schema.addField(
'sources_read', type=
'Flag', doc=
"This visit had sources read.")
594 def _makeFgcmObjSchema(self):
596 Make a schema for the objIndexCat
from fgcmMakeStars
600 schema: `afwTable.Schema`
603 objSchema = afwTable.Schema()
604 objSchema.addField('fgcm_id', type=np.int32, doc=
'FGCM Unique ID')
606 objSchema.addField(
'ra', type=np.float64, doc=
'Mean object RA (deg)')
607 objSchema.addField(
'dec', type=np.float64, doc=
'Mean object Dec (deg)')
608 objSchema.addField(
'obsArrIndex', type=np.int32,
609 doc=
'Index in obsIndexTable for first observation')
610 objSchema.addField(
'nObs', type=np.int32, doc=
'Total number of observations')
614 def _makeFgcmObsSchema(self):
616 Make a schema for the obsIndexCat
from fgcmMakeStars
620 schema: `afwTable.Schema`
623 obsSchema = afwTable.Schema()
624 obsSchema.addField('obsIndex', type=np.int32, doc=
'Index in observation table')
628 def _makeFgcmRefSchema(self, nReferenceBands):
630 Make a schema for the referenceCat
from fgcmMakeStars
634 nReferenceBands: `int`
635 Number of reference bands
639 schema: `afwTable.Schema`
642 refSchema = afwTable.Schema()
643 refSchema.addField('fgcm_id', type=np.int32, doc=
'FGCM Unique ID')
644 refSchema.addField(
'refMag', type=
'ArrayF', doc=
'Reference magnitude array (AB)',
645 size=nReferenceBands)
646 refSchema.addField(
'refMagErr', type=
'ArrayF', doc=
'Reference magnitude error array',
647 size=nReferenceBands)
651 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
653 Get the reference filter names, in wavelength order,
from the visitCat
and
654 information
from the look-up-table.
658 visitCat: `afw.table.BaseCatalog`
659 Catalog
with visit data
for FGCM
660 stdFilterDict: `dict`
661 Mapping of filterName to stdFilterName
from LUT
662 stdLambdaDict: `dict`
663 Mapping of stdFilterName to stdLambda
from LUT
667 referenceFilterNames: `list`
668 Wavelength-ordered list of reference filter names
672 filterNames = np.unique(visitCat.asAstropy()[
'physicalFilter'])
675 stdFilterNames = {stdFilterDict[filterName]
for filterName
in filterNames}
678 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
680 return referenceFilterNames
def _fillVisitCatalog(self, visitCat, groupedHandles, bkgHandleDict=None)
def __init__(self, initInputs=None, **kwargs)
def _makeFgcmVisitSchema(self, nCcd)
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
def _makeFgcmObjSchema(self)
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def _makeFgcmObsSchema(self)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def fgcmMakeVisitCatalog(self, camera, groupedHandles, bkgHandleDict=None)
def _makeFgcmRefSchema(self, nReferenceBands)