cycleNumber = pexConfig.Field(
doc="Final fit cycle from FGCM fit",
dtype=int,
default=0,
deprecated=("This config is no longer used, and will be removed after v25. "
"Please set config.connections.cycleNumber directly instead."),
)
physicalFilterMap = pexConfig.DictField(
doc="Mapping from 'physicalFilter' to band.",
keytype=str,
itemtype=str,
default={},
)
# The following fields refer to calibrating from a reference
# catalog, but in the future this might need to be expanded
doReferenceCalibration = pexConfig.Field(
doc=("Transfer 'absolute' calibration from reference catalog? "
"This afterburner step is unnecessary if reference stars "
"were used in the full fit in FgcmFitCycleTask."),
dtype=bool,
default=False,
)
doRefcatOutput = pexConfig.Field(
doc="Output standard stars in reference catalog format",
dtype=bool,
default=False,
deprecated="doRefcatOutput is no longer supported; this config will be removed after v24"
)
doAtmosphereOutput = pexConfig.Field(
doc="Output atmospheres in transmission_atmosphere_fgcm format",
dtype=bool,
default=True,
)
doZeropointOutput = pexConfig.Field(
doc="Output zeropoints in fgcm_photoCalib format",
dtype=bool,
default=True,
)
doComposeWcsJacobian = pexConfig.Field(
doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?",
dtype=bool,
default=True,
)
doApplyMeanChromaticCorrection = pexConfig.Field(
doc="Apply the mean chromatic correction to the zeropoints?",
dtype=bool,
default=True,
)
refObjLoader = pexConfig.ConfigurableField(
target=LoadIndexedReferenceObjectsTask,
doc="reference object loader for 'absolute' photometric calibration",
deprecated="refObjLoader is deprecated, and will be removed after v24",
)
photoCal = pexConfig.ConfigurableField(
target=PhotoCalTask,
doc="task to perform 'absolute' calibration",
)
referencePixelizationNside = pexConfig.Field(
doc="Healpix nside to pixelize catalog to compare to reference catalog",
dtype=int,
default=64,
)
referencePixelizationMinStars = pexConfig.Field(
doc=("Minimum number of stars per healpix pixel to select for comparison"
"to the specified reference catalog"),
dtype=int,
default=200,
)
referenceMinMatch = pexConfig.Field(
doc="Minimum number of stars matched to reference catalog to be used in statistics",
dtype=int,
default=50,
)
referencePixelizationNPixels = pexConfig.Field(
doc=("Number of healpix pixels to sample to do comparison. "
"Doing too many will take a long time and not yield any more "
"precise results because the final number is the median offset "
"(per band) from the set of pixels."),
dtype=int,
default=100,
)
datasetConfig = pexConfig.ConfigField(
dtype=DatasetConfig,
doc="Configuration for writing/reading ingested catalog",
deprecated="The datasetConfig was only used for gen2; this config will be removed after v24.",
)
def setDefaults(self):
pexConfig.Config.setDefaults(self)
# In order to transfer the "absolute" calibration from a reference
# catalog to the relatively calibrated FGCM standard stars (one number
# per band), we use the PhotoCalTask to match stars in a sample of healpix
# pixels. These basic settings ensure that only well-measured, good stars
# from the source and reference catalogs are used for the matching.
# applyColorTerms needs to be False if doReferenceCalibration is False,
# as is the new default after DM-16702
self.photoCal.applyColorTerms = False
self.photoCal.fluxField = 'instFlux'
self.photoCal.magErrFloor = 0.003
self.photoCal.match.referenceSelection.doSignalToNoise = True
self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0
self.photoCal.match.sourceSelection.doSignalToNoise = True
self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0
self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux'
self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr'
self.photoCal.match.sourceSelection.doFlags = True
self.photoCal.match.sourceSelection.flags.good = []
self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']
self.photoCal.match.sourceSelection.doUnresolved = False
class FgcmOutputProductsTask(pipeBase.PipelineTask):
ConfigClass = FgcmOutputProductsConfig
_DefaultName = "fgcmOutputProducts"
def __init__(self, **kwargs):
super().__init__(**kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = {}
handleDict['camera'] = butlerQC.get(inputRefs.camera)
handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
handleDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars)
if self.config.doZeropointOutput:
handleDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints)
photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']:
photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib}
if self.config.doAtmosphereOutput:
handleDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters)
atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for
atmRef in outputRefs.fgcmTransmissionAtmosphere}
if self.config.doReferenceCalibration:
refConfig = LoadReferenceObjectsConfig()
self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
name=self.config.connections.refCat,
log=self.log,
config=refConfig)
else:
self.refObjLoader = None
struct = self.run(handleDict, self.config.physicalFilterMap)
# Output the photoCalib exposure catalogs
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
# Output the atmospheres
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
if self.config.doReferenceCalibration:
# Turn offset into simple catalog for persistence if necessary
schema = afwTable.Schema()
schema.addField('offset', type=np.float64,
doc="Post-process calibration offset (mag)")
offsetCat = afwTable.BaseCatalog(schema)
offsetCat.resize(len(struct.offsets))
offsetCat['offset'][:] = struct.offsets
butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
return
def run(self, handleDict, physicalFilterMap):
stdCat = handleDict['fgcmStandardStars'].get()
md = stdCat.getMetadata()
bands = md.getArray('BANDS')
if self.config.doReferenceCalibration:
lutCat = handleDict['fgcmLookUpTable'].get()
offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands)
else:
offsets = np.zeros(len(bands))
del stdCat
if self.config.doZeropointOutput:
zptCat = handleDict['fgcmZeropoints'].get()
visitCat = handleDict['fgcmVisitCatalog'].get()
pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands,
physicalFilterMap)
else:
pcgen = None
if self.config.doAtmosphereOutput:
atmCat = handleDict['fgcmAtmosphereParameters'].get()
atmgen = self._outputAtmospheres(handleDict, atmCat)
else:
atmgen = None
retStruct = pipeBase.Struct(offsets=offsets,
atmospheres=atmgen)
retStruct.photoCalibCatalogs = pcgen
return retStruct
def generateTractOutputProducts(self, handleDict, tract,
visitCat, zptCat, atmCat, stdCat,
fgcmBuildStarsConfig):
physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
md = stdCat.getMetadata()
bands = md.getArray('BANDS')
if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
"in fgcmBuildStarsTask.")
if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
if self.config.doReferenceCalibration:
lutCat = handleDict['fgcmLookUpTable'].get()
offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap)
else:
offsets = np.zeros(len(bands))
if self.config.doZeropointOutput:
pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands,
physicalFilterMap)
else:
pcgen = None
if self.config.doAtmosphereOutput:
atmgen = self._outputAtmospheres(handleDict, atmCat)
else:
atmgen = None
retStruct = pipeBase.Struct(offsets=offsets,
atmospheres=atmgen)
retStruct.photoCalibCatalogs = pcgen
return retStruct
def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
# Only use stars that are observed in all the bands that were actually used
# This will ensure that we use the same healpix pixels for the absolute
# calibration of each band.
minObs = stdCat['ngood'].min(axis=1)
goodStars = (minObs >= 1)
stdCat = stdCat[goodStars]
self.log.info("Found %d stars with at least 1 good observation in each band" %
(len(stdCat)))
# Associate each band with the appropriate physicalFilter and make
# filterLabels
filterLabels = []
lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
physicalFilterMapBands = list(physicalFilterMap.values())
physicalFilterMapFilters = list(physicalFilterMap.keys())
for band in bands:
# Find a physical filter associated from the band by doing
# a reverse lookup on the physicalFilterMap dict
physicalFilterMapIndex = physicalFilterMapBands.index(band)
physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
# Find the appropriate fgcm standard physicalFilter
lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
filterLabels.append(afwImage.FilterLabel(band=band,
physical=stdPhysicalFilter))
# We have to make a table for each pixel with flux/fluxErr
# This is a temporary table generated for input to the photoCal task.
# These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
# have had chromatic corrections applied to get to the standard system
# specified by the atmosphere/instrumental parameters), nor are they
# in Jansky (since they don't have a proper absolute calibration: the overall
# zeropoint is estimated from the telescope size, etc.)
sourceMapper = afwTable.SchemaMapper(stdCat.schema)
sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
sourceMapper.editOutputSchema().addField('instFlux', type=np.float64,
doc="instrumental flux (counts)")
sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64,
doc="instrumental flux error (counts)")
badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
type='Flag',
doc="bad flag")
# Split up the stars
# Note that there is an assumption here that the ra/dec coords stored
# on-disk are in radians, and therefore that starObs['coord_ra'] /
# starObs['coord_dec'] return radians when used as an array of numpy float64s.
ipring = hpg.angle_to_pixel(
self.config.referencePixelizationNside,
stdCat['coord_ra'],
stdCat['coord_dec'],
degrees=False,
)
h, rev = esutil.stat.histogram(ipring, rev=True)
gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
self.log.info("Found %d pixels (nside=%d) with at least %d good stars" %
(gdpix.size,
self.config.referencePixelizationNside,
self.config.referencePixelizationMinStars))
if gdpix.size < self.config.referencePixelizationNPixels:
self.log.warning("Found fewer good pixels (%d) than preferred in configuration (%d)" %
(gdpix.size, self.config.referencePixelizationNPixels))
else:
# Sample out the pixels we want to use
gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)
results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),
('nstar', 'i4', len(bands)),
('nmatch', 'i4', len(bands)),
('zp', 'f4', len(bands)),
('zpErr', 'f4', len(bands))])
results['hpix'] = ipring[rev[rev[gdpix]]]
# We need a boolean index to deal with catalogs...
selected = np.zeros(len(stdCat), dtype=bool)
refFluxFields = [None]*len(bands)
for p_index, pix in enumerate(gdpix):
i1a = rev[rev[pix]: rev[pix + 1]]
# the stdCat afwTable can only be indexed with boolean arrays,
# and not numpy index arrays (see DM-16497). This little trick
# converts the index array into a boolean array
selected[:] = False
selected[i1a] = True
for b_index, filterLabel in enumerate(filterLabels):
struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index,
filterLabel, stdCat,
selected, refFluxFields)
results['nstar'][p_index, b_index] = len(i1a)
results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
results['zp'][p_index, b_index] = struct.zp
results['zpErr'][p_index, b_index] = struct.sigma
# And compute the summary statistics
offsets = np.zeros(len(bands))
for b_index, band in enumerate(bands):
# make configurable
ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
offsets[b_index] = np.median(results['zp'][ok, b_index])
# use median absolute deviation to estimate Normal sigma
# see https://en.wikipedia.org/wiki/Median_absolute_deviation
madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index]))
self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f",
band, offsets[b_index], madSigma)
return offsets
def _computeOffsetOneBand(self, sourceMapper, badStarKey,
b_index, filterLabel, stdCat, selected, refFluxFields):
Definition at line 667 of file fgcmOutputProducts.py.