23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs
as much cleaning of the
29input catalog
as possible.
37import lsst.pex.config as pexConfig
38import lsst.pipe.base as pipeBase
39from lsst.pipe.base import connectionTypes
40import lsst.afw.table as afwTable
41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
45from .utilities import lookupStaticCalibrations
47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument",),
53 camera = connectionTypes.PrerequisiteInput(
54 doc=
"Camera instrument",
56 storageClass=
"Camera",
57 dimensions=(
"instrument",),
58 lookupFunction=lookupStaticCalibrations,
62 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
63 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
64 "chromatic corrections."),
65 name=
"fgcmLookUpTable",
66 storageClass=
"Catalog",
67 dimensions=(
"instrument",),
71 sourceSchema = connectionTypes.InitInput(
72 doc=
"Schema for source catalogs",
74 storageClass=
"SourceCatalog",
77 refCat = connectionTypes.PrerequisiteInput(
78 doc=
"Reference catalog to use for photometric calibration",
80 storageClass=
"SimpleCatalog",
81 dimensions=(
"skypix",),
86 sourceTable_visit = connectionTypes.Input(
87 doc=
"Source table in parquet format, per visit",
88 name=
"sourceTable_visit",
89 storageClass=
"DataFrame",
90 dimensions=(
"instrument",
"visit"),
95 visitSummary = connectionTypes.Input(
96 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
97 "detector id for the id and must be sorted for fast lookups of a "
100 storageClass=
"ExposureCatalog",
101 dimensions=(
"instrument",
"visit"),
106 background = connectionTypes.Input(
107 doc=
"Calexp background model",
108 name=
"calexpBackground",
109 storageClass=
"Background",
110 dimensions=(
"instrument",
"visit",
"detector"),
115 fgcmVisitCatalog = connectionTypes.Output(
116 doc=
"Catalog of visit information for fgcm",
117 name=
"fgcmVisitCatalog",
118 storageClass=
"Catalog",
119 dimensions=(
"instrument",),
122 fgcmStarObservations = connectionTypes.Output(
123 doc=
"Catalog of star observations for fgcm",
124 name=
"fgcmStarObservations",
125 storageClass=
"Catalog",
126 dimensions=(
"instrument",),
129 fgcmStarIds = connectionTypes.Output(
130 doc=
"Catalog of fgcm calibration star IDs",
132 storageClass=
"Catalog",
133 dimensions=(
"instrument",),
136 fgcmStarIndices = connectionTypes.Output(
137 doc=
"Catalog of fgcm calibration star indices",
138 name=
"fgcmStarIndices",
139 storageClass=
"Catalog",
140 dimensions=(
"instrument",),
143 fgcmReferenceStars = connectionTypes.Output(
144 doc=
"Catalog of fgcm-matched reference stars",
145 name=
"fgcmReferenceStars",
146 storageClass=
"Catalog",
147 dimensions=(
"instrument",),
153 if not config.doReferenceMatches:
154 self.prerequisiteInputs.remove(
"refCat")
155 self.prerequisiteInputs.remove(
"fgcmLookUpTable")
157 if not config.doModelErrorsWithBackground:
158 self.inputs.remove(
"background")
160 if not config.doReferenceMatches:
161 self.outputs.remove(
"fgcmReferenceStars")
165 pipelineConnections=FgcmBuildStarsTableConnections):
166 """Config for FgcmBuildStarsTableTask"""
168 referenceCCD = pexConfig.Field(
169 doc=
"Reference CCD for checking PSF and background",
191 sourceSelector.flags.bad = [
'pixelFlags_edge',
192 'pixelFlags_interpolatedCenter',
193 'pixelFlags_saturatedCenter',
194 'pixelFlags_crCenter',
196 'pixelFlags_interpolated',
197 'pixelFlags_saturated',
203 sourceSelector.flags.bad.append(localBackgroundFlagName)
208 sourceSelector.isolated.parentName =
'parentSourceId'
209 sourceSelector.isolated.nChildName =
'deblend_nChild'
211 sourceSelector.unresolved.name =
'extendedness'
216 Build stars for the FGCM
global calibration, using sourceTable_visit catalogs.
218 ConfigClass = FgcmBuildStarsTableConfig
219 _DefaultName = "fgcmBuildStarsTable"
221 canMultiprocess =
False
224 super().
__init__(initInputs=initInputs, **kwargs)
225 if initInputs
is not None:
229 inputRefDict = butlerQC.get(inputRefs)
231 sourceTableHandles = inputRefDict[
'sourceTable_visit']
233 self.log.info(
"Running with %d sourceTable_visit handles",
234 len(sourceTableHandles))
236 sourceTableHandleDict = {sourceTableHandle.dataId[
'visit']: sourceTableHandle
for
237 sourceTableHandle
in sourceTableHandles}
239 if self.config.doReferenceMatches:
241 lutHandle = inputRefDict[
'fgcmLookUpTable']
244 refConfig = LoadReferenceObjectsConfig()
245 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
246 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
247 for ref
in inputRefs.refCat],
248 refCats=butlerQC.get(inputRefs.refCat),
249 name=self.config.connections.refCat,
252 self.makeSubtask(
'fgcmLoadReferenceCatalog',
253 refObjLoader=refObjLoader,
254 refCatName=self.config.connections.refCat)
260 calibFluxApertureRadius =
None
261 if self.config.doSubtractLocalBackground:
263 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
264 except RuntimeError
as e:
265 raise RuntimeError(
"Could not determine aperture radius from %s. "
266 "Cannot use doSubtractLocalBackground." %
267 (self.config.instFluxField))
from e
269 visitSummaryHandles = inputRefDict[
'visitSummary']
270 visitSummaryHandleDict = {visitSummaryHandle.dataId[
'visit']: visitSummaryHandle
for
271 visitSummaryHandle
in visitSummaryHandles}
273 camera = inputRefDict[
'camera']
275 visitSummaryHandleDict)
277 if self.config.doModelErrorsWithBackground:
278 bkgHandles = inputRefDict[
'background']
279 bkgHandleDict = {(bkgHandle.dataId.byName()[
'visit'],
280 bkgHandle.dataId.byName()[
'detector']): bkgHandle
for
281 bkgHandle
in bkgHandles}
286 bkgHandleDict=bkgHandleDict)
288 rad = calibFluxApertureRadius
293 calibFluxApertureRadius=rad)
295 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
296 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
298 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.
fgcmMatchStars(visitCat,
299 fgcmStarObservationCat,
302 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
303 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
304 if fgcmRefCat
is not None:
305 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
307 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
308 """Group sourceTable and visitSummary handles.
312 sourceTableHandleDict : `dict` [`int`, `str`]
313 Dict of source tables, keyed by visit.
314 visitSummaryHandleDict : `dict` [int, `str`]
315 Dict of visit summary catalogs, keyed by visit.
319 groupedHandles : `dict` [`int`, `list`]
320 Dictionary with sorted visit keys,
and `list`s
with
321 `lsst.daf.butler.DeferredDataSetHandle`. The first
322 item
in the list will be the visitSummary ref,
and
323 the second will be the source table ref.
325 groupedHandles = collections.defaultdict(list)
326 visits = sorted(sourceTableHandleDict.keys())
329 groupedHandles[visit] = [visitSummaryHandleDict[visit],
330 sourceTableHandleDict[visit]]
332 return groupedHandles
337 calibFluxApertureRadius=None):
338 startTime = time.time()
340 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
341 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
346 outputSchema = sourceMapper.getOutputSchema()
350 for ccdIndex, detector
in enumerate(camera):
351 ccdMapping[detector.getId()] = ccdIndex
353 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
355 fullCatalog = afwTable.BaseCatalog(outputSchema)
357 visitKey = outputSchema[
'visit'].asKey()
358 ccdKey = outputSchema[
'ccd'].asKey()
359 instMagKey = outputSchema[
'instMag'].asKey()
360 instMagErrKey = outputSchema[
'instMagErr'].asKey()
361 deltaMagAperKey = outputSchema[
'deltaMagAper'].asKey()
364 if self.config.doSubtractLocalBackground:
365 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
371 for counter, visit
in enumerate(visitCat):
372 expTime = visit[
'exptime']
374 handle = groupedHandles[visit[
'visit']][-1]
377 inColumns = handle.get(component=
'columns')
379 df = handle.get(parameters={
'columns': columns})
381 goodSrc = self.sourceSelector.selectSources(df)
385 if self.config.doSubtractLocalBackground:
386 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
387 use, = np.where((goodSrc.selected)
388 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
390 use, = np.where(goodSrc.selected)
392 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
393 tempCat.resize(use.size)
395 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
396 tempCat[
'dec'][:] = np.deg2rad(df[
'decl'].values[use])
397 tempCat[
'x'][:] = df[
'x'].values[use]
398 tempCat[
'y'][:] = df[
'y'].values[use]
400 tempCat[visitKey][:] = df[
'visit'].values[use]
401 tempCat[ccdKey][:] = df[detColumn].values[use]
402 tempCat[
'psf_candidate'] = df[self.config.psfCandidateName].values[use]
404 with np.warnings.catch_warnings():
406 np.warnings.simplefilter(
"ignore")
408 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
409 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use]
410 / df[self.config.apertureInnerInstFluxField].values[use])
411 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
412 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use]
413 / df[self.config.apertureOuterInstFluxField].values[use])
414 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
416 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
418 if self.config.doSubtractLocalBackground:
432 tempCat[
'deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
433 - localBackground[use]) -
434 -2.5*np.log10(df[self.config.instFluxField].values[use]))
436 tempCat[
'deltaMagBkg'][:] = 0.0
439 for detector
in camera:
440 ccdId = detector.getId()
442 use2 = (tempCat[ccdKey] == ccdId)
443 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
445 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
446 * visit[
'scaling'][ccdMapping[ccdId]])
447 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
451 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use]
452 / df[self.config.instFluxField].values[use])
455 if self.config.doApplyWcsJacobian:
456 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
458 fullCatalog.extend(tempCat)
460 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
461 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
463 visit[
'deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
464 visit[
'sources_read'] =
True
466 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
467 use.size, visit[
'visit'], visit[
'deltaAper'])
469 self.log.info(
"Found all good star observations in %.2f s" %
470 (time.time() - startTime))
474 def _get_sourceTable_visit_columns(self, inColumns):
476 Get the sourceTable_visit columns from the config.
481 List of columns available
in the sourceTable_visit
486 List of columns to read
from sourceTable_visit.
487 detectorColumn : `str`
488 Name of the detector column.
490 if 'detector' in inColumns:
492 detectorColumn =
'detector'
495 detectorColumn =
'ccd'
497 columns = [
'visit', detectorColumn,
498 'ra',
'decl',
'x',
'y', self.config.psfCandidateName,
499 self.config.instFluxField, self.config.instFluxField +
'Err',
500 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
501 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
502 if self.sourceSelector.config.doFlags:
503 columns.extend(self.sourceSelector.config.flags.bad)
504 if self.sourceSelector.config.doUnresolved:
505 columns.append(self.sourceSelector.config.unresolved.name)
506 if self.sourceSelector.config.doIsolated:
507 columns.append(self.sourceSelector.config.isolated.parentName)
508 columns.append(self.sourceSelector.config.isolated.nChildName)
509 if self.config.doSubtractLocalBackground:
510 columns.append(self.config.localBackgroundFluxField)
512 return columns, detectorColumn
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def _makeSourceMapper(self, sourceSchema)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def fgcmMakeVisitCatalog(self, camera, groupedHandles, bkgHandleDict=None)
apertureOuterInstFluxField
apertureInnerInstFluxField
doSubtractLocalBackground
apertureInnerInstFluxField
apertureOuterInstFluxField
def __init__(self, *config=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def _get_sourceTable_visit_columns(self, inColumns)
def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)
def __init__(self, initInputs=None, **kwargs)