lsst.fgcmcal g33a6e8c39e+92dd209c29
Loading...
Searching...
No Matches
fgcmFitCycle.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
24
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
29
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
35
36import copy
37
38import numpy as np
39
40import lsst.pex.config as pexConfig
41import lsst.pipe.base as pipeBase
42from lsst.pipe.base import connectionTypes
43import lsst.afw.table as afwTable
44
45from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
46from .utilities import extractReferenceMags
47from .utilities import makeZptSchema, makeZptCat
48from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
49from .sedterms import SedboundarytermDict, SedtermDict
50from .focalPlaneProjector import FocalPlaneProjector
51
52import fgcm
53
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask']
55
56MULTIPLE_CYCLES_MAX = 10
57
58
59class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("instrument",),
61 defaultTemplates={"previousCycleNumber": "-1",
62 "cycleNumber": "0"}):
63 camera = connectionTypes.PrerequisiteInput(
64 doc="Camera instrument",
65 name="camera",
66 storageClass="Camera",
67 dimensions=("instrument",),
68 isCalibration=True,
69 )
70
71 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
72 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
73 "chromatic corrections."),
74 name="fgcmLookUpTable",
75 storageClass="Catalog",
76 dimensions=("instrument",),
77 deferLoad=True,
78 )
79
80 fgcmVisitCatalog = connectionTypes.Input(
81 doc="Catalog of visit information for fgcm",
82 name="fgcmVisitCatalog",
83 storageClass="Catalog",
84 dimensions=("instrument",),
85 deferLoad=True,
86 )
87
88 fgcmStarObservationsParquet = connectionTypes.Input(
89 doc=("Catalog of star observations for fgcm, in parquet format. "
90 "Used if useParquetCatalogFormat is True."),
91 name="fgcm_star_observations",
92 storageClass="ArrowAstropy",
93 dimensions=("instrument",),
94 deferLoad=True,
95 )
96
97 fgcmStarIdsParquet = connectionTypes.Input(
98 doc=("Catalog of fgcm calibration star IDs, in parquet format. "
99 "Used if useParquetCatalogFormat is True."),
100 name="fgcm_star_ids",
101 storageClass="ArrowAstropy",
102 dimensions=("instrument",),
103 deferLoad=True,
104 )
105
106 fgcmReferenceStarsParquet = connectionTypes.Input(
107 doc=("Catalog of fgcm-matched reference stars, in parquet format. "
108 "Used if useParquetCatalogFormat is True."),
109 name="fgcm_reference_stars",
110 storageClass="ArrowAstropy",
111 dimensions=("instrument",),
112 deferLoad=True,
113 )
114
115 fgcmStarObservations = connectionTypes.Input(
116 doc=("Catalog of star observations for fgcm; old format. "
117 "Used if useParquetCatalogFormat is False."),
118 name="fgcmStarObservations",
119 storageClass="Catalog",
120 dimensions=("instrument",),
121 deferLoad=True,
122 )
123
124 fgcmStarIds = connectionTypes.Input(
125 doc=("Catalog of fgcm calibration star IDs. "
126 "Used if useParquetCatalogFormat is False."),
127 name="fgcmStarIds",
128 storageClass="Catalog",
129 dimensions=("instrument",),
130 deferLoad=True,
131 )
132
133 fgcmStarIndices = connectionTypes.Input(
134 doc=("Catalog of fgcm calibration star indices; old format."
135 "Used if useParquetCatalogFormat is False."),
136 name="fgcmStarIndices",
137 storageClass="Catalog",
138 dimensions=("instrument",),
139 deferLoad=True,
140 )
141
142 fgcmReferenceStars = connectionTypes.Input(
143 doc=("Catalog of fgcm-matched reference stars; old format."
144 "Used if useParquetCatalogFormat is False."),
145 name="fgcmReferenceStars",
146 storageClass="Catalog",
147 dimensions=("instrument",),
148 deferLoad=True,
149 )
150
151 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
152 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
153 name="fgcmFlaggedStars{previousCycleNumber}",
154 storageClass="Catalog",
155 dimensions=("instrument",),
156 deferLoad=True,
157 )
158
159 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
160 doc="Catalog of fgcm fit parameters from previous fit cycle",
161 name="fgcmFitParameters{previousCycleNumber}",
162 storageClass="Catalog",
163 dimensions=("instrument",),
164 deferLoad=True,
165 )
166
167 fgcmFitParameters = connectionTypes.Output(
168 doc="Catalog of fgcm fit parameters from current fit cycle",
169 name="fgcmFitParameters{cycleNumber}",
170 storageClass="Catalog",
171 dimensions=("instrument",),
172 )
173
174 fgcmFlaggedStars = connectionTypes.Output(
175 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
176 name="fgcmFlaggedStars{cycleNumber}",
177 storageClass="Catalog",
178 dimensions=("instrument",),
179 )
180
181 fgcmZeropoints = connectionTypes.Output(
182 doc="Catalog of fgcm zeropoint data from current fit cycle",
183 name="fgcmZeropoints{cycleNumber}",
184 storageClass="Catalog",
185 dimensions=("instrument",),
186 )
187
188 fgcmAtmosphereParameters = connectionTypes.Output(
189 doc="Catalog of atmospheric fit parameters from current fit cycle",
190 name="fgcmAtmosphereParameters{cycleNumber}",
191 storageClass="Catalog",
192 dimensions=("instrument",),
193 )
194
195 fgcmStandardStars = connectionTypes.Output(
196 doc="Catalog of standard star magnitudes from current fit cycle",
197 name="fgcmStandardStars{cycleNumber}",
198 storageClass="SimpleCatalog",
199 dimensions=("instrument",),
200 )
201
202 # Add connections for running multiple cycles
203 # This uses vars() to alter the class __dict__ to programmatically
204 # write many similar outputs.
205 for cycle in range(MULTIPLE_CYCLES_MAX):
206 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
207 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
208 name=f"fgcmFitParameters{cycle}",
209 storageClass="Catalog",
210 dimensions=("instrument",),
211 )
212 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
213 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
214 name=f"fgcmFlaggedStars{cycle}",
215 storageClass="Catalog",
216 dimensions=("instrument",),
217 )
218 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
219 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
220 name=f"fgcmZeropoints{cycle}",
221 storageClass="Catalog",
222 dimensions=("instrument",),
223 )
224 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
225 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
226 name=f"fgcmAtmosphereParameters{cycle}",
227 storageClass="Catalog",
228 dimensions=("instrument",),
229 )
230 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
231 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
232 name=f"fgcmStandardStars{cycle}",
233 storageClass="SimpleCatalog",
234 dimensions=("instrument",),
235 )
236
237 def __init__(self, *, config=None):
238 super().__init__(config=config)
239
240 if not config.doReferenceCalibration:
241 self.inputs.remove("fgcmReferenceStars")
242 self.inputs.remove("fgcmReferenceStarsParquet")
243
244 if config.useParquetCatalogFormat:
245 self.inputs.remove("fgcmStarObservations")
246 self.inputs.remove("fgcmStarIds")
247 self.inputs.remove("fgcmStarIndices")
248 if config.doReferenceCalibration:
249 self.inputs.remove("fgcmReferenceStars")
250 else:
251 self.inputs.remove("fgcmStarObservationsParquet")
252 self.inputs.remove("fgcmStarIdsParquet")
253 if config.doReferenceCalibration:
254 self.inputs.remove("fgcmReferenceStarsParquet")
255
256 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
257 raise ValueError("cycleNumber must be of integer format")
258 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
259 raise ValueError("previousCycleNumber must be of integer format")
260 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
261 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
262
263 if int(config.connections.cycleNumber) == 0:
264 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
265 self.prerequisiteInputs.remove("fgcmFitParametersInput")
266
267 if not self.config.doMultipleCycles:
268 # Single-cycle run
269 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
270 self.outputs.remove("fgcmStandardStars")
271
272 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
273 self.outputs.remove("fgcmZeropoints")
274 self.outputs.remove("fgcmAtmosphereParameters")
275
276 # Remove all multiple cycle outputs
277 for cycle in range(0, MULTIPLE_CYCLES_MAX):
278 self.outputs.remove(f"fgcmFitParameters{cycle}")
279 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
280 self.outputs.remove(f"fgcmZeropoints{cycle}")
281 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
282 self.outputs.remove(f"fgcmStandardStars{cycle}")
283
284 else:
285 # Multiple-cycle run
286 # Remove single-cycle outputs
287 self.outputs.remove("fgcmFitParameters")
288 self.outputs.remove("fgcmFlaggedStars")
289 self.outputs.remove("fgcmZeropoints")
290 self.outputs.remove("fgcmAtmosphereParameters")
291 self.outputs.remove("fgcmStandardStars")
292
293 # Remove outputs from cycles that are not used
294 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
295 MULTIPLE_CYCLES_MAX):
296 self.outputs.remove(f"fgcmFitParameters{cycle}")
297 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
298 self.outputs.remove(f"fgcmZeropoints{cycle}")
299 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
300 self.outputs.remove(f"fgcmStandardStars{cycle}")
301
302 # Remove non-final-cycle outputs if necessary
303 for cycle in range(self.config.multipleCyclesFinalCycleNumber):
304 if not self.config.outputZeropointsBeforeFinalCycle:
305 self.outputs.remove(f"fgcmZeropoints{cycle}")
306 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
307 if not self.config.outputStandardsBeforeFinalCycle:
308 self.outputs.remove(f"fgcmStandardStars{cycle}")
309
310
311class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
312 pipelineConnections=FgcmFitCycleConnections):
313 """Config for FgcmFitCycle"""
314
315 doMultipleCycles = pexConfig.Field(
316 doc="Run multiple fit cycles in one task",
317 dtype=bool,
318 default=False,
319 )
320 useParquetCatalogFormat = pexConfig.Field(
321 doc="Use parquet catalog format?",
322 dtype=bool,
323 default=True,
324 )
325 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
326 doc=("Final cycle number in multiple cycle mode. The initial cycle "
327 "is 0, with limited parameters fit. The next cycle is 1 with "
328 "full parameter fit. The final cycle is a clean-up with no "
329 "parameters fit. There will be a total of "
330 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
331 "cycle number cannot be less than 2."),
332 dtype=int,
333 default=5,
334 min=2,
335 max=MULTIPLE_CYCLES_MAX,
336 inclusiveMax=True,
337 )
338 bands = pexConfig.ListField(
339 doc="Bands to run calibration",
340 dtype=str,
341 default=[],
342 )
343 fitBands = pexConfig.ListField(
344 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
345 "the atmosphere constrained from the 'fitBands' on the same night. "
346 "Must be a subset of `config.bands`"),
347 dtype=str,
348 default=[],
349 )
350 requiredBands = pexConfig.ListField(
351 doc=("Bands that are required for a star to be considered a calibration star. "
352 "Must be a subset of `config.bands`"),
353 dtype=str,
354 default=[],
355 )
356 physicalFilterMap = pexConfig.DictField(
357 doc="Mapping from 'physicalFilter' to band.",
358 keytype=str,
359 itemtype=str,
360 default={},
361 )
362 doReferenceCalibration = pexConfig.Field(
363 doc="Use reference catalog as additional constraint on calibration",
364 dtype=bool,
365 default=True,
366 )
367 refStarSnMin = pexConfig.Field(
368 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
369 dtype=float,
370 default=50.0,
371 )
372 refStarOutlierNSig = pexConfig.Field(
373 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
374 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
375 dtype=float,
376 default=4.0,
377 )
378 applyRefStarColorCuts = pexConfig.Field(
379 doc=("Apply color cuts defined in ``starColorCuts`` to reference stars? "
380 "These cuts are in addition to any cuts defined in ``refStarColorCuts``"),
381 dtype=bool,
382 default=True,
383 )
384 useExposureReferenceOffset = pexConfig.Field(
385 doc=("Use per-exposure (visit) offsets between calibrated stars and reference stars "
386 "for final zeropoints? This may help uniformity for disjoint surveys."),
387 dtype=bool,
388 default=False,
389 )
390 nCore = pexConfig.Field(
391 doc="Number of cores to use",
392 dtype=int,
393 default=4,
394 deprecated="Number of cores is deprecated as a config, and will be removed after v27. "
395 "Please use ``pipetask run --cores-per-quantum`` instead.",
396 )
397 nStarPerRun = pexConfig.Field(
398 doc="Number of stars to run in each chunk",
399 dtype=int,
400 default=200000,
401 )
402 nExpPerRun = pexConfig.Field(
403 doc="Number of exposures to run in each chunk",
404 dtype=int,
405 default=1000,
406 )
407 reserveFraction = pexConfig.Field(
408 doc="Fraction of stars to reserve for testing",
409 dtype=float,
410 default=0.1,
411 )
412 freezeStdAtmosphere = pexConfig.Field(
413 doc="Freeze atmosphere parameters to standard (for testing)",
414 dtype=bool,
415 default=False,
416 )
417 precomputeSuperStarInitialCycle = pexConfig.Field(
418 doc="Precompute superstar flat for initial cycle",
419 dtype=bool,
420 default=False,
421 )
422 superStarSubCcdDict = pexConfig.DictField(
423 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
424 "Must have one entry per band."),
425 keytype=str,
426 itemtype=bool,
427 default={},
428 )
429 superStarSubCcdChebyshevOrder = pexConfig.Field(
430 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
431 "Global default is first-order polynomials, and should be overridden "
432 "on a camera-by-camera basis depending on the ISR."),
433 dtype=int,
434 default=1,
435 )
436 superStarSubCcdTriangular = pexConfig.Field(
437 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
438 "suppress high-order cross terms?"),
439 dtype=bool,
440 default=False,
441 )
442 superStarSigmaClip = pexConfig.Field(
443 doc="Number of sigma to clip outliers when selecting for superstar flats",
444 dtype=float,
445 default=5.0,
446 )
447 superStarPlotCcdResiduals = pexConfig.Field(
448 doc="If plotting is enabled, should per-detector residuals be plotted? "
449 "This may produce a lot of output, and should be used only for "
450 "debugging purposes.",
451 dtype=bool,
452 default=False,
453 )
454 focalPlaneSigmaClip = pexConfig.Field(
455 doc="Number of sigma to clip outliers per focal-plane.",
456 dtype=float,
457 default=4.0,
458 )
459 ccdGraySubCcdDict = pexConfig.DictField(
460 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
461 "('ccd gray') on a sub-ccd scale."),
462 keytype=str,
463 itemtype=bool,
464 default={},
465 )
466 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
467 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
468 dtype=int,
469 default=1,
470 )
471 ccdGraySubCcdTriangular = pexConfig.Field(
472 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
473 "suppress high-order cross terms?"),
474 dtype=bool,
475 default=True,
476 )
477 ccdGrayFocalPlaneDict = pexConfig.DictField(
478 doc=("Per-band specification on whether to compute focal-plane residual "
479 "('ccd gray') corrections."),
480 keytype=str,
481 itemtype=bool,
482 default={},
483 )
484 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
485 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
486 "gray corrections. If there are fewer good CCDs then the gray "
487 "correction is computed per-ccd."),
488 dtype=int,
489 default=1,
490 )
491 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
492 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
493 dtype=int,
494 default=3,
495 )
496 cycleNumber = pexConfig.Field(
497 doc=("FGCM fit cycle number. This is automatically incremented after each run "
498 "and stage of outlier rejection. See cookbook for details."),
499 dtype=int,
500 default=None,
501 )
502 isFinalCycle = pexConfig.Field(
503 doc=("Is this the final cycle of the fitting? Will automatically compute final "
504 "selection of stars and photometric exposures, and will output zeropoints "
505 "and standard stars for use in fgcmOutputProducts"),
506 dtype=bool,
507 default=False,
508 )
509 maxIterBeforeFinalCycle = pexConfig.Field(
510 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
511 "will always be 0 in the final cycle for cleanup and final selection."),
512 dtype=int,
513 default=50,
514 )
515 deltaMagBkgOffsetPercentile = pexConfig.Field(
516 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
517 "offset from local background subtraction."),
518 dtype=float,
519 default=0.25,
520 )
521 deltaMagBkgPerCcd = pexConfig.Field(
522 doc=("Compute net offset from local background subtraction per-ccd? "
523 "Otherwise, use computation per visit."),
524 dtype=bool,
525 default=False,
526 )
527 utBoundary = pexConfig.Field(
528 doc="Boundary (in UTC) from day-to-day",
529 dtype=float,
530 default=None,
531 )
532 washMjds = pexConfig.ListField(
533 doc="Mirror wash MJDs",
534 dtype=float,
535 default=(0.0,),
536 )
537 epochMjds = pexConfig.ListField(
538 doc="Epoch boundaries in MJD",
539 dtype=float,
540 default=(0.0,),
541 )
542 minObsPerBand = pexConfig.Field(
543 doc="Minimum good observations per band",
544 dtype=int,
545 default=2,
546 )
547 # TODO: When DM-16511 is done, it will be possible to get the
548 # telescope latitude directly from the camera.
549 latitude = pexConfig.Field(
550 doc="Observatory latitude",
551 dtype=float,
552 default=None,
553 )
554 mirrorArea = pexConfig.Field(
555 doc="Mirror area (square meters) of telescope. If not set, will "
556 "try to estimate from camera.telescopeDiameter.",
557 dtype=float,
558 default=None,
559 optional=True,
560 )
561 defaultCameraOrientation = pexConfig.Field(
562 doc="Default camera orientation for QA plots.",
563 dtype=float,
564 default=None,
565 )
566 brightObsGrayMax = pexConfig.Field(
567 doc="Maximum gray extinction to be considered bright observation",
568 dtype=float,
569 default=0.15,
570 )
571 minStarPerCcd = pexConfig.Field(
572 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
573 "CCDs with fewer stars will have their calibration estimated from other "
574 "CCDs in the same visit, with zeropoint error increased accordingly."),
575 dtype=int,
576 default=5,
577 )
578 minCcdPerExp = pexConfig.Field(
579 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
580 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
581 dtype=int,
582 default=5,
583 )
584 maxCcdGrayErr = pexConfig.Field(
585 doc="Maximum error on CCD gray offset to be considered photometric",
586 dtype=float,
587 default=0.05,
588 )
589 minStarPerExp = pexConfig.Field(
590 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
591 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
592 dtype=int,
593 default=600,
594 )
595 minExpPerNight = pexConfig.Field(
596 doc="Minimum number of good exposures/visits to consider a partly photometric night",
597 dtype=int,
598 default=10,
599 )
600 expGrayInitialCut = pexConfig.Field(
601 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
602 "observations."),
603 dtype=float,
604 default=-0.25,
605 )
606 expGrayPhotometricCutDict = pexConfig.DictField(
607 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
608 "('gray term') for a visit to be considered photometric. Must have one "
609 "entry per band. Broad-band filters should be -0.05."),
610 keytype=str,
611 itemtype=float,
612 default={},
613 )
614 expGrayHighCutDict = pexConfig.DictField(
615 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
616 "('gray term') for a visit to be considered photometric. Must have one "
617 "entry per band. Broad-band filters should be 0.2."),
618 keytype=str,
619 itemtype=float,
620 default={},
621 )
622 expGrayRecoverCut = pexConfig.Field(
623 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
624 "Visits with more gray extinction will only get CCD zeropoints if there are "
625 "sufficient star observations (minStarPerCcd) on that CCD."),
626 dtype=float,
627 default=-1.0,
628 )
629 expVarGrayPhotometricCutDict = pexConfig.DictField(
630 doc=("Per-band specification on maximum exposure variance to be considered possibly "
631 "photometric. Must have one entry per band. Broad-band filters should be "
632 "0.0005."),
633 keytype=str,
634 itemtype=float,
635 default={},
636 )
637 expGrayErrRecoverCut = pexConfig.Field(
638 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
639 "Visits with more gray variance will only get CCD zeropoints if there are "
640 "sufficient star observations (minStarPerCcd) on that CCD."),
641 dtype=float,
642 default=0.05,
643 )
644 aperCorrFitNBins = pexConfig.Field(
645 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
646 "no fit will be performed, and the config.aperCorrInputSlopes will be "
647 "used if available."),
648 dtype=int,
649 default=10,
650 )
651 aperCorrInputSlopeDict = pexConfig.DictField(
652 doc=("Per-band specification of aperture correction input slope parameters. These "
653 "are used on the first fit iteration, and aperture correction parameters will "
654 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
655 "to set this when there is insufficient data to fit the parameters (e.g. "
656 "tract mode)."),
657 keytype=str,
658 itemtype=float,
659 default={},
660 )
661 sedboundaryterms = pexConfig.ConfigField(
662 doc="Mapping from bands to SED boundary term names used is sedterms.",
663 dtype=SedboundarytermDict,
664 )
665 sedterms = pexConfig.ConfigField(
666 doc="Mapping from terms to bands for fgcm linear SED approximations.",
667 dtype=SedtermDict,
668 )
669 sigFgcmMaxErr = pexConfig.Field(
670 doc="Maximum mag error for fitting sigma_FGCM",
671 dtype=float,
672 default=0.01,
673 )
674 sigFgcmMaxEGrayDict = pexConfig.DictField(
675 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
676 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
677 "should be 0.05."),
678 keytype=str,
679 itemtype=float,
680 default={},
681 )
682 ccdGrayMaxStarErr = pexConfig.Field(
683 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
684 "computation"),
685 dtype=float,
686 default=0.10,
687 )
688 approxThroughputDict = pexConfig.DictField(
689 doc=("Per-band specification of the approximate overall throughput at the start of "
690 "calibration observations. Must have one entry per band. Typically should "
691 "be 1.0."),
692 keytype=str,
693 itemtype=float,
694 default={},
695 )
696 sigmaCalRange = pexConfig.ListField(
697 doc="Allowed range for systematic error floor estimation",
698 dtype=float,
699 default=(0.001, 0.003),
700 )
701 sigmaCalFitPercentile = pexConfig.ListField(
702 doc="Magnitude percentile range to fit systematic error floor",
703 dtype=float,
704 default=(0.05, 0.15),
705 )
706 sigmaCalPlotPercentile = pexConfig.ListField(
707 doc="Magnitude percentile range to plot systematic error floor",
708 dtype=float,
709 default=(0.05, 0.95),
710 )
711 sigma0Phot = pexConfig.Field(
712 doc="Systematic error floor for all zeropoints",
713 dtype=float,
714 default=0.003,
715 )
716 mapLongitudeRef = pexConfig.Field(
717 doc="Reference longitude for plotting maps",
718 dtype=float,
719 default=0.0,
720 )
721 mapNSide = pexConfig.Field(
722 doc="Healpix nside for plotting maps",
723 dtype=int,
724 default=256,
725 )
726 outfileBase = pexConfig.Field(
727 doc="Filename start for plot output files",
728 dtype=str,
729 default=None,
730 )
731 starColorCuts = pexConfig.ListField(
732 doc=("Encoded star-color cuts (using calibration star colors). "
733 "This is a list with each entry a string of the format "
734 "``band1,band2,low,high`` such that only stars of color "
735 "low < band1 - band2 < high will be used for calibration."),
736 dtype=str,
737 default=("NO_DATA",),
738 )
739 refStarColorCuts = pexConfig.ListField(
740 doc=("Encoded star color cuts specifically to apply to reference stars. "
741 "This is a list with each entry a string of the format "
742 "``band1,band2,low,high`` such that only stars of color "
743 "low < band1 - band2 < high will be used as reference stars."),
744 dtype=str,
745 default=("NO_DATA",),
746 )
747 colorSplitBands = pexConfig.ListField(
748 doc="Band names to use to split stars by color. Must have 2 entries.",
749 dtype=str,
750 length=2,
751 default=('g', 'i'),
752 )
753 modelMagErrors = pexConfig.Field(
754 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
755 dtype=bool,
756 default=True,
757 )
758 useQuadraticPwv = pexConfig.Field(
759 doc="Model PWV with a quadratic term for variation through the night?",
760 dtype=bool,
761 default=False,
762 )
763 instrumentParsPerBand = pexConfig.Field(
764 doc=("Model instrumental parameters per band? "
765 "Otherwise, instrumental parameters (QE changes with time) are "
766 "shared among all bands."),
767 dtype=bool,
768 default=False,
769 )
770 instrumentSlopeMinDeltaT = pexConfig.Field(
771 doc=("Minimum time change (in days) between observations to use in constraining "
772 "instrument slope."),
773 dtype=float,
774 default=20.0,
775 )
776 fitMirrorChromaticity = pexConfig.Field(
777 doc="Fit (intraband) mirror chromatic term?",
778 dtype=bool,
779 default=False,
780 )
781 fitCcdChromaticityDict = pexConfig.DictField(
782 doc="Specification on whether to compute first-order quantum efficiency (QE) "
783 "adjustments. Key is band, and value will be True or False. Any band "
784 "not explicitly specified will default to False.",
785 keytype=str,
786 itemtype=bool,
787 default={},
788 )
789 coatingMjds = pexConfig.ListField(
790 doc="Mirror coating dates in MJD",
791 dtype=float,
792 default=(0.0,),
793 )
794 outputStandardsBeforeFinalCycle = pexConfig.Field(
795 doc="Output standard stars prior to final cycle? Used in debugging.",
796 dtype=bool,
797 default=False,
798 )
799 outputZeropointsBeforeFinalCycle = pexConfig.Field(
800 doc="Output standard stars prior to final cycle? Used in debugging.",
801 dtype=bool,
802 default=False,
803 )
804 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
805 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
806 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
807 keytype=str,
808 itemtype=bool,
809 default={},
810 )
811 autoPhotometricCutNSig = pexConfig.Field(
812 doc=("Number of sigma for automatic computation of (low) photometric cut. "
813 "Cut is based on exposure gray width (per band), unless "
814 "useRepeatabilityForExpGrayCuts is set, in which case the star "
815 "repeatability is used (also per band)."),
816 dtype=float,
817 default=3.0,
818 )
819 autoHighCutNSig = pexConfig.Field(
820 doc=("Number of sigma for automatic computation of (high) outlier cut. "
821 "Cut is based on exposure gray width (per band), unless "
822 "useRepeatabilityForExpGrayCuts is set, in which case the star "
823 "repeatability is used (also per band)."),
824 dtype=float,
825 default=4.0,
826 )
827 quietMode = pexConfig.Field(
828 doc="Be less verbose with logging.",
829 dtype=bool,
830 default=False,
831 )
832 doPlots = pexConfig.Field(
833 doc="Make fgcm QA plots.",
834 dtype=bool,
835 default=True,
836 )
837 randomSeed = pexConfig.Field(
838 doc="Random seed for fgcm for consistency in tests.",
839 dtype=int,
840 default=None,
841 optional=True,
842 )
843 deltaAperFitMinNgoodObs = pexConfig.Field(
844 doc="Minimum number of good observations to use mean delta-aper values in fits.",
845 dtype=int,
846 default=2,
847 )
848 deltaAperFitPerCcdNx = pexConfig.Field(
849 doc=("Number of x bins per ccd when computing delta-aper background offsets. "
850 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
851 dtype=int,
852 default=10,
853 )
854 deltaAperFitPerCcdNy = pexConfig.Field(
855 doc=("Number of y bins per ccd when computing delta-aper background offsets. "
856 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
857 dtype=int,
858 default=10,
859 )
860 deltaAperFitSpatialNside = pexConfig.Field(
861 doc="Healpix nside to compute spatial delta-aper background offset maps.",
862 dtype=int,
863 default=64,
864 )
865 deltaAperInnerRadiusArcsec = pexConfig.Field(
866 doc=("Inner radius used to compute deltaMagAper (arcseconds). "
867 "Must be positive and less than ``deltaAperOuterRadiusArcsec`` if "
868 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
869 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
870 dtype=float,
871 default=0.0,
872 )
873 deltaAperOuterRadiusArcsec = pexConfig.Field(
874 doc=("Outer radius used to compute deltaMagAper (arcseconds). "
875 "Must be positive and greater than ``deltaAperInnerRadiusArcsec`` if "
876 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
877 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
878 dtype=float,
879 default=0.0,
880 )
881 doComputeDeltaAperPerVisit = pexConfig.Field(
882 doc=("Do the computation of delta-aper background offsets per visit? "
883 "Note: this option can be very slow when there are many visits."),
884 dtype=bool,
885 default=False,
886 )
887 doComputeDeltaAperPerStar = pexConfig.Field(
888 doc="Do the computation of delta-aper mean values per star?",
889 dtype=bool,
890 default=True,
891 )
892 doComputeDeltaAperMap = pexConfig.Field(
893 doc=("Do the computation of delta-aper spatial maps? "
894 "This is only used if ``doComputeDeltaAperPerStar`` is True,"),
895 dtype=bool,
896 default=False,
897 )
898 doComputeDeltaAperPerCcd = pexConfig.Field(
899 doc="Do the computation of per-ccd delta-aper background offsets?",
900 dtype=bool,
901 default=False,
902 )
903
904 def validate(self):
905 super().validate()
906
907 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
908 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
909 raise RuntimeError(msg)
910 if self.connections.cycleNumber != str(self.cycleNumber):
911 msg = "cycleNumber in template must be equal to connections.cycleNumber"
912 raise RuntimeError(msg)
913
914 for band in self.fitBands:
915 if band not in self.bands:
916 msg = 'fitBand %s not in bands' % (band)
917 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
918 for band in self.requiredBands:
919 if band not in self.bands:
920 msg = 'requiredBand %s not in bands' % (band)
921 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
922 for band in self.colorSplitBands:
923 if band not in self.bands:
924 msg = 'colorSplitBand %s not in bands' % (band)
925 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
926 for band in self.bands:
927 if band not in self.superStarSubCcdDict:
928 msg = 'band %s not in superStarSubCcdDict' % (band)
929 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
930 self, msg)
931 if band not in self.ccdGraySubCcdDict:
932 msg = 'band %s not in ccdGraySubCcdDict' % (band)
933 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
934 self, msg)
935 if band not in self.expGrayPhotometricCutDict:
936 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
937 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
938 self, msg)
939 if band not in self.expGrayHighCutDict:
940 msg = 'band %s not in expGrayHighCutDict' % (band)
941 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
942 self, msg)
943 if band not in self.expVarGrayPhotometricCutDict:
944 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
945 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
946 self, msg)
947 if band not in self.sigFgcmMaxEGrayDict:
948 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
949 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
950 self, msg)
951 if band not in self.approxThroughputDict:
952 msg = 'band %s not in approxThroughputDict' % (band)
953 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
954 self, msg)
955 if band not in self.useRepeatabilityForExpGrayCutsDict:
956 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
957 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
958 self, msg)
959
960 if self.doComputeDeltaAperPerVisit or self.doComputeDeltaAperMap \
961 or self.doComputeDeltaAperPerCcd:
962 if self.deltaAperInnerRadiusArcsec <= 0.0:
963 msg = 'deltaAperInnerRadiusArcsec must be positive if deltaAper computations are turned on.'
964 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperInnerRadiusArcsec,
965 self, msg)
966 if self.deltaAperOuterRadiusArcsec <= 0.0:
967 msg = 'deltaAperOuterRadiusArcsec must be positive if deltaAper computations are turned on.'
968 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
969 self, msg)
970 if self.deltaAperOuterRadiusArcsec <= self.deltaAperInnerRadiusArcsec:
971 msg = ('deltaAperOuterRadiusArcsec must be greater than deltaAperInnerRadiusArcsec if '
972 'deltaAper computations are turned on.')
973 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
974 self, msg)
975
976
977class FgcmFitCycleTask(pipeBase.PipelineTask):
978 """
979 Run Single fit cycle for FGCM global calibration
980 """
981
982 ConfigClass = FgcmFitCycleConfig
983 _DefaultName = "fgcmFitCycle"
984
985 def __init__(self, initInputs=None, **kwargs):
986 super().__init__(**kwargs)
987
988 def runQuantum(self, butlerQC, inputRefs, outputRefs):
989 camera = butlerQC.get(inputRefs.camera)
990
991 nCore = butlerQC.resources.num_cores
992
993 handleDict = {}
994
995 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
996 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
997
998 if self.config.useParquetCatalogFormat:
999 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservationsParquet)
1000 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIdsParquet)
1001 if self.config.doReferenceCalibration:
1002 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStarsParquet)
1003 else:
1004 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
1005 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
1006 handleDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
1007 if self.config.doReferenceCalibration:
1008 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
1009 if self.config.cycleNumber > 0:
1010 handleDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
1011 handleDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
1012
1013 fgcmDatasetDict = None
1014 if self.config.doMultipleCycles:
1015 # Run multiple cycles at once.
1016 config = copy.copy(self.config)
1017 config.update(cycleNumber=0)
1018 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
1019 if cycle == self.config.multipleCyclesFinalCycleNumber:
1020 config.update(isFinalCycle=True)
1021
1022 if cycle > 0:
1023 handleDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
1024 handleDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
1025
1026 fgcmDatasetDict, config = self._fgcmFitCycle(camera, handleDict, config=config, nCore=nCore)
1027 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
1028 getattr(outputRefs, f'fgcmFitParameters{cycle}'))
1029 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
1030 getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
1031 if self.outputZeropoints:
1032 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
1033 getattr(outputRefs, f'fgcmZeropoints{cycle}'))
1034 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
1035 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
1036 if self.outputStandards:
1037 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
1038 getattr(outputRefs, f'fgcmStandardStars{cycle}'))
1039 else:
1040 # Run a single cycle
1041 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, handleDict, nCore=nCore)
1042
1043 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
1044 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
1045 if self.outputZeropoints:
1046 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
1047 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
1048 if self.outputStandards:
1049 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
1050
1051 def _fgcmFitCycle(self, camera, handleDict, config=None, nCore=1):
1052 """
1053 Run the fit cycle
1054
1055 Parameters
1056 ----------
1057 camera : `lsst.afw.cameraGeom.Camera`
1058 handleDict : `dict`
1059 All handles are `lsst.daf.butler.DeferredDatasetHandle`
1060 handle dictionary with keys:
1061
1062 ``"fgcmLookUpTable"``
1063 handle for the FGCM look-up table.
1064 ``"fgcmVisitCatalog"``
1065 handle for visit summary catalog.
1066 ``"fgcmStarObservations"``
1067 handle for star observation catalog.
1068 ``"fgcmStarIds"``
1069 handle for star id catalog.
1070 ``"fgcmStarIndices"``
1071 handle for star index catalog.
1072 ``"fgcmReferenceStars"``
1073 handle for matched reference star catalog.
1074 ``"fgcmFlaggedStars"``
1075 handle for flagged star catalog.
1076 ``"fgcmFitParameters"``
1077 handle for fit parameter catalog.
1078 config : `lsst.pex.config.Config`, optional
1079 Configuration to use to override self.config.
1080 nCore : `int`, optional
1081 Number of cores to use during fitting.
1082
1083 Returns
1084 -------
1085 fgcmDatasetDict : `dict`
1086 Dictionary of datasets to persist.
1087 """
1088 if config is not None:
1089 _config = config
1090 else:
1091 _config = self.config
1092
1093 # Set defaults on whether to output standards and zeropoints
1094 self.maxIter = _config.maxIterBeforeFinalCycle
1095 self.outputStandards = _config.outputStandardsBeforeFinalCycle
1096 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1097 self.resetFitParameters = True
1098
1099 if _config.isFinalCycle:
1100 # This is the final fit cycle, so we do not want to reset fit
1101 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1102 # and we always want to output standards and zeropoints
1103 self.maxIter = 0
1104 self.outputStandards = True
1105 self.outputZeropoints = True
1106 self.resetFitParameters = False
1107
1108 lutCat = handleDict['fgcmLookUpTable'].get()
1109 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1110 dict(_config.physicalFilterMap))
1111 del lutCat
1112
1113 configDict = makeConfigDict(_config, self.log, camera,
1114 self.maxIter, self.resetFitParameters,
1115 self.outputZeropoints,
1116 lutIndexVals[0]['FILTERNAMES'],
1117 nCore=nCore)
1118
1119 # next we need the exposure/visit information
1120 visitCat = handleDict['fgcmVisitCatalog'].get()
1121 fgcmExpInfo = translateVisitCatalog(visitCat)
1122 del visitCat
1123
1124 focalPlaneProjector = FocalPlaneProjector(camera,
1125 self.config.defaultCameraOrientation)
1126
1127 noFitsDict = {'lutIndex': lutIndexVals,
1128 'lutStd': lutStd,
1129 'expInfo': fgcmExpInfo,
1130 'focalPlaneProjector': focalPlaneProjector}
1131
1132 # set up the fitter object
1133 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1134 noFitsDict=noFitsDict, noOutput=True)
1135
1136 # create the parameter object
1137 if (fgcmFitCycle.initialCycle):
1138 # cycle = 0, initial cycle
1139 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1140 fgcmLut,
1141 fgcmExpInfo)
1142 else:
1143 if isinstance(handleDict['fgcmFitParameters'], afwTable.BaseCatalog):
1144 parCat = handleDict['fgcmFitParameters']
1145 else:
1146 parCat = handleDict['fgcmFitParameters'].get()
1147 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1148 del parCat
1149 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1150 fgcmExpInfo,
1151 inParInfo,
1152 inParams,
1153 inSuperStar)
1154
1155 # set up the stars...
1156 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1157
1158 starObs = handleDict['fgcmStarObservations'].get()
1159 starIds = handleDict['fgcmStarIds'].get()
1160 if not self.config.useParquetCatalogFormat:
1161 starIndices = handleDict['fgcmStarIndices'].get()
1162 else:
1163 starIndices = None
1164
1165 # grab the flagged stars if available
1166 if 'fgcmFlaggedStars' in handleDict:
1167 if isinstance(handleDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1168 flaggedStars = handleDict['fgcmFlaggedStars']
1169 else:
1170 flaggedStars = handleDict['fgcmFlaggedStars'].get()
1171 flagId = flaggedStars['objId'][:]
1172 flagFlag = flaggedStars['objFlag'][:]
1173
1174 del flaggedStars
1175 elif self.config.useParquetCatalogFormat:
1176 # If we are using the parquet catalog format, then that means that
1177 # reserved stars have already been flagged. We extract the flags here
1178 # to input to fgcm, which will then be persisted (with additional
1179 # quality flags) as the fgcmFlaggedStars datatype in subsequent
1180 # fit cycles.
1181 (flagged,) = (starIds['obj_flag'] > 0).nonzero()
1182 flagId = starIds['fgcm_id'][flagged]
1183 flagFlag = starIds['obj_flag'][flagged]
1184 else:
1185 flagId = None
1186 flagFlag = None
1187
1188 if _config.doReferenceCalibration:
1189 refStars = handleDict['fgcmReferenceStars'].get()
1190
1191 refMag, refMagErr = extractReferenceMags(refStars,
1192 _config.bands,
1193 _config.physicalFilterMap)
1194
1195 refId = refStars['fgcm_id'][:]
1196 else:
1197 refStars = None
1198 refId = None
1199 refMag = None
1200 refMagErr = None
1201
1202 # match star observations to visits
1203 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1204 # actually be transferred into fgcm using the indexing below.
1205 if self.config.useParquetCatalogFormat:
1206 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'])
1207 else:
1208 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1209
1210 # The fgcmStars.loadStars method will copy all the star information into
1211 # special shared memory objects that will not blow up the memory usage when
1212 # used with python multiprocessing. Once all the numbers are copied,
1213 # it is necessary to release all references to the objects that previously
1214 # stored the data to ensure that the garbage collector can clear the memory,
1215 # and ensure that this memory is not copied when multiprocessing kicks in.
1216
1217 if self.config.useParquetCatalogFormat:
1218 # Note that the ra/dec coordinates for the parquet format are in
1219 # degrees, which is what fgcm expects.
1220 fgcmStars.loadStars(fgcmPars,
1221 starObs['visit'][:],
1222 starObs['detector'][:],
1223 starObs['ra'][:],
1224 starObs['dec'][:],
1225 starObs['inst_mag'][:],
1226 starObs['inst_mag_err'][:],
1227 fgcmExpInfo['FILTERNAME'][visitIndex],
1228 starIds['fgcm_id'][:],
1229 starIds['ra'][:],
1230 starIds['dec'][:],
1231 starIds['obs_arr_index'][:],
1232 starIds['n_obs'][:],
1233 obsX=starObs['x'][:],
1234 obsY=starObs['y'][:],
1235 obsDeltaMagBkg=starObs['delta_mag_bkg'][:],
1236 obsDeltaAper=starObs['delta_mag_aper'][:],
1237 refID=refId,
1238 refMag=refMag,
1239 refMagErr=refMagErr,
1240 flagID=flagId,
1241 flagFlag=flagFlag,
1242 computeNobs=True)
1243 else:
1244 # We determine the conversion from the native units (typically radians) to
1245 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1246 # numpy arrays rather than Angles, which would we approximately 600x slower.
1247 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1248
1249 fgcmStars.loadStars(fgcmPars,
1250 starObs['visit'][starIndices['obsIndex']],
1251 starObs['ccd'][starIndices['obsIndex']],
1252 starObs['ra'][starIndices['obsIndex']] * conv,
1253 starObs['dec'][starIndices['obsIndex']] * conv,
1254 starObs['instMag'][starIndices['obsIndex']],
1255 starObs['instMagErr'][starIndices['obsIndex']],
1256 fgcmExpInfo['FILTERNAME'][visitIndex],
1257 starIds['fgcm_id'][:],
1258 starIds['ra'][:],
1259 starIds['dec'][:],
1260 starIds['obsArrIndex'][:],
1261 starIds['nObs'][:],
1262 obsX=starObs['x'][starIndices['obsIndex']],
1263 obsY=starObs['y'][starIndices['obsIndex']],
1264 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1265 obsDeltaAper=starObs['deltaMagAper'][starIndices['obsIndex']],
1266 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1267 refID=refId,
1268 refMag=refMag,
1269 refMagErr=refMagErr,
1270 flagID=flagId,
1271 flagFlag=flagFlag,
1272 computeNobs=True)
1273
1274 # Release all references to temporary objects holding star data (see above)
1275 del starObs
1276 del starIds
1277 del starIndices
1278 del flagId
1279 del flagFlag
1280 del refStars
1281 del refId
1282 del refMag
1283 del refMagErr
1284
1285 # and set the bits in the cycle object
1286 fgcmFitCycle.setLUT(fgcmLut)
1287 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1288 fgcmFitCycle.setPars(fgcmPars)
1289
1290 # finish the setup
1291 fgcmFitCycle.finishSetup()
1292
1293 # and run
1294 fgcmFitCycle.run()
1295
1296
1299
1300 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1301
1302 # Output the config for the next cycle
1303 # We need to make a copy since the input one has been frozen
1304
1305 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1306 i, b in enumerate(_config.bands)}
1307 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1308 i, band in enumerate(_config.bands)}
1309
1310 outConfig = copy.copy(_config)
1311 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1312 precomputeSuperStarInitialCycle=False,
1313 freezeStdAtmosphere=False,
1314 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1315 expGrayHighCutDict=updatedHighCutDict)
1316
1317 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1318 cycleNumber=str(_config.cycleNumber + 1))
1319
1320 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1321 outConfig.cycleNumber)
1322 outConfig.save(configFileName)
1323
1324 if _config.isFinalCycle == 1:
1325 # We are done, ready to output products
1326 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1327 else:
1328 self.log.info("Saved config for next cycle to %s" % (configFileName))
1329 self.log.info("Be sure to look at:")
1330 self.log.info(" config.expGrayPhotometricCut")
1331 self.log.info(" config.expGrayHighCut")
1332 self.log.info("If you are satisfied with the fit, please set:")
1333 self.log.info(" config.isFinalCycle = True")
1334
1335 fgcmFitCycle.freeSharedMemory()
1336
1337 return fgcmDatasetDict, outConfig
1338
1339 def _loadParameters(self, parCat):
1340 """
1341 Load FGCM parameters from a previous fit cycle
1342
1343 Parameters
1344 ----------
1345 parCat : `lsst.afw.table.BaseCatalog`
1346 Parameter catalog in afw table form.
1347
1348 Returns
1349 -------
1350 inParInfo: `numpy.ndarray`
1351 Numpy array parameter information formatted for input to fgcm
1352 inParameters: `numpy.ndarray`
1353 Numpy array parameter values formatted for input to fgcm
1354 inSuperStar: `numpy.array`
1355 Superstar flat formatted for input to fgcm
1356 """
1357 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1358 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1359
1360 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1361 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1362 (parLutFilterNames.size, )),
1363 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1364 ('LNTAUUNIT', 'f8'),
1365 ('LNTAUSLOPEUNIT', 'f8'),
1366 ('ALPHAUNIT', 'f8'),
1367 ('LNPWVUNIT', 'f8'),
1368 ('LNPWVSLOPEUNIT', 'f8'),
1369 ('LNPWVQUADRATICUNIT', 'f8'),
1370 ('LNPWVGLOBALUNIT', 'f8'),
1371 ('O3UNIT', 'f8'),
1372 ('QESYSUNIT', 'f8'),
1373 ('FILTEROFFSETUNIT', 'f8'),
1374 ('HASEXTERNALPWV', 'i2'),
1375 ('HASEXTERNALTAU', 'i2')])
1376 inParInfo['NCCD'] = parCat['nCcd']
1377 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1378 inParInfo['FITBANDS'][:] = parFitBands
1379 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1380 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1381
1382 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1383 ('PARO3', 'f8', (parCat['parO3'].size, )),
1384 ('PARLNTAUINTERCEPT', 'f8',
1385 (parCat['parLnTauIntercept'].size, )),
1386 ('PARLNTAUSLOPE', 'f8',
1387 (parCat['parLnTauSlope'].size, )),
1388 ('PARLNPWVINTERCEPT', 'f8',
1389 (parCat['parLnPwvIntercept'].size, )),
1390 ('PARLNPWVSLOPE', 'f8',
1391 (parCat['parLnPwvSlope'].size, )),
1392 ('PARLNPWVQUADRATIC', 'f8',
1393 (parCat['parLnPwvQuadratic'].size, )),
1394 ('PARQESYSINTERCEPT', 'f8',
1395 (parCat['parQeSysIntercept'].size, )),
1396 ('COMPQESYSSLOPE', 'f8',
1397 (parCat['compQeSysSlope'].size, )),
1398 ('PARFILTEROFFSET', 'f8',
1399 (parCat['parFilterOffset'].size, )),
1400 ('PARFILTEROFFSETFITFLAG', 'i2',
1401 (parCat['parFilterOffsetFitFlag'].size, )),
1402 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1403 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1404 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1405 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1406 ('COMPABSTHROUGHPUT', 'f8',
1407 (parCat['compAbsThroughput'].size, )),
1408 ('COMPREFOFFSET', 'f8',
1409 (parCat['compRefOffset'].size, )),
1410 ('COMPREFSIGMA', 'f8',
1411 (parCat['compRefSigma'].size, )),
1412 ('COMPMIRRORCHROMATICITY', 'f8',
1413 (parCat['compMirrorChromaticity'].size, )),
1414 ('MIRRORCHROMATICITYPIVOT', 'f8',
1415 (parCat['mirrorChromaticityPivot'].size, )),
1416 ('COMPCCDCHROMATICITY', 'f8',
1417 (parCat['compCcdChromaticity'].size, )),
1418 ('COMPMEDIANSEDSLOPE', 'f8',
1419 (parCat['compMedianSedSlope'].size, )),
1420 ('COMPAPERCORRPIVOT', 'f8',
1421 (parCat['compAperCorrPivot'].size, )),
1422 ('COMPAPERCORRSLOPE', 'f8',
1423 (parCat['compAperCorrSlope'].size, )),
1424 ('COMPAPERCORRSLOPEERR', 'f8',
1425 (parCat['compAperCorrSlopeErr'].size, )),
1426 ('COMPAPERCORRRANGE', 'f8',
1427 (parCat['compAperCorrRange'].size, )),
1428 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1429 (parCat['compModelErrExptimePivot'].size, )),
1430 ('COMPMODELERRFWHMPIVOT', 'f8',
1431 (parCat['compModelErrFwhmPivot'].size, )),
1432 ('COMPMODELERRSKYPIVOT', 'f8',
1433 (parCat['compModelErrSkyPivot'].size, )),
1434 ('COMPMODELERRPARS', 'f8',
1435 (parCat['compModelErrPars'].size, )),
1436 ('COMPEXPGRAY', 'f8',
1437 (parCat['compExpGray'].size, )),
1438 ('COMPVARGRAY', 'f8',
1439 (parCat['compVarGray'].size, )),
1440 ('COMPEXPDELTAMAGBKG', 'f8',
1441 (parCat['compExpDeltaMagBkg'].size, )),
1442 ('COMPNGOODSTARPEREXP', 'i4',
1443 (parCat['compNGoodStarPerExp'].size, )),
1444 ('COMPEXPREFOFFSET', 'f8',
1445 (parCat['compExpRefOffset'].size, )),
1446 ('COMPSIGFGCM', 'f8',
1447 (parCat['compSigFgcm'].size, )),
1448 ('COMPSIGMACAL', 'f8',
1449 (parCat['compSigmaCal'].size, )),
1450 ('COMPRETRIEVEDLNPWV', 'f8',
1451 (parCat['compRetrievedLnPwv'].size, )),
1452 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1453 (parCat['compRetrievedLnPwvRaw'].size, )),
1454 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1455 (parCat['compRetrievedLnPwvFlag'].size, )),
1456 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1457 (parCat['compRetrievedTauNight'].size, )),
1458 ('COMPEPSILON', 'f8',
1459 (parCat['compEpsilon'].size, )),
1460 ('COMPMEDDELTAAPER', 'f8',
1461 (parCat['compMedDeltaAper'].size, )),
1462 ('COMPGLOBALEPSILON', 'f4',
1463 (parCat['compGlobalEpsilon'].size, )),
1464 ('COMPEPSILONMAP', 'f4',
1465 (parCat['compEpsilonMap'].size, )),
1466 ('COMPEPSILONNSTARMAP', 'i4',
1467 (parCat['compEpsilonNStarMap'].size, )),
1468 ('COMPEPSILONCCDMAP', 'f4',
1469 (parCat['compEpsilonCcdMap'].size, )),
1470 ('COMPEPSILONCCDNSTARMAP', 'i4',
1471 (parCat['compEpsilonCcdNStarMap'].size, ))])
1472
1473 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1474 inParams['PARO3'][:] = parCat['parO3'][0, :]
1475 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1476 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1477 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1478 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1479 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1480 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1481 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1482 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1483 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1484 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1485 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1486 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1487 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1488 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1489 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1490 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1491 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1492 inParams['COMPCCDCHROMATICITY'][:] = parCat['compCcdChromaticity'][0, :]
1493 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1494 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1495 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1496 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1497 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1498 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1499 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1500 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1501 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1502 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1503 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1504 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1505 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1506 inParams['COMPEXPREFOFFSET'][:] = parCat['compExpRefOffset'][0, :]
1507 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1508 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1509 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1510 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1511 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1512 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1513 inParams['COMPEPSILON'][:] = parCat['compEpsilon'][0, :]
1514 inParams['COMPMEDDELTAAPER'][:] = parCat['compMedDeltaAper'][0, :]
1515 inParams['COMPGLOBALEPSILON'][:] = parCat['compGlobalEpsilon'][0, :]
1516 inParams['COMPEPSILONMAP'][:] = parCat['compEpsilonMap'][0, :]
1517 inParams['COMPEPSILONNSTARMAP'][:] = parCat['compEpsilonNStarMap'][0, :]
1518 inParams['COMPEPSILONCCDMAP'][:] = parCat['compEpsilonCcdMap'][0, :]
1519 inParams['COMPEPSILONCCDNSTARMAP'][:] = parCat['compEpsilonCcdNStarMap'][0, :]
1520
1521 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1522 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1523
1524 return (inParInfo, inParams, inSuperStar)
1525
1526 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1527 """
1528 Persist FGCM datasets through the butler.
1529
1530 Parameters
1531 ----------
1532 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1533 Fgcm Fit cycle object
1534 """
1535 fgcmDatasetDict = {}
1536
1537 # Save the parameters
1538 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1539
1540 parSchema = afwTable.Schema()
1541
1542 comma = ','
1543 lutFilterNameString = comma.join([n.decode('utf-8')
1544 for n in parInfo['LUTFILTERNAMES'][0]])
1545 fitBandString = comma.join([n.decode('utf-8')
1546 for n in parInfo['FITBANDS'][0]])
1547
1548 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1549 lutFilterNameString, fitBandString)
1550 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1551 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1552 lutFilterNameString, fitBandString)
1553
1554 fgcmDatasetDict['fgcmFitParameters'] = parCat
1555
1556 # Save the indices of the flagged stars
1557 # (stars that have been (a) reserved from the fit for testing and
1558 # (b) bad stars that have failed quality checks.)
1559 flagStarSchema = self._makeFlagStarSchema()
1560 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1561 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1562
1563 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1564
1565 # Save the zeropoint information and atmospheres only if desired
1566 if self.outputZeropoints:
1567 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1568 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1569
1570 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1571 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1572
1573 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1574
1575 # Save atmosphere values
1576 # These are generated by the same code that generates zeropoints
1577 atmSchema = makeAtmSchema()
1578 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1579
1580 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1581
1582 # Save the standard stars (if configured)
1583 if self.outputStandards:
1584 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1585 stdSchema = makeStdSchema(len(goodBands))
1586 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1587
1588 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1589
1590 return fgcmDatasetDict
1591
1592 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1593 lutFilterNameString, fitBandString):
1594 """
1595 Make the parameter persistence schema
1596
1597 Parameters
1598 ----------
1599 parInfo: `numpy.ndarray`
1600 Parameter information returned by fgcm
1601 pars: `numpy.ndarray`
1602 Parameter values returned by fgcm
1603 parSuperStarFlat: `numpy.array`
1604 Superstar flat values returned by fgcm
1605 lutFilterNameString: `str`
1606 Combined string of all the lutFilterNames
1607 fitBandString: `str`
1608 Combined string of all the fitBands
1609
1610 Returns
1611 -------
1612 parSchema: `afwTable.schema`
1613 """
1614
1615 parSchema = afwTable.Schema()
1616
1617 # parameter info section
1618 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1619 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1620 size=len(lutFilterNameString))
1621 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1622 size=len(fitBandString))
1623 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1624 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1625 doc='Step units for ln(AOD) slope')
1626 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1627 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1628 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1629 doc='Step units for ln(pwv) slope')
1630 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1631 doc='Step units for ln(pwv) quadratic term')
1632 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1633 doc='Step units for global ln(pwv) parameters')
1634 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1635 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1636 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1637 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1638 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1639
1640 # parameter section
1641 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1642 size=pars['PARALPHA'].size)
1643 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1644 size=pars['PARO3'].size)
1645 parSchema.addField('parLnTauIntercept', type='ArrayD',
1646 doc='ln(Tau) intercept parameter vector',
1647 size=pars['PARLNTAUINTERCEPT'].size)
1648 parSchema.addField('parLnTauSlope', type='ArrayD',
1649 doc='ln(Tau) slope parameter vector',
1650 size=pars['PARLNTAUSLOPE'].size)
1651 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1652 size=pars['PARLNPWVINTERCEPT'].size)
1653 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1654 size=pars['PARLNPWVSLOPE'].size)
1655 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1656 size=pars['PARLNPWVQUADRATIC'].size)
1657 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1658 size=pars['PARQESYSINTERCEPT'].size)
1659 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1660 size=pars[0]['COMPQESYSSLOPE'].size)
1661 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1662 size=pars['PARFILTEROFFSET'].size)
1663 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1664 size=pars['PARFILTEROFFSETFITFLAG'].size)
1665 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1666 doc='Global scale for retrieved ln(pwv)')
1667 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1668 doc='Global offset for retrieved ln(pwv)')
1669 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1670 doc='Nightly offset for retrieved ln(pwv)',
1671 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1672 parSchema.addField('compAbsThroughput', type='ArrayD',
1673 doc='Absolute throughput (relative to transmission curves)',
1674 size=pars['COMPABSTHROUGHPUT'].size)
1675 parSchema.addField('compRefOffset', type='ArrayD',
1676 doc='Offset between reference stars and calibrated stars',
1677 size=pars['COMPREFOFFSET'].size)
1678 parSchema.addField('compRefSigma', type='ArrayD',
1679 doc='Width of reference star/calibrated star distribution',
1680 size=pars['COMPREFSIGMA'].size)
1681 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1682 doc='Computed mirror chromaticity terms',
1683 size=pars['COMPMIRRORCHROMATICITY'].size)
1684 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1685 doc='Mirror chromaticity pivot mjd',
1686 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1687 parSchema.addField('compCcdChromaticity', type='ArrayD',
1688 doc='Computed CCD chromaticity terms',
1689 size=pars['COMPCCDCHROMATICITY'].size)
1690 parSchema.addField('compMedianSedSlope', type='ArrayD',
1691 doc='Computed median SED slope (per band)',
1692 size=pars['COMPMEDIANSEDSLOPE'].size)
1693 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1694 size=pars['COMPAPERCORRPIVOT'].size)
1695 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1696 size=pars['COMPAPERCORRSLOPE'].size)
1697 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1698 size=pars['COMPAPERCORRSLOPEERR'].size)
1699 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1700 size=pars['COMPAPERCORRRANGE'].size)
1701 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1702 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1703 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1704 size=pars['COMPMODELERRFWHMPIVOT'].size)
1705 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1706 size=pars['COMPMODELERRSKYPIVOT'].size)
1707 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1708 size=pars['COMPMODELERRPARS'].size)
1709 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1710 size=pars['COMPEXPGRAY'].size)
1711 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1712 size=pars['COMPVARGRAY'].size)
1713 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1714 doc='Computed exposure offset due to background',
1715 size=pars['COMPEXPDELTAMAGBKG'].size)
1716 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1717 doc='Computed number of good stars per exposure',
1718 size=pars['COMPNGOODSTARPEREXP'].size)
1719 parSchema.addField('compExpRefOffset', type='ArrayD',
1720 doc='Computed per-visit median offset between standard stars and ref stars.',
1721 size=pars['COMPEXPREFOFFSET'].size)
1722 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1723 size=pars['COMPSIGFGCM'].size)
1724 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1725 size=pars['COMPSIGMACAL'].size)
1726 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1727 size=pars['COMPRETRIEVEDLNPWV'].size)
1728 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1729 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1730 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1731 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1732 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1733 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1734 parSchema.addField('compEpsilon', type='ArrayD',
1735 doc='Computed epsilon background offset per visit (nJy/arcsec2)',
1736 size=pars['COMPEPSILON'].size)
1737 parSchema.addField('compMedDeltaAper', type='ArrayD',
1738 doc='Median delta mag aper per visit',
1739 size=pars['COMPMEDDELTAAPER'].size)
1740 parSchema.addField('compGlobalEpsilon', type='ArrayD',
1741 doc='Computed epsilon bkg offset (global) (nJy/arcsec2)',
1742 size=pars['COMPGLOBALEPSILON'].size)
1743 parSchema.addField('compEpsilonMap', type='ArrayD',
1744 doc='Computed epsilon maps (nJy/arcsec2)',
1745 size=pars['COMPEPSILONMAP'].size)
1746 parSchema.addField('compEpsilonNStarMap', type='ArrayI',
1747 doc='Number of stars per pixel in computed epsilon maps',
1748 size=pars['COMPEPSILONNSTARMAP'].size)
1749 parSchema.addField('compEpsilonCcdMap', type='ArrayD',
1750 doc='Computed epsilon ccd maps (nJy/arcsec2)',
1751 size=pars['COMPEPSILONCCDMAP'].size)
1752 parSchema.addField('compEpsilonCcdNStarMap', type='ArrayI',
1753 doc='Number of stars per ccd bin in epsilon ccd maps',
1754 size=pars['COMPEPSILONCCDNSTARMAP'].size)
1755 # superstarflat section
1756 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1757 size=4)
1758 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1759 size=parSuperStarFlat.size)
1760
1761 return parSchema
1762
1763 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1764 lutFilterNameString, fitBandString):
1765 """
1766 Make the FGCM parameter catalog for persistence
1767
1768 Parameters
1769 ----------
1770 parSchema: `lsst.afw.table.Schema`
1771 Parameter catalog schema
1772 pars: `numpy.ndarray`
1773 FGCM parameters to put into parCat
1774 parSuperStarFlat: `numpy.array`
1775 FGCM superstar flat array to put into parCat
1776 lutFilterNameString: `str`
1777 Combined string of all the lutFilterNames
1778 fitBandString: `str`
1779 Combined string of all the fitBands
1780
1781 Returns
1782 -------
1783 parCat: `afwTable.BasicCatalog`
1784 Atmosphere and instrumental model parameter catalog for persistence
1785 """
1786
1787 parCat = afwTable.BaseCatalog(parSchema)
1788 parCat.reserve(1)
1789
1790 # The parameter catalog just has one row, with many columns for all the
1791 # atmosphere and instrument fit parameters
1792 rec = parCat.addNew()
1793
1794 # info section
1795 rec['nCcd'] = parInfo['NCCD'][0]
1796 rec['lutFilterNames'] = lutFilterNameString
1797 rec['fitBands'] = fitBandString
1798 # note these are not currently supported here.
1799 rec['hasExternalPwv'] = 0
1800 rec['hasExternalTau'] = 0
1801
1802 # parameter section
1803
1804 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1805
1806 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1807 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1808 'parQeSysIntercept', 'compQeSysSlope',
1809 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1810 'parFilterOffset', 'parFilterOffsetFitFlag',
1811 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1812 'compMirrorChromaticity', 'mirrorChromaticityPivot', 'compCcdChromaticity',
1813 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1814 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1815 'compModelErrSkyPivot', 'compModelErrPars',
1816 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1817 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1818 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1819 'compRetrievedTauNight', 'compEpsilon', 'compMedDeltaAper',
1820 'compGlobalEpsilon', 'compEpsilonMap', 'compEpsilonNStarMap',
1821 'compEpsilonCcdMap', 'compEpsilonCcdNStarMap', 'compExpRefOffset']
1822
1823 for scalarName in scalarNames:
1824 rec[scalarName] = pars[scalarName.upper()][0]
1825
1826 for arrName in arrNames:
1827 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1828
1829 # superstar section
1830 rec['superstarSize'][:] = parSuperStarFlat.shape
1831 rec['superstar'][:] = parSuperStarFlat.ravel()
1832
1833 return parCat
1834
1835 def _makeFlagStarSchema(self):
1836 """
1837 Make the flagged-stars schema
1838
1839 Returns
1840 -------
1841 flagStarSchema: `lsst.afw.table.Schema`
1842 """
1843
1844 flagStarSchema = afwTable.Schema()
1845
1846 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1847 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1848
1849 return flagStarSchema
1850
1851 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1852 """
1853 Make the flagged star catalog for persistence
1854
1855 Parameters
1856 ----------
1857 flagStarSchema: `lsst.afw.table.Schema`
1858 Flagged star schema
1859 flagStarStruct: `numpy.ndarray`
1860 Flagged star structure from fgcm
1861
1862 Returns
1863 -------
1864 flagStarCat: `lsst.afw.table.BaseCatalog`
1865 Flagged star catalog for persistence
1866 """
1867
1868 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1869 flagStarCat.resize(flagStarStruct.size)
1870
1871 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1872 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1873
1874 return flagStarCat