28import astropy.units
as u
33import lsst.pipe.base
as pipeBase
39from lsst.obs.base
import Instrument
40from lsst.pipe.tasks.colorterms
import ColortermLibrary
41from lsst.verify
import Job, Measurement
44 ReferenceObjectLoader)
46from lsst.utils.timer
import timeMethod
48from .dataIds
import PerTractCcdDataIdContainer
53__all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
55Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
56Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
61 meas = Measurement(job.metrics[name], value)
62 job.measurements.insert(meas)
66 """Subclass of TaskRunner for jointcalTask (gen2)
68 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
69 extracted
from the command line (whereas most CmdLineTasks
' runDataRef methods take
70 single dataRef, are are called repeatedly). This class transforms the processed
71 arguments generated by the ArgumentParser into the arguments expected by
72 Jointcal.runDataRef().
74 See pipeBase.TaskRunner
for more information.
80 Return a list of tuples per tract, each containing (dataRefs, kwargs).
82 Jointcal operates on lists of dataRefs simultaneously.
84 kwargs['butler'] = parsedCmd.butler
88 for ref
in parsedCmd.id.refList:
89 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
91 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
99 Arguments for Task.runDataRef()
104 if self.doReturnResults
is False:
106 - ``exitStatus``: 0
if the task completed successfully, 1 otherwise.
108 if self.doReturnResults
is True:
110 - ``result``: the result of calling jointcal.runDataRef()
111 - ``exitStatus``: 0
if the task completed successfully, 1 otherwise.
116 dataRefList, kwargs = args
117 butler = kwargs.pop(
'butler')
118 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
121 result = task.runDataRef(dataRefList, **kwargs)
122 exitStatus = result.exitStatus
123 job_path = butler.get(
'verify_job_filename')
124 result.job.write(job_path[0])
125 except Exception
as e:
130 eName = type(e).__name__
131 tract = dataRefList[0].dataId[
'tract']
132 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
135 kwargs[
'butler'] = butler
136 if self.doReturnResults:
137 return pipeBase.Struct(result=result, exitStatus=exitStatus)
139 return pipeBase.Struct(exitStatus=exitStatus)
143 """Lookup function that asserts/hopes that a static calibration dataset
144 exists in a particular collection, since this task can
't provide a single
145 date/time to use to search for one properly.
147 This
is mostly useful
for the ``camera`` dataset,
in cases where the task
's
148 quantum dimensions do *not* include something temporal, like ``exposure``
153 datasetType : `lsst.daf.butler.DatasetType`
154 Type of dataset being searched
for.
155 registry : `lsst.daf.butler.Registry`
156 Data repository registry to search.
157 quantumDataId : `lsst.daf.butler.DataCoordinate`
158 Data ID of the quantum this camera should match.
159 collections : `Iterable` [ `str` ]
160 Collections that should be searched - but this lookup function works
161 by ignoring this
in favor of a more-
or-less hard-coded value.
165 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
166 Iterator over dataset references; should have only one element.
170 This implementation duplicates one
in fgcmcal,
and is at least quite
171 similar to another
in cp_pipe. This duplicate has the most documentation.
172 Fixing this
is DM-29661.
174 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
175 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
176 return registry.queryDatasets(datasetType,
177 dataId=quantumDataId,
178 collections=[unboundedCollection],
183 """Lookup function that finds all refcats for all visits that overlap a
184 tract, rather than just the refcats that directly overlap the tract.
188 datasetType : `lsst.daf.butler.DatasetType`
189 Type of dataset being searched for.
190 registry : `lsst.daf.butler.Registry`
191 Data repository registry to search.
192 quantumDataId : `lsst.daf.butler.DataCoordinate`
193 Data ID of the quantum; expected to be something we can use
as a
194 constraint to query
for overlapping visits.
195 collections : `Iterable` [ `str` ]
196 Collections to search.
200 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
201 Iterator over refcat references.
210 for visit_data_id
in set(registry.queryDataIds(
"visit", dataId=quantumDataId).expanded()):
212 registry.queryDatasets(
214 collections=collections,
215 dataId=visit_data_id,
223 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter")):
224 """Middleware input/output connections for jointcal data."""
225 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
226 doc=
"The camera instrument that took these observations.",
228 storageClass=
"Camera",
229 dimensions=(
"instrument",),
231 lookupFunction=lookupStaticCalibrations,
233 inputSourceTableVisit = pipeBase.connectionTypes.Input(
234 doc=
"Source table in parquet format, per visit",
235 name=
"sourceTable_visit",
236 storageClass=
"DataFrame",
237 dimensions=(
"instrument",
"visit"),
241 inputVisitSummary = pipeBase.connectionTypes.Input(
242 doc=(
"Per-visit consolidated exposure metadata built from calexps. "
243 "These catalogs use detector id for the id and must be sorted for "
244 "fast lookups of a detector."),
246 storageClass=
"ExposureCatalog",
247 dimensions=(
"instrument",
"visit"),
251 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
252 doc=
"The astrometry reference catalog to match to loaded input catalog sources.",
253 name=
"gaia_dr2_20200414",
254 storageClass=
"SimpleCatalog",
255 dimensions=(
"skypix",),
258 lookupFunction=lookupVisitRefCats,
260 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
261 doc=
"The photometry reference catalog to match to loaded input catalog sources.",
262 name=
"ps1_pv3_3pi_20170110",
263 storageClass=
"SimpleCatalog",
264 dimensions=(
"skypix",),
267 lookupFunction=lookupVisitRefCats,
270 outputWcs = pipeBase.connectionTypes.Output(
271 doc=(
"Per-tract, per-visit world coordinate systems derived from the fitted model."
272 " These catalogs only contain entries for detectors with an output, and use"
273 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
274 name=
"jointcalSkyWcsCatalog",
275 storageClass=
"ExposureCatalog",
276 dimensions=(
"instrument",
"visit",
"skymap",
"tract"),
279 outputPhotoCalib = pipeBase.connectionTypes.Output(
280 doc=(
"Per-tract, per-visit photometric calibrations derived from the fitted model."
281 " These catalogs only contain entries for detectors with an output, and use"
282 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
283 name=
"jointcalPhotoCalibCatalog",
284 storageClass=
"ExposureCatalog",
285 dimensions=(
"instrument",
"visit",
"skymap",
"tract"),
293 for name
in (
"astrometry",
"photometry"):
294 vars()[f
"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output(
295 doc=f
"The number of cross-matched fittedStars for {name}",
296 name=f
"metricvalue_jointcal_{name}_matched_fittedStars",
297 storageClass=
"MetricValue",
298 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
300 vars()[f
"{name}_collected_refStars"] = pipeBase.connectionTypes.Output(
301 doc=f
"The number of {name} reference stars drawn from the reference catalog, before matching.",
302 name=f
"metricvalue_jointcal_{name}_collected_refStars",
303 storageClass=
"MetricValue",
304 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
306 vars()[f
"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output(
307 doc=f
"The number of {name} reference stars matched to fittedStars.",
308 name=f
"metricvalue_jointcal_{name}_prepared_refStars",
309 storageClass=
"MetricValue",
310 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
312 vars()[f
"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output(
313 doc=f
"The number of cross-matched fittedStars after cleanup, for {name}.",
314 name=f
"metricvalue_jointcal_{name}_prepared_fittedStars",
315 storageClass=
"MetricValue",
316 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
318 vars()[f
"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output(
319 doc=f
"The number of ccdImages that will be fit for {name}, after cleanup.",
320 name=f
"metricvalue_jointcal_{name}_prepared_ccdImages",
321 storageClass=
"MetricValue",
322 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
324 vars()[f
"{name}_final_chi2"] = pipeBase.connectionTypes.Output(
325 doc=f
"The final chi2 of the {name} fit.",
326 name=f
"metricvalue_jointcal_{name}_final_chi2",
327 storageClass=
"MetricValue",
328 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
330 vars()[f
"{name}_final_ndof"] = pipeBase.connectionTypes.Output(
331 doc=f
"The number of degrees of freedom of the fitted {name}.",
332 name=f
"metricvalue_jointcal_{name}_final_ndof",
333 storageClass=
"MetricValue",
334 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
345 if not config.doAstrometry:
346 self.prerequisiteInputs.remove(
"astrometryRefCat")
347 self.outputs.remove(
"outputWcs")
348 for key
in list(self.outputs):
349 if "metricvalue_jointcal_astrometry" in key:
350 self.outputs.remove(key)
351 if not config.doPhotometry:
352 self.prerequisiteInputs.remove(
"photometryRefCat")
353 self.outputs.remove(
"outputPhotoCalib")
354 for key
in list(self.outputs):
355 if "metricvalue_jointcal_photometry" in key:
356 self.outputs.remove(key)
360 pipelineConnections=JointcalTaskConnections):
361 """Configuration for JointcalTask"""
363 doAstrometry = pexConfig.Field(
364 doc=
"Fit astrometry and write the fitted result.",
368 doPhotometry = pexConfig.Field(
369 doc=
"Fit photometry and write the fitted result.",
373 coaddName = pexConfig.Field(
374 doc=
"Type of coadd, typically deep or goodSeeing",
375 deprecated=
"Only applies to gen2; will be removed when gen2 support is removed (DM-20572).",
380 sourceFluxType = pexConfig.Field(
382 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
385 positionErrorPedestal = pexConfig.Field(
386 doc=
"Systematic term to apply to the measured position error (pixels)",
390 photometryErrorPedestal = pexConfig.Field(
391 doc=
"Systematic term to apply to the measured error on flux or magnitude as a "
392 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
397 matchCut = pexConfig.Field(
398 doc=
"Matching radius between fitted and reference stars (arcseconds)",
402 minMeasurements = pexConfig.Field(
403 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
407 minMeasuredStarsPerCcd = pexConfig.Field(
408 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
412 minRefStarsPerCcd = pexConfig.Field(
413 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
417 allowLineSearch = pexConfig.Field(
418 doc=
"Allow a line search during minimization, if it is reasonable for the model"
419 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
423 astrometrySimpleOrder = pexConfig.Field(
424 doc=
"Polynomial order for fitting the simple astrometry model.",
428 astrometryChipOrder = pexConfig.Field(
429 doc=
"Order of the per-chip transform for the constrained astrometry model.",
433 astrometryVisitOrder = pexConfig.Field(
434 doc=
"Order of the per-visit transform for the constrained astrometry model.",
438 useInputWcs = pexConfig.Field(
439 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
443 astrometryModel = pexConfig.ChoiceField(
444 doc=
"Type of model to fit to astrometry",
446 default=
"constrained",
447 allowed={
"simple":
"One polynomial per ccd",
448 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
450 photometryModel = pexConfig.ChoiceField(
451 doc=
"Type of model to fit to photometry",
453 default=
"constrainedMagnitude",
454 allowed={
"simpleFlux":
"One constant zeropoint per ccd and visit, fitting in flux space.",
455 "constrainedFlux":
"Constrained zeropoint per ccd, and one polynomial per visit,"
456 " fitting in flux space.",
457 "simpleMagnitude":
"One constant zeropoint per ccd and visit,"
458 " fitting in magnitude space.",
459 "constrainedMagnitude":
"Constrained zeropoint per ccd, and one polynomial per visit,"
460 " fitting in magnitude space.",
463 applyColorTerms = pexConfig.Field(
464 doc=
"Apply photometric color terms to reference stars?"
465 "Requires that colorterms be set to a ColortermLibrary",
469 colorterms = pexConfig.ConfigField(
470 doc=
"Library of photometric reference catalog name to color term dict.",
471 dtype=ColortermLibrary,
473 photometryVisitOrder = pexConfig.Field(
474 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
478 photometryDoRankUpdate = pexConfig.Field(
479 doc=(
"Do the rank update step during minimization. "
480 "Skipping this can help deal with models that are too non-linear."),
484 astrometryDoRankUpdate = pexConfig.Field(
485 doc=(
"Do the rank update step during minimization (should not change the astrometry fit). "
486 "Skipping this can help deal with models that are too non-linear."),
490 outlierRejectSigma = pexConfig.Field(
491 doc=
"How many sigma to reject outliers at during minimization.",
495 astrometryOutlierRelativeTolerance = pexConfig.Field(
496 doc=(
"Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
497 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
498 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
499 "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
500 "close to the solution found when removing all outliers."),
504 maxPhotometrySteps = pexConfig.Field(
505 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
509 maxAstrometrySteps = pexConfig.Field(
510 doc=
"Maximum number of minimize iterations to take when fitting astrometry.",
514 astrometryRefObjLoader = pexConfig.ConfigurableField(
515 target=LoadIndexedReferenceObjectsTask,
516 doc=
"Reference object loader for astrometric fit",
518 photometryRefObjLoader = pexConfig.ConfigurableField(
519 target=LoadIndexedReferenceObjectsTask,
520 doc=
"Reference object loader for photometric fit",
522 sourceSelector = sourceSelectorRegistry.makeField(
523 doc=
"How to select sources for cross-matching",
526 astrometryReferenceSelector = pexConfig.ConfigurableField(
527 target=ReferenceSourceSelectorTask,
528 doc=
"How to down-select the loaded astrometry reference catalog.",
530 photometryReferenceSelector = pexConfig.ConfigurableField(
531 target=ReferenceSourceSelectorTask,
532 doc=
"How to down-select the loaded photometry reference catalog.",
534 astrometryReferenceErr = pexConfig.Field(
535 doc=(
"Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
536 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
537 "If specified, overrides any existing `coord_*Err` values."),
544 writeInitMatrix = pexConfig.Field(
546 doc=(
"Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
547 "Output files will be written to `config.debugOutputPath` and will "
548 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
549 "Note that these files are the dense versions of the matrix, and so may be very large."),
552 writeChi2FilesInitialFinal = pexConfig.Field(
554 doc=(
"Write .csv files containing the contributions to chi2 for the initialization and final fit. "
555 "Output files will be written to `config.debugOutputPath` and will "
556 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
559 writeChi2FilesOuterLoop = pexConfig.Field(
561 doc=(
"Write .csv files containing the contributions to chi2 for the outer fit loop. "
562 "Output files will be written to `config.debugOutputPath` and will "
563 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
566 writeInitialModel = pexConfig.Field(
568 doc=(
"Write the pre-initialization model to text files, for debugging. "
569 "Output files will be written to `config.debugOutputPath` and will be "
570 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
573 debugOutputPath = pexConfig.Field(
576 doc=(
"Path to write debug output files to. Used by "
577 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
579 detailedProfile = pexConfig.Field(
582 doc=
"Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
588 msg =
"applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
589 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
591 msg = (
"Only doing astrometry, but Colorterms are not applied for astrometry;"
592 "applyColorTerms=True will be ignored.")
593 lsst.log.warning(msg)
601 self.
sourceSelectorsourceSelector[
'science'].doSignalToNoise =
True
604 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.minimum = 10.
606 fluxField = f
"slot_{self.sourceFluxType}Flux_instFlux"
607 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.fluxField = fluxField
608 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.errField = fluxField +
"Err"
614 badFlags = [
'base_PixelFlags_flag_edge',
'base_PixelFlags_flag_saturated',
615 'base_PixelFlags_flag_interpolatedCenter',
'base_SdssCentroid_flag',
616 'base_PsfFlux_flag',
'base_PixelFlags_flag_suspectCenter']
628 """Write model to outfile."""
629 with open(filename,
"w")
as file:
630 file.write(repr(model))
631 log.info(
"Wrote %s to file: %s", model, filename)
634@dataclasses.dataclass
636 """The input data jointcal needs for each detector/visit."""
638 """The visit identifier of this exposure."""
640 """The catalog derived from this exposure."""
642 """The VisitInfo of this exposure."""
644 """The detector of this exposure."""
646 """The photometric calibration of this exposure."""
648 """The WCS of this exposure."""
650 """The bounding box of this exposure."""
652 """The filter of this exposure."""
656 """Astrometricly and photometricly calibrate across multiple visits of the
661 butler : `lsst.daf.persistence.Butler`
662 The butler is passed to the refObjLoader constructor
in case it
is
663 needed. Ignored
if the refObjLoader argument provides a loader directly.
664 Used to initialize the astrometry
and photometry refObjLoaders.
665 initInputs : `dict`, optional
666 Dictionary used to initialize PipelineTasks (empty
for jointcal).
669 ConfigClass = JointcalConfig
670 RunnerClass = JointcalRunner
671 _DefaultName = "jointcal"
673 def __init__(self, butler=None, initInputs=None, **kwargs):
675 self.makeSubtask(
"sourceSelector")
676 if self.config.doAstrometry:
677 if initInputs
is None:
679 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
680 self.makeSubtask(
"astrometryReferenceSelector")
683 if self.config.doPhotometry:
684 if initInputs
is None:
686 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
687 self.makeSubtask(
"photometryReferenceSelector")
692 self.
jobjob = Job.load_metrics_package(subset=
'jointcal')
697 inputs = butlerQC.get(inputRefs)
699 tract = butlerQC.quantum.dataId[
'tract']
700 if self.config.doAstrometry:
702 dataIds=[ref.datasetRef.dataId
703 for ref
in inputRefs.astrometryRefCat],
704 refCats=inputs.pop(
'astrometryRefCat'),
705 config=self.config.astrometryRefObjLoader,
707 if self.config.doPhotometry:
709 dataIds=[ref.datasetRef.dataId
710 for ref
in inputRefs.photometryRefCat],
711 refCats=inputs.pop(
'photometryRefCat'),
712 config=self.config.photometryRefObjLoader,
714 outputs = self.
runrun(**inputs, tract=tract)
715 self.
_put_metrics_put_metrics(butlerQC, outputs.job, outputRefs)
716 if self.config.doAstrometry:
717 self.
_put_output_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
718 inputs[
'inputCamera'],
"setWcs")
719 if self.config.doPhotometry:
720 self.
_put_output_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
721 inputs[
'inputCamera'],
"setPhotoCalib")
723 def _put_metrics(self, butlerQC, job, outputRefs):
724 """Persist all measured metrics stored in a job.
728 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
729 A butler which is specialized to operate
in the context of a
730 `lsst.daf.butler.Quantum`; This
is the input to `runQuantum`.
731 job : `lsst.verify.job.Job`
732 Measurements of metrics to persist.
733 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
734 The DatasetRefs to persist the data to.
736 for key
in job.measurements.keys():
737 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace(
'jointcal.',
'')))
739 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
740 """Persist the output datasets to their appropriate datarefs.
744 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
745 A butler which is specialized to operate
in the context of a
746 `lsst.daf.butler.Quantum`; This
is the input to `runQuantum`.
749 The fitted objects to persist.
750 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
751 The DatasetRefs to persist the data to.
753 The camera
for this instrument, to get detector ids
from.
755 The method to call on the ExposureCatalog to set each output.
758 schema.addField('visit', type=
'I', doc=
'Visit number')
760 def new_catalog(visit, size):
761 """Return an catalog ready to be filled with appropriate output."""
764 catalog[
'visit'] = visit
766 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
767 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
771 detectors_per_visit = collections.defaultdict(int)
774 detectors_per_visit[key[0]] += 1
776 for ref
in outputRefs:
777 visit = ref.dataId[
'visit']
778 catalog = new_catalog(visit, detectors_per_visit[visit])
781 for detector
in camera:
782 detectorId = detector.getId()
783 key = (ref.dataId[
'visit'], detectorId)
784 if key
not in outputs:
786 self.log.debug(
"No %s output for detector %s in visit %s",
787 setter[3:], detectorId, visit)
790 catalog[i].setId(detectorId)
791 getattr(catalog[i], setter)(outputs[key])
795 butlerQC.put(catalog, ref)
796 self.log.info(
"Wrote %s detectors to %s", i, ref)
798 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
804 sourceFluxField =
"flux"
808 oldWcsList, bands = self.
_load_data_load_data(inputSourceTableVisit,
814 boundingCircle, center, radius, defaultFilter, epoch = self.
_prep_sky_prep_sky(associations, bands)
816 if self.config.doAstrometry:
820 referenceSelector=self.astrometryReferenceSelector,
824 astrometry_output = self.
_make_output_make_output(associations.getCcdImageList(),
828 astrometry_output =
None
830 if self.config.doPhotometry:
834 referenceSelector=self.photometryReferenceSelector,
838 reject_bad_fluxes=
True)
839 photometry_output = self.
_make_output_make_output(associations.getCcdImageList(),
843 photometry_output =
None
845 return pipeBase.Struct(outputWcs=astrometry_output,
846 outputPhotoCalib=photometry_output,
851 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
852 jointcalControl, camera):
853 """Read the data that jointcal needs to run. (Gen3 version)
855 Modifies ``associations`` in-place
with the loaded data.
859 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
860 References to visit-level DataFrames to load the catalog data
from.
861 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
862 Visit-level exposure summary catalog
with metadata.
864 Object to add the loaded data to by constructing new CcdImages.
866 Control object
for C++ associations management.
868 Camera object
for detector geometry.
873 The original WCS of the input data, to aid
in writing tests.
874 bands : `list` [`str`]
875 The filter bands of each input dataset.
879 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile
else ''
880 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
883 catalogMap = {ref.dataId[
'visit']: i
for i, ref
in enumerate(inputSourceTableVisit)}
884 detectorDict = {detector.getId(): detector
for detector
in camera}
888 for visitSummaryRef
in inputVisitSummary:
889 visitSummary = visitSummaryRef.get()
891 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId[
'visit']]]
893 inColumns = dataRef.get(component=
'columns')
897 visitCatalog = dataRef.get(parameters={
'columns': columns})
899 selected = self.sourceSelector.
run(visitCatalog)
902 detectors = {id: index
for index, id
in enumerate(visitSummary[
'id'])}
903 for id, index
in detectors.items():
909 self.config.sourceFluxType,
913 data = self.
_make_one_input_data_make_one_input_data(visitSummary[index], catalog, detectorDict)
914 result = self.
_build_ccdImage_build_ccdImage(data, associations, jointcalControl)
915 if result
is not None:
916 oldWcsList.append(result.wcs)
918 filters.append(data.filter)
919 if len(filters) == 0:
920 raise RuntimeError(
"No data to process: did source selector remove all sources?")
921 filters = collections.Counter(filters)
923 return oldWcsList, filters
925 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
926 """Return a data structure for this detector+visit."""
929 visitInfo=visitRecord.getVisitInfo(),
930 detector=detectorDict[visitRecord.getId()],
931 photoCalib=visitRecord.getPhotoCalib(),
932 wcs=visitRecord.getWcs(),
933 bbox=visitRecord.getBBox(),
937 physical=visitRecord[
'physical_filter']))
942 def _getMetadataName(self):
946 def _makeArgumentParser(cls):
947 """Create an argument parser"""
948 parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
949 parser.add_id_argument("--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
950 ContainerClass=PerTractCcdDataIdContainer)
953 def _build_ccdImage(self, data, associations, jointcalControl):
955 Extract the necessary things from this catalog+metadata to add a new
960 data : `JointcalInputData`
961 The loaded input data.
963 Object to add the info to, to construct a new CcdImage
965 Control object
for associations management
971 The TAN WCS of this image, read
from the calexp
974 A key to identify this dataRef by its visit
and ccd ids
977 if there are no sources
in the loaded catalog.
979 if len(data.catalog) == 0:
980 self.log.warning(
"No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
983 associations.createCcdImage(data.catalog,
987 data.filter.physicalLabel,
991 data.detector.getId(),
994 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key'))
995 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
996 return Result(data.wcs, Key(data.visit, data.detector.getId()))
998 def _readDataId(self, butler, dataId):
999 """Read all of the data for one dataId from the butler. (gen2 version)"""
1001 if "visit" in dataId.keys():
1002 visit = dataId[
"visit"]
1004 visit = butler.getButler().queryMetadata(
"calexp", (
"visit"), butler.dataId)[0]
1005 detector = butler.get(
'calexp_detector', dataId=dataId)
1007 catalog = butler.get(
'src',
1008 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
1010 goodSrc = self.sourceSelector.
run(catalog)
1011 self.log.debug(
"%d sources selected in visit %d detector %d",
1012 len(goodSrc.sourceCat),
1016 catalog=goodSrc.sourceCat,
1017 visitInfo=butler.get(
'calexp_visitInfo', dataId=dataId),
1019 photoCalib=butler.get(
'calexp_photoCalib', dataId=dataId),
1020 wcs=butler.get(
'calexp_wcs', dataId=dataId),
1021 bbox=butler.get(
'calexp_bbox', dataId=dataId),
1022 filter=butler.get(
'calexp_filterLabel', dataId=dataId))
1024 def loadData(self, dataRefs, associations, jointcalControl):
1025 """Read the data that jointcal needs to run. (Gen2 version)"""
1026 visit_ccd_to_dataRef = {}
1029 load_cat_prof_file =
'jointcal_loadData.prof' if self.config.detailedProfile
else ''
1030 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1032 camera = dataRefs[0].get(
'camera', immediate=
True)
1034 for dataRef
in dataRefs:
1035 data = self.
_readDataId_readDataId(dataRef.getButler(), dataRef.dataId)
1036 result = self.
_build_ccdImage_build_ccdImage(data, associations, jointcalControl)
1039 oldWcsList.append(result.wcs)
1040 visit_ccd_to_dataRef[result.key] = dataRef
1041 filters.append(data.filter)
1042 if len(filters) == 0:
1043 raise RuntimeError(
"No data to process: did source selector remove all sources?")
1044 filters = collections.Counter(filters)
1046 return oldWcsList, filters, visit_ccd_to_dataRef
1048 def _getDebugPath(self, filename):
1049 """Constructs a path to filename using the configured debug path.
1051 return os.path.join(self.config.debugOutputPath, filename)
1053 def _prep_sky(self, associations, filters):
1054 """Prepare on-sky and other data that must be computed after data has
1057 associations.computeCommonTangentPoint()
1059 boundingCircle = associations.computeBoundingCircle()
1061 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1063 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1066 defaultFilter = filters.most_common(1)[0][0]
1067 self.log.debug(
"Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1071 associations.setEpoch(epoch.jyear)
1073 return boundingCircle, center, radius, defaultFilter, epoch
1078 Jointly calibrate the astrometry and photometry across a set of images.
1080 NOTE: this
is for gen2 middleware only.
1084 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1085 List of data references to the exposures to be fit.
1089 result : `lsst.pipe.base.Struct`
1090 Struct of metadata
from the fit, containing:
1093 The provided data references that were fit (
with updated WCSs)
1095 The original WCS
from each dataRef
1097 Dictionary of internally-computed metrics
for testing/validation.
1099 if len(dataRefs) == 0:
1100 raise ValueError(
'Need a non-empty list of data references!')
1104 sourceFluxField =
"slot_%sFlux" % (self.config.sourceFluxType,)
1108 oldWcsList, filters, visit_ccd_to_dataRef = self.
loadDataloadData(dataRefs,
1112 boundingCircle, center, radius, defaultFilter, epoch = self.
_prep_sky_prep_sky(associations, filters)
1114 tract = dataRefs[0].dataId[
'tract']
1116 if self.config.doAstrometry:
1120 referenceSelector=self.astrometryReferenceSelector,
1128 if self.config.doPhotometry:
1132 referenceSelector=self.photometryReferenceSelector,
1136 reject_bad_fluxes=
True)
1141 return pipeBase.Struct(dataRefs=dataRefs,
1142 oldWcsList=oldWcsList,
1146 defaultFilter=defaultFilter,
1148 exitStatus=exitStatus)
1150 def _get_refcat_coordinate_error_override(self, refCat, name):
1151 """Check whether we should override the refcat coordinate errors, and
1152 return the overridden error
if necessary.
1157 The reference catalog to check
for a ``coord_raErr`` field.
1159 Whether we are doing
"astrometry" or "photometry".
1163 refCoordErr : `float`
1164 The refcat coordinate error to use,
or NaN
if we are
not overriding
1169 lsst.pex.config.FieldValidationError
1170 Raised
if the refcat does
not contain coordinate errors
and
1171 ``config.astrometryReferenceErr``
is not set.
1175 if name.lower() ==
"photometry":
1176 if 'coord_raErr' not in refCat.schema:
1181 if self.config.astrometryReferenceErr
is None and 'coord_raErr' not in refCat.schema:
1182 msg = (
"Reference catalog does not contain coordinate errors, "
1183 "and config.astrometryReferenceErr not supplied.")
1184 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1188 if self.config.astrometryReferenceErr
is not None and 'coord_raErr' in refCat.schema:
1189 self.log.warning(
"Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1190 self.config.astrometryReferenceErr)
1192 if self.config.astrometryReferenceErr
is None:
1195 return self.config.astrometryReferenceErr
1197 def _compute_proper_motion_epoch(self, ccdImageList):
1198 """Return the proper motion correction epoch of the provided images.
1203 The images to compute the appropriate epoch for.
1207 epoch : `astropy.time.Time`
1208 The date to use
for proper motion corrections.
1210 return astropy.time.Time(np.mean([ccdImage.getEpoch()
for ccdImage
in ccdImageList]),
1214 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1215 tract="", match_cut=3.0,
1216 reject_bad_fluxes=False, *,
1217 name="", refObjLoader=None, referenceSelector=None,
1218 fit_function=None, epoch=None):
1219 """Load reference catalog, perform the fit, and return the result.
1224 The star/reference star associations to fit.
1226 filter to load from reference catalog.
1228 ICRS center of field to load
from reference catalog.
1230 On-sky radius to load
from reference catalog.
1232 Name of thing being fit:
"astrometry" or "photometry".
1233 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1234 Reference object loader to use to load a reference catalog.
1235 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1236 Selector to use to pick objects
from the loaded reference catalog.
1237 fit_function : callable
1238 Function to call to perform fit (takes Associations object).
1239 tract : `str`, optional
1240 Name of tract currently being fit.
1241 match_cut : `float`, optional
1242 Radius
in arcseconds to find cross-catalog matches to during
1243 associations.associateCatalogs.
1244 reject_bad_fluxes : `bool`, optional
1245 Reject refCat sources
with NaN/inf flux
or NaN/0 fluxErr.
1246 epoch : `astropy.time.Time`, optional
1247 Epoch to which to correct refcat proper motion
and parallax,
1248 or `
None` to
not apply such corrections.
1252 result : `Photometry`
or `Astrometry`
1253 Result of `fit_function()`
1255 self.log.info("====== Now processing %s...", name)
1258 associations.associateCatalogs(match_cut)
1260 associations.fittedStarListSize())
1262 applyColorterms =
False if name.lower() ==
"astrometry" else self.config.applyColorTerms
1264 center, radius, defaultFilter,
1265 applyColorterms=applyColorterms,
1269 associations.collectRefStars(refCat,
1270 self.config.matchCut*lsst.geom.arcseconds,
1272 refCoordinateErr=refCoordErr,
1273 rejectBadFluxes=reject_bad_fluxes)
1275 associations.refStarListSize())
1277 associations.prepareFittedStars(self.config.minMeasurements)
1281 associations.nFittedStarsWithAssociatedRefStar())
1283 associations.fittedStarListSize())
1285 associations.nCcdImagesValidForFit())
1287 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if self.config.detailedProfile
else ''
1288 dataName =
"{}_{}".format(tract, defaultFilter.physicalLabel)
1289 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1290 result = fit_function(associations, dataName)
1293 if self.config.writeChi2FilesInitialFinal:
1294 baseName = self.
_getDebugPath_getDebugPath(f
"{name}_final_chi2-{dataName}")
1295 result.fit.saveChi2Contributions(baseName+
"{type}")
1296 self.log.info(
"Wrote chi2 contributions files: %s", baseName)
1300 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1301 applyColorterms=False, epoch=None):
1302 """Load the necessary reference catalog sources, convert fluxes to
1303 correct units, and apply color term corrections
if requested.
1307 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1308 The reference catalog loader to use to get the data.
1309 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1310 Source selector to apply to loaded reference catalog.
1312 The center around which to load sources.
1314 The radius around ``center`` to load sources
in.
1316 The camera filter to load fluxes
for.
1317 applyColorterms : `bool`
1318 Apply colorterm corrections to the refcat
for ``filterName``?
1319 epoch : `astropy.time.Time`, optional
1320 Epoch to which to correct refcat proper motion
and parallax,
1321 or `
None` to
not apply such corrections.
1326 The loaded reference catalog.
1328 The name of the reference catalog flux field appropriate
for ``filterName``.
1330 skyCircle = refObjLoader.loadSkyCircle(center,
1332 filterLabel.bandLabel,
1335 selected = referenceSelector.run(skyCircle.refCat)
1337 if not selected.sourceCat.isContiguous():
1338 refCat = selected.sourceCat.copy(deep=
True)
1340 refCat = selected.sourceCat
1344 refCatName = refObjLoader.ref_dataset_name
1345 except AttributeError:
1346 refCatName = self.config.connections.photometryRefCat
1348 self.log.info(
"Applying color terms for physical filter=%r reference catalog=%s",
1349 filterLabel.physicalLabel, refCatName)
1350 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1354 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1355 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1357 refCat[skyCircle.fluxField+
'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1359 return refCat, skyCircle.fluxField
1361 def _check_star_lists(self, associations, name):
1363 if associations.nCcdImagesValidForFit() == 0:
1364 raise RuntimeError(
'No images in the ccdImageList!')
1365 if associations.fittedStarListSize() == 0:
1366 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
1367 if associations.refStarListSize() == 0:
1368 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
1370 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1371 """Compute chi2, log it, validate the model, and return chi2.
1376 The star/reference star associations to fit.
1378 The fitter to use for minimization.
1379 model : `lsst.jointcal.Model`
1380 The model being fit.
1382 Label to describe the chi2 (e.g.
"Initialized",
"Final").
1383 writeChi2Name : `str`, optional
1384 Filename prefix to write the chi2 contributions to.
1385 Do
not supply an extension: an appropriate one will be added.
1390 The chi2 object
for the current fitter
and model.
1395 Raised
if chi2
is infinite
or NaN.
1397 Raised
if the model
is not valid.
1399 if writeChi2Name
is not None:
1401 fit.saveChi2Contributions(fullpath+
"{type}")
1402 self.log.info(
"Wrote chi2 contributions files: %s", fullpath)
1404 chi2 = fit.computeChi2()
1405 self.log.info(
"%s %s", chi2Label, chi2)
1407 if not np.isfinite(chi2.chi2):
1408 raise FloatingPointError(f
'{chi2Label} chi2 is invalid: {chi2}')
1409 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1410 raise ValueError(
"Model is not valid: check log messages for warnings.")
1413 def _fit_photometry(self, associations, dataName=None):
1415 Fit the photometric data.
1420 The star/reference star associations to fit.
1422 Name of the data being processed (e.g. "1234_HSC-Y"),
for
1423 identifying debugging files.
1427 fit_result : `namedtuple`
1429 The photometric fitter used to perform the fit.
1431 The photometric model that was fit.
1433 self.log.info("=== Starting photometric fitting...")
1436 if self.config.photometryModel ==
"constrainedFlux":
1439 visitOrder=self.config.photometryVisitOrder,
1440 errorPedestal=self.config.photometryErrorPedestal)
1442 doLineSearch = self.config.allowLineSearch
1443 elif self.config.photometryModel ==
"constrainedMagnitude":
1446 visitOrder=self.config.photometryVisitOrder,
1447 errorPedestal=self.config.photometryErrorPedestal)
1449 doLineSearch = self.config.allowLineSearch
1450 elif self.config.photometryModel ==
"simpleFlux":
1452 errorPedestal=self.config.photometryErrorPedestal)
1453 doLineSearch =
False
1454 elif self.config.photometryModel ==
"simpleMagnitude":
1456 errorPedestal=self.config.photometryErrorPedestal)
1457 doLineSearch =
False
1462 if self.config.writeChi2FilesInitialFinal:
1463 baseName = f
"photometry_initial_chi2-{dataName}"
1466 if self.config.writeInitialModel:
1467 fullpath = self.
_getDebugPath_getDebugPath(f
"initial_photometry_model-{dataName}.txt")
1469 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialized", writeChi2Name=baseName)
1471 def getChi2Name(whatToFit):
1472 if self.config.writeChi2FilesOuterLoop:
1473 return f
"photometry_init-%s_chi2-{dataName}" % whatToFit
1479 if self.config.writeInitMatrix:
1480 dumpMatrixFile = self.
_getDebugPath_getDebugPath(f
"photometry_preinit-{dataName}")
1483 if self.config.photometryModel.startswith(
"constrained"):
1486 fit.minimize(
"ModelVisit", dumpMatrixFile=dumpMatrixFile)
1487 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize ModelVisit",
1488 writeChi2Name=getChi2Name(
"ModelVisit"))
1491 fit.minimize(
"Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1493 writeChi2Name=getChi2Name(
"Model"))
1495 fit.minimize(
"Fluxes")
1496 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Fluxes",
1497 writeChi2Name=getChi2Name(
"Fluxes"))
1499 fit.minimize(
"Model Fluxes", doLineSearch=doLineSearch)
1500 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize ModelFluxes",
1501 writeChi2Name=getChi2Name(
"ModelFluxes"))
1503 model.freezeErrorTransform()
1504 self.log.debug(
"Photometry error scales are frozen.")
1508 self.config.maxPhotometrySteps,
1511 doRankUpdate=self.config.photometryDoRankUpdate,
1512 doLineSearch=doLineSearch,
1519 def _fit_astrometry(self, associations, dataName=None):
1521 Fit the astrometric data.
1526 The star/reference star associations to fit.
1528 Name of the data being processed (e.g. "1234_HSC-Y"),
for
1529 identifying debugging files.
1533 fit_result : `namedtuple`
1535 The astrometric fitter used to perform the fit.
1537 The astrometric model that was fit.
1539 The model
for the sky to tangent plane projection that was used
in the fit.
1542 self.log.info("=== Starting astrometric fitting...")
1544 associations.deprojectFittedStars()
1551 if self.config.astrometryModel ==
"constrained":
1553 sky_to_tan_projection,
1554 chipOrder=self.config.astrometryChipOrder,
1555 visitOrder=self.config.astrometryVisitOrder)
1556 elif self.config.astrometryModel ==
"simple":
1558 sky_to_tan_projection,
1559 self.config.useInputWcs,
1561 order=self.config.astrometrySimpleOrder)
1566 if self.config.writeChi2FilesInitialFinal:
1567 baseName = f
"astrometry_initial_chi2-{dataName}"
1570 if self.config.writeInitialModel:
1571 fullpath = self.
_getDebugPath_getDebugPath(f
"initial_astrometry_model-{dataName}.txt")
1573 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initial", writeChi2Name=baseName)
1575 def getChi2Name(whatToFit):
1576 if self.config.writeChi2FilesOuterLoop:
1577 return f
"astrometry_init-%s_chi2-{dataName}" % whatToFit
1581 if self.config.writeInitMatrix:
1582 dumpMatrixFile = self.
_getDebugPath_getDebugPath(f
"astrometry_preinit-{dataName}")
1587 if self.config.astrometryModel ==
"constrained":
1588 fit.minimize(
"DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1589 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize DistortionsVisit",
1590 writeChi2Name=getChi2Name(
"DistortionsVisit"))
1593 fit.minimize(
"Distortions", dumpMatrixFile=dumpMatrixFile)
1594 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Distortions",
1595 writeChi2Name=getChi2Name(
"Distortions"))
1597 fit.minimize(
"Positions")
1598 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Positions",
1599 writeChi2Name=getChi2Name(
"Positions"))
1601 fit.minimize(
"Distortions Positions")
1602 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize DistortionsPositions",
1603 writeChi2Name=getChi2Name(
"DistortionsPositions"))
1607 self.config.maxAstrometrySteps,
1609 "Distortions Positions",
1610 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1611 doRankUpdate=self.config.astrometryDoRankUpdate,
1617 return Astrometry(fit, model, sky_to_tan_projection)
1619 def _check_stars(self, associations):
1620 """Count measured and reference stars per ccd and warn/log them."""
1621 for ccdImage
in associations.getCcdImageList():
1622 nMeasuredStars, nRefStars = ccdImage.countStars()
1623 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
1624 ccdImage.getName(), nMeasuredStars, nRefStars)
1625 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1626 self.log.warning(
"ccdImage %s has only %s measuredStars (desired %s)",
1627 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1628 if nRefStars < self.config.minRefStarsPerCcd:
1629 self.log.warning(
"ccdImage %s has only %s RefStars (desired %s)",
1630 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1632 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1634 sigmaRelativeTolerance=0,
1636 doLineSearch=False):
1637 """Run fitter.minimize up to max_steps times, returning the final chi2.
1642 The star/reference star associations to fit.
1644 The fitter to use for minimization.
1646 Maximum number of steps to run outlier rejection before declaring
1647 convergence failure.
1648 name : {
'photometry' or 'astrometry'}
1649 What type of data are we fitting (
for logs
and debugging files).
1651 Passed to ``fitter.minimize()`` to define the parameters to fit.
1652 dataName : `str`, optional
1653 Descriptive name
for this dataset (e.g. tract
and filter),
1655 sigmaRelativeTolerance : `float`, optional
1656 Convergence tolerance
for the fractional change
in the chi2 cut
1657 level
for determining outliers. If set to zero, iterations will
1658 continue until there are no outliers.
1659 doRankUpdate : `bool`, optional
1660 Do an Eigen rank update during minimization,
or recompute the full
1661 matrix
and gradient?
1662 doLineSearch : `bool`, optional
1663 Do a line search
for the optimum step during minimization?
1668 The final chi2 after the fit converges,
or is forced to end.
1673 Raised
if the fitter fails
with a non-finite value.
1675 Raised
if the fitter fails
for some other reason;
1676 log messages will provide further details.
1678 if self.config.writeInitMatrix:
1679 dumpMatrixFile = self.
_getDebugPath_getDebugPath(f
"{name}_postinit-{dataName}")
1683 oldChi2.chi2 = float(
"inf")
1684 for i
in range(max_steps):
1685 if self.config.writeChi2FilesOuterLoop:
1686 writeChi2Name = f
"{name}_iterate_{i}_chi2-{dataName}"
1688 writeChi2Name =
None
1689 result = fitter.minimize(whatToFit,
1690 self.config.outlierRejectSigma,
1691 sigmaRelativeTolerance=sigmaRelativeTolerance,
1692 doRankUpdate=doRankUpdate,
1693 doLineSearch=doLineSearch,
1694 dumpMatrixFile=dumpMatrixFile)
1696 chi2 = self.
_logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
1697 f
"Fit iteration {i}", writeChi2Name=writeChi2Name)
1699 if result == MinimizeResult.Converged:
1701 self.log.debug(
"fit has converged - no more outliers - redo minimization "
1702 "one more time in case we have lost accuracy in rank update.")
1704 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1705 sigmaRelativeTolerance=sigmaRelativeTolerance)
1706 chi2 = self.
_logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
"Fit completed")
1709 if chi2.chi2/chi2.ndof >= 4.0:
1710 self.log.error(
"Potentially bad fit: High chi-squared/ndof.")
1713 elif result == MinimizeResult.Chi2Increased:
1714 self.log.warning(
"Still some outliers remaining but chi2 increased - retry")
1716 chi2Ratio = chi2.chi2 / oldChi2.chi2
1718 self.log.warning(
'Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1719 chi2.chi2, oldChi2.chi2, chi2Ratio)
1726 msg = (
"Large chi2 increase between steps: fit likely cannot converge."
1727 " Try setting one or more of the `writeChi2*` config fields and looking"
1728 " at how individual star chi2-values evolve during the fit.")
1729 raise RuntimeError(msg)
1731 elif result == MinimizeResult.NonFinite:
1732 filename = self.
_getDebugPath_getDebugPath(
"{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1734 fitter.saveChi2Contributions(filename+
"{type}")
1735 msg =
"Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1736 raise FloatingPointError(msg.format(filename))
1737 elif result == MinimizeResult.Failed:
1738 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
1740 raise RuntimeError(
"Unxepected return code from minimize().")
1742 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
1746 def _make_output(self, ccdImageList, model, func):
1747 """Return the internal jointcal models converted to the afw
1748 structures that will be saved to disk.
1753 The list of CcdImages to get the output for.
1757 The name of the function to call on ``model`` to get the converted
1764 The data to be saved, keyed on (visit, detector).
1767 for ccdImage
in ccdImageList:
1768 ccd = ccdImage.ccdId
1769 visit = ccdImage.visit
1770 self.log.debug(
"%s for visit: %d, ccd: %d", func, visit, ccd)
1771 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1774 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1776 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1781 The star/reference star associations to fit.
1783 The astrometric model that was fit.
1784 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1785 Dict of ccdImage identifiers to dataRefs that were fit.
1787 ccdImageList = associations.getCcdImageList()
1788 output = self._make_output_make_output(ccdImageList, model, "makeSkyWcs")
1789 for key, skyWcs
in output.items():
1790 dataRef = visit_ccd_to_dataRef[key]
1792 dataRef.put(skyWcs,
'jointcal_wcs')
1793 except pexExceptions.Exception
as e:
1794 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
1797 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1799 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1804 The star/reference star associations to fit.
1806 The photoometric model that was fit.
1807 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1808 Dict of ccdImage identifiers to dataRefs that were fit.
1811 ccdImageList = associations.getCcdImageList()
1812 output = self._make_output_make_output(ccdImageList, model, "toPhotoCalib")
1813 for key, photoCalib
in output.items():
1814 dataRef = visit_ccd_to_dataRef[key]
1816 dataRef.put(photoCalib,
'jointcal_photoCalib')
1817 except pexExceptions.Exception
as e:
1818 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
1823 """Return an afw SourceTable to use as a base for creating the
1824 SourceCatalog to insert values from the dataFrame into.
1829 Table
with schema
and slots to use to make SourceCatalogs.
1832 schema.addField("centroid_x",
"D")
1833 schema.addField(
"centroid_y",
"D")
1834 schema.addField(
"centroid_xErr",
"F")
1835 schema.addField(
"centroid_yErr",
"F")
1836 schema.addField(
"shape_xx",
"D")
1837 schema.addField(
"shape_yy",
"D")
1838 schema.addField(
"shape_xy",
"D")
1839 schema.addField(
"flux_instFlux",
"D")
1840 schema.addField(
"flux_instFluxErr",
"D")
1842 table.defineCentroid(
"centroid")
1843 table.defineShape(
"shape")
1849 Get the sourceTable_visit columns to load from the catalogs.
1854 List of columns known to be available
in the sourceTable_visit.
1855 config : `JointcalConfig`
1856 A filled-
in config to to help define column names.
1857 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask`
1858 A configured source selector to define column names to load.
1863 List of columns to read
from sourceTable_visit.
1864 detectorColumn : `str`
1865 Name of the detector column.
1867 Name of the ixx/iyy/ixy columns.
1869 if 'detector' in inColumns:
1871 detectorColumn =
'detector'
1874 detectorColumn =
'ccd'
1876 columns = [
'visit', detectorColumn,
1877 'sourceId',
'x',
'xErr',
'y',
'yErr',
1878 config.sourceFluxType +
'_instFlux', config.sourceFluxType +
'_instFluxErr']
1880 if 'ixx' in inColumns:
1882 ixxColumns = [
'ixx',
'iyy',
'ixy']
1885 ixxColumns = [
'Ixx',
'Iyy',
'Ixy']
1886 columns.extend(ixxColumns)
1888 if sourceSelector.config.doFlags:
1889 columns.extend(sourceSelector.config.flags.bad)
1890 if sourceSelector.config.doUnresolved:
1891 columns.append(sourceSelector.config.unresolved.name)
1892 if sourceSelector.config.doIsolated:
1893 columns.append(sourceSelector.config.isolated.parentName)
1894 columns.append(sourceSelector.config.isolated.nChildName)
1896 return columns, detectorColumn, ixxColumns
1900 detectorColumn, ixxColumns, sourceFluxType, log):
1901 """Return an afw SourceCatalog extracted from a visit-level dataframe,
1902 limited to just one detector.
1907 Table factory to use to make the SourceCatalog that will be
1908 populated with data
from ``visitCatalog``.
1909 visitCatalog : `pandas.DataFrame`
1910 DataFrame to extract a detector catalog
from.
1912 Numeric id of the detector to extract
from ``visitCatalog``.
1913 detectorColumn : `str`
1914 Name of the detector column
in the catalog.
1915 ixxColumns : `list` [`str`]
1916 Names of the ixx/iyy/ixy columns
in the catalog.
1917 sourceFluxType : `str`
1918 Name of the catalog field to load instFluxes
from.
1920 Logging instance to log to.
1925 Detector-level catalog extracted
from ``visitCatalog``,
or `
None`
1926 if there was no data to load.
1929 mapping = {
'x':
'centroid_x',
1931 'xErr':
'centroid_xErr',
1932 'yErr':
'centroid_yErr',
1933 ixxColumns[0]:
'shape_xx',
1934 ixxColumns[1]:
'shape_yy',
1935 ixxColumns[2]:
'shape_xy',
1936 f
'{sourceFluxType}_instFlux':
'flux_instFlux',
1937 f
'{sourceFluxType}_instFluxErr':
'flux_instFluxErr',
1941 matched = visitCatalog[detectorColumn] == detectorId
1945 catalog.resize(sum(matched))
1946 view = visitCatalog.loc[matched]
1947 catalog[
'id'] = view.index
1948 for dfCol, afwCol
in mapping.items():
1949 catalog[afwCol] = view[dfCol]
1951 log.debug(
"%d sources selected in visit %d detector %d",
1953 view[
'visit'].iloc[0],
static Schema makeMinimalSchema()
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Class that handles the astrometric least squares problem.
Interface between AstrometryFit and the combinations of Mappings from pixels to some tangent plane (a...
Handler of an actual image from a single CCD.
Base class for Chi2Statistic and Chi2List, to allow addEntry inside Fitter for either class.
Simple structure to accumulate chi2 and ndof.
A multi-component model, fitting mappings for sensors and visits simultaneously.
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
A model where there is one independent transform per CcdImage.
def getTargetList(parsedCmd, **kwargs)
def __init__(self, *config=None)
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
def _compute_proper_motion_epoch(self, ccdImageList)
def _get_refcat_coordinate_error_override(self, refCat, name)
def _getDebugPath(self, filename)
def _check_star_lists(self, associations, name)
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", sigmaRelativeTolerance=0, doRankUpdate=True, doLineSearch=False)
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def _check_stars(self, associations)
def _prep_sky(self, associations, filters)
def _readDataId(self, butler, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
def _put_metrics(self, butlerQC, job, outputRefs)
def _build_ccdImage(self, data, associations, jointcalControl)
def runDataRef(self, dataRefs)
def __init__(self, butler=None, initInputs=None, **kwargs)
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
def _fit_photometry(self, associations, dataName=None)
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
def _make_output(self, ccdImageList, model, func)
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
def _fit_astrometry(self, associations, dataName=None)
def loadData(self, dataRefs, associations, jointcalControl)
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
def add_measurement(job, name, value)
def get_sourceTable_visit_columns(inColumns, config, sourceSelector)
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, detectorColumn, ixxColumns, sourceFluxType, log)
def writeModel(model, filename, log)
This is a virtual class that allows a lot of freedom in the choice of the projection from "Sky" (wher...