28 import astropy.units
as u
39 from lsst.obs.base
import Instrument
40 from lsst.pipe.tasks.colorterms
import ColortermLibrary
41 from lsst.verify
import Job, Measurement
44 ReferenceObjectLoader)
47 from .dataIds
import PerTractCcdDataIdContainer
52 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
54 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
55 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
60 meas = Measurement(job.metrics[name], value)
61 job.measurements.insert(meas)
65 """Subclass of TaskRunner for jointcalTask (gen2)
67 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
68 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
69 single dataRef, are are called repeatedly). This class transforms the processed
70 arguments generated by the ArgumentParser into the arguments expected by
71 Jointcal.runDataRef().
73 See pipeBase.TaskRunner for more information.
79 Return a list of tuples per tract, each containing (dataRefs, kwargs).
81 Jointcal operates on lists of dataRefs simultaneously.
83 kwargs[
'butler'] = parsedCmd.butler
87 for ref
in parsedCmd.id.refList:
88 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
90 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
98 Arguments for Task.runDataRef()
103 if self.doReturnResults is False:
105 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
107 if self.doReturnResults is True:
109 - ``result``: the result of calling jointcal.runDataRef()
110 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
115 dataRefList, kwargs = args
116 butler = kwargs.pop(
'butler')
117 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
120 result = task.runDataRef(dataRefList, **kwargs)
121 exitStatus = result.exitStatus
122 job_path = butler.get(
'verify_job_filename')
123 result.job.write(job_path[0])
124 except Exception
as e:
129 eName = type(e).__name__
130 tract = dataRefList[0].dataId[
'tract']
131 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
134 kwargs[
'butler'] = butler
135 if self.doReturnResults:
136 return pipeBase.Struct(result=result, exitStatus=exitStatus)
138 return pipeBase.Struct(exitStatus=exitStatus)
142 """Lookup function that asserts/hopes that a static calibration dataset
143 exists in a particular collection, since this task can't provide a single
144 date/time to use to search for one properly.
146 This is mostly useful for the ``camera`` dataset, in cases where the task's
147 quantum dimensions do *not* include something temporal, like ``exposure``
152 datasetType : `lsst.daf.butler.DatasetType`
153 Type of dataset being searched for.
154 registry : `lsst.daf.butler.Registry`
155 Data repository registry to search.
156 quantumDataId : `lsst.daf.butler.DataCoordinate`
157 Data ID of the quantum this camera should match.
158 collections : `Iterable` [ `str` ]
159 Collections that should be searched - but this lookup function works
160 by ignoring this in favor of a more-or-less hard-coded value.
164 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
165 Iterator over dataset references; should have only one element.
169 This implementation duplicates one in fgcmcal, and is at least quite
170 similar to another in cp_pipe. This duplicate has the most documentation.
171 Fixing this is DM-29661.
173 instrument = Instrument.fromName(quantumDataId[
"instrument"], registry)
174 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
175 return registry.queryDatasets(datasetType,
176 dataId=quantumDataId,
177 collections=[unboundedCollection],
182 """Lookup function that finds all refcats for all visits that overlap a
183 tract, rather than just the refcats that directly overlap the tract.
187 datasetType : `lsst.daf.butler.DatasetType`
188 Type of dataset being searched for.
189 registry : `lsst.daf.butler.Registry`
190 Data repository registry to search.
191 quantumDataId : `lsst.daf.butler.DataCoordinate`
192 Data ID of the quantum; expected to be something we can use as a
193 constraint to query for overlapping visits.
194 collections : `Iterable` [ `str` ]
195 Collections to search.
199 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
200 Iterator over refcat references.
209 for visit_data_id
in set(registry.queryDataIds(
"visit", dataId=quantumDataId).expanded()):
211 registry.queryDatasets(
213 collections=collections,
214 dataId=visit_data_id,
222 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter")):
223 """Middleware input/output connections for jointcal data."""
224 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
225 doc=
"The camera instrument that took these observations.",
227 storageClass=
"Camera",
228 dimensions=(
"instrument",),
230 lookupFunction=lookupStaticCalibrations,
232 inputSourceTableVisit = pipeBase.connectionTypes.Input(
233 doc=
"Source table in parquet format, per visit",
234 name=
"sourceTable_visit",
235 storageClass=
"DataFrame",
236 dimensions=(
"instrument",
"visit"),
240 inputVisitSummary = pipeBase.connectionTypes.Input(
241 doc=(
"Per-visit consolidated exposure metadata built from calexps. "
242 "These catalogs use detector id for the id and must be sorted for "
243 "fast lookups of a detector."),
245 storageClass=
"ExposureCatalog",
246 dimensions=(
"instrument",
"visit"),
250 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
251 doc=
"The astrometry reference catalog to match to loaded input catalog sources.",
252 name=
"gaia_dr2_20200414",
253 storageClass=
"SimpleCatalog",
254 dimensions=(
"skypix",),
257 lookupFunction=lookupVisitRefCats,
259 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
260 doc=
"The photometry reference catalog to match to loaded input catalog sources.",
261 name=
"ps1_pv3_3pi_20170110",
262 storageClass=
"SimpleCatalog",
263 dimensions=(
"skypix",),
266 lookupFunction=lookupVisitRefCats,
269 outputWcs = pipeBase.connectionTypes.Output(
270 doc=(
"Per-tract, per-visit world coordinate systems derived from the fitted model."
271 " These catalogs only contain entries for detectors with an output, and use"
272 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
273 name=
"jointcalSkyWcsCatalog",
274 storageClass=
"ExposureCatalog",
275 dimensions=(
"instrument",
"visit",
"skymap",
"tract"),
278 outputPhotoCalib = pipeBase.connectionTypes.Output(
279 doc=(
"Per-tract, per-visit photometric calibrations derived from the fitted model."
280 " These catalogs only contain entries for detectors with an output, and use"
281 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
282 name=
"jointcalPhotoCalibCatalog",
283 storageClass=
"ExposureCatalog",
284 dimensions=(
"instrument",
"visit",
"skymap",
"tract"),
296 if not config.doAstrometry:
297 self.prerequisiteInputs.remove(
"astrometryRefCat")
298 self.outputs.remove(
"outputWcs")
299 if not config.doPhotometry:
300 self.prerequisiteInputs.remove(
"photometryRefCat")
301 self.outputs.remove(
"outputPhotoCalib")
305 pipelineConnections=JointcalTaskConnections):
306 """Configuration for JointcalTask"""
308 doAstrometry = pexConfig.Field(
309 doc=
"Fit astrometry and write the fitted result.",
313 doPhotometry = pexConfig.Field(
314 doc=
"Fit photometry and write the fitted result.",
318 coaddName = pexConfig.Field(
319 doc=
"Type of coadd, typically deep or goodSeeing",
324 sourceFluxType = pexConfig.Field(
326 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
329 positionErrorPedestal = pexConfig.Field(
330 doc=
"Systematic term to apply to the measured position error (pixels)",
334 photometryErrorPedestal = pexConfig.Field(
335 doc=
"Systematic term to apply to the measured error on flux or magnitude as a "
336 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
341 matchCut = pexConfig.Field(
342 doc=
"Matching radius between fitted and reference stars (arcseconds)",
346 minMeasurements = pexConfig.Field(
347 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
351 minMeasuredStarsPerCcd = pexConfig.Field(
352 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
356 minRefStarsPerCcd = pexConfig.Field(
357 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
361 allowLineSearch = pexConfig.Field(
362 doc=
"Allow a line search during minimization, if it is reasonable for the model"
363 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
367 astrometrySimpleOrder = pexConfig.Field(
368 doc=
"Polynomial order for fitting the simple astrometry model.",
372 astrometryChipOrder = pexConfig.Field(
373 doc=
"Order of the per-chip transform for the constrained astrometry model.",
377 astrometryVisitOrder = pexConfig.Field(
378 doc=
"Order of the per-visit transform for the constrained astrometry model.",
382 useInputWcs = pexConfig.Field(
383 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
387 astrometryModel = pexConfig.ChoiceField(
388 doc=
"Type of model to fit to astrometry",
390 default=
"constrained",
391 allowed={
"simple":
"One polynomial per ccd",
392 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
394 photometryModel = pexConfig.ChoiceField(
395 doc=
"Type of model to fit to photometry",
397 default=
"constrainedMagnitude",
398 allowed={
"simpleFlux":
"One constant zeropoint per ccd and visit, fitting in flux space.",
399 "constrainedFlux":
"Constrained zeropoint per ccd, and one polynomial per visit,"
400 " fitting in flux space.",
401 "simpleMagnitude":
"One constant zeropoint per ccd and visit,"
402 " fitting in magnitude space.",
403 "constrainedMagnitude":
"Constrained zeropoint per ccd, and one polynomial per visit,"
404 " fitting in magnitude space.",
407 applyColorTerms = pexConfig.Field(
408 doc=
"Apply photometric color terms to reference stars?"
409 "Requires that colorterms be set to a ColortermLibrary",
413 colorterms = pexConfig.ConfigField(
414 doc=
"Library of photometric reference catalog name to color term dict.",
415 dtype=ColortermLibrary,
417 photometryVisitOrder = pexConfig.Field(
418 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
422 photometryDoRankUpdate = pexConfig.Field(
423 doc=(
"Do the rank update step during minimization. "
424 "Skipping this can help deal with models that are too non-linear."),
428 astrometryDoRankUpdate = pexConfig.Field(
429 doc=(
"Do the rank update step during minimization (should not change the astrometry fit). "
430 "Skipping this can help deal with models that are too non-linear."),
434 outlierRejectSigma = pexConfig.Field(
435 doc=
"How many sigma to reject outliers at during minimization.",
439 maxPhotometrySteps = pexConfig.Field(
440 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
444 maxAstrometrySteps = pexConfig.Field(
445 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
449 astrometryRefObjLoader = pexConfig.ConfigurableField(
450 target=LoadIndexedReferenceObjectsTask,
451 doc=
"Reference object loader for astrometric fit",
453 photometryRefObjLoader = pexConfig.ConfigurableField(
454 target=LoadIndexedReferenceObjectsTask,
455 doc=
"Reference object loader for photometric fit",
457 sourceSelector = sourceSelectorRegistry.makeField(
458 doc=
"How to select sources for cross-matching",
461 astrometryReferenceSelector = pexConfig.ConfigurableField(
462 target=ReferenceSourceSelectorTask,
463 doc=
"How to down-select the loaded astrometry reference catalog.",
465 photometryReferenceSelector = pexConfig.ConfigurableField(
466 target=ReferenceSourceSelectorTask,
467 doc=
"How to down-select the loaded photometry reference catalog.",
469 astrometryReferenceErr = pexConfig.Field(
470 doc=(
"Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
471 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
472 "If specified, overrides any existing `coord_*Err` values."),
479 writeInitMatrix = pexConfig.Field(
481 doc=(
"Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
482 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory. "
483 "Note that these files are the dense versions of the matrix, and so may be very large."),
486 writeChi2FilesInitialFinal = pexConfig.Field(
488 doc=
"Write .csv files containing the contributions to chi2 for the initialization and final fit.",
491 writeChi2FilesOuterLoop = pexConfig.Field(
493 doc=
"Write .csv files containing the contributions to chi2 for the outer fit loop.",
496 writeInitialModel = pexConfig.Field(
498 doc=(
"Write the pre-initialization model to text files, for debugging."
499 " Output is written to `initial[Astro|Photo]metryModel.txt` in the current working directory."),
502 debugOutputPath = pexConfig.Field(
505 doc=(
"Path to write debug output files to. Used by "
506 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
508 detailedProfile = pexConfig.Field(
511 doc=
"Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
517 msg =
"applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
518 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
520 msg = (
"Only doing astrometry, but Colorterms are not applied for astrometry;"
521 "applyColorTerms=True will be ignored.")
530 self.
sourceSelectorsourceSelector[
'science'].doSignalToNoise =
True
533 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.minimum = 10.
535 fluxField = f
"slot_{self.sourceFluxType}Flux_instFlux"
536 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.fluxField = fluxField
537 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.errField = fluxField +
"Err"
543 badFlags = [
'base_PixelFlags_flag_edge',
'base_PixelFlags_flag_saturated',
544 'base_PixelFlags_flag_interpolatedCenter',
'base_SdssCentroid_flag',
545 'base_PsfFlux_flag',
'base_PixelFlags_flag_suspectCenter']
557 """Write model to outfile."""
558 with open(filename,
"w")
as file:
559 file.write(repr(model))
560 log.info(
"Wrote %s to file: %s", model, filename)
563 @dataclasses.dataclass
565 """The input data jointcal needs for each detector/visit."""
567 """The visit identifier of this exposure."""
569 """The catalog derived from this exposure."""
571 """The VisitInfo of this exposure."""
573 """The detector of this exposure."""
575 """The photometric calibration of this exposure."""
577 """The WCS of this exposure."""
579 """The bounding box of this exposure."""
581 """The filter of this exposure."""
585 """Astrometricly and photometricly calibrate across multiple visits of the
590 butler : `lsst.daf.persistence.Butler`
591 The butler is passed to the refObjLoader constructor in case it is
592 needed. Ignored if the refObjLoader argument provides a loader directly.
593 Used to initialize the astrometry and photometry refObjLoaders.
594 initInputs : `dict`, optional
595 Dictionary used to initialize PipelineTasks (empty for jointcal).
598 ConfigClass = JointcalConfig
599 RunnerClass = JointcalRunner
600 _DefaultName =
"jointcal"
602 def __init__(self, butler=None, initInputs=None, **kwargs):
604 self.makeSubtask(
"sourceSelector")
605 if self.config.doAstrometry:
606 if initInputs
is None:
608 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
609 self.makeSubtask(
"astrometryReferenceSelector")
612 if self.config.doPhotometry:
613 if initInputs
is None:
615 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
616 self.makeSubtask(
"photometryReferenceSelector")
621 self.
jobjob = Job.load_metrics_package(subset=
'jointcal')
626 inputs = butlerQC.get(inputRefs)
628 tract = butlerQC.quantum.dataId[
'tract']
629 if self.config.doAstrometry:
631 dataIds=[ref.datasetRef.dataId
632 for ref
in inputRefs.astrometryRefCat],
633 refCats=inputs.pop(
'astrometryRefCat'),
634 config=self.config.astrometryRefObjLoader,
636 if self.config.doPhotometry:
638 dataIds=[ref.datasetRef.dataId
639 for ref
in inputRefs.photometryRefCat],
640 refCats=inputs.pop(
'photometryRefCat'),
641 config=self.config.photometryRefObjLoader,
643 outputs = self.
runrun(**inputs, tract=tract)
644 if self.config.doAstrometry:
645 self.
_put_output_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
646 inputs[
'inputCamera'],
"setWcs")
647 if self.config.doPhotometry:
648 self.
_put_output_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
649 inputs[
'inputCamera'],
"setPhotoCalib")
651 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
652 """Persist the output datasets to their appropriate datarefs.
656 butlerQC : `ButlerQuantumContext`
657 A butler which is specialized to operate in the context of a
658 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
659 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
660 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
661 The fitted objects to persist.
662 outputRefs : `list` [`OutputQuantizedConnection`]
663 The DatasetRefs to persist the data to.
664 camera : `lsst.afw.cameraGeom.Camera`
665 The camera for this instrument, to get detector ids from.
667 The method to call on the ExposureCatalog to set each output.
670 schema.addField(
'visit', type=
'I', doc=
'Visit number')
672 def new_catalog(visit, size):
673 """Return an catalog ready to be filled with appropriate output."""
676 catalog[
'visit'] = visit
678 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
679 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
683 detectors_per_visit = collections.defaultdict(int)
686 detectors_per_visit[key[0]] += 1
688 for ref
in outputRefs:
689 visit = ref.dataId[
'visit']
690 catalog = new_catalog(visit, detectors_per_visit[visit])
693 for detector
in camera:
694 detectorId = detector.getId()
695 key = (ref.dataId[
'visit'], detectorId)
696 if key
not in outputs:
698 self.log.debug(
"No %s output for detector %s in visit %s",
699 setter[3:], detectorId, visit)
702 catalog[i].setId(detectorId)
703 getattr(catalog[i], setter)(outputs[key])
707 butlerQC.put(catalog, ref)
708 self.log.info(
"Wrote %s detectors to %s", i, ref)
710 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
716 sourceFluxField =
"flux"
720 oldWcsList, bands = self.
_load_data_load_data(inputSourceTableVisit,
726 boundingCircle, center, radius, defaultFilter = self.
_prep_sky_prep_sky(associations, bands)
729 if self.config.doAstrometry:
733 referenceSelector=self.astrometryReferenceSelector,
737 astrometry_output = self.
_make_output_make_output(associations.getCcdImageList(),
741 astrometry_output =
None
743 if self.config.doPhotometry:
747 referenceSelector=self.photometryReferenceSelector,
751 reject_bad_fluxes=
True)
752 photometry_output = self.
_make_output_make_output(associations.getCcdImageList(),
756 photometry_output =
None
758 return pipeBase.Struct(outputWcs=astrometry_output,
759 outputPhotoCalib=photometry_output,
764 def _make_schema_table(self):
765 """Return an afw SourceTable to use as a base for creating the
766 SourceCatalog to insert values from the dataFrame into.
770 table : `lsst.afw.table.SourceTable`
771 Table with schema and slots to use to make SourceCatalogs.
774 schema.addField(
"centroid_x",
"D")
775 schema.addField(
"centroid_y",
"D")
776 schema.addField(
"centroid_xErr",
"F")
777 schema.addField(
"centroid_yErr",
"F")
778 schema.addField(
"shape_xx",
"D")
779 schema.addField(
"shape_yy",
"D")
780 schema.addField(
"shape_xy",
"D")
781 schema.addField(
"flux_instFlux",
"D")
782 schema.addField(
"flux_instFluxErr",
"D")
784 table.defineCentroid(
"centroid")
785 table.defineShape(
"shape")
788 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId):
789 """Return an afw SourceCatalog extracted from a visit-level dataframe,
790 limited to just one detector.
794 table : `lsst.afw.table.SourceTable`
795 Table factory to use to make the SourceCatalog that will be
796 populated with data from ``visitCatalog``.
797 visitCatalog : `pandas.DataFrame`
798 DataFrame to extract a detector catalog from.
800 Numeric id of the detector to extract from ``visitCatalog``.
804 catalog : `lsst.afw.table.SourceCatalog`
805 Detector-level catalog extracted from ``visitCatalog``.
808 mapping = {
'sourceId':
'id',
811 'xErr':
'centroid_xErr',
812 'yErr':
'centroid_yErr',
816 f
'{self.config.sourceFluxType}_instFlux':
'flux_instFlux',
817 f
'{self.config.sourceFluxType}_instFluxErr':
'flux_instFluxErr',
825 detector_column =
"detector" if "detector" in visitCatalog.columns
else "ccd"
827 matched = visitCatalog[detector_column] == detectorId
828 catalog.resize(sum(matched))
829 view = visitCatalog.loc[matched]
830 for dfCol, afwCol
in mapping.items():
831 catalog[afwCol] = view[dfCol]
833 self.log.debug(
"%d sources selected in visit %d detector %d",
835 view[
'visit'].iloc[0],
839 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
840 jointcalControl, camera):
841 """Read the data that jointcal needs to run. (Gen3 version)
843 Modifies ``associations`` in-place with the loaded data.
847 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
848 References to visit-level DataFrames to load the catalog data from.
849 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
850 Visit-level exposure summary catalog with metadata.
851 associations : `lsst.jointcal.Associations`
852 Object to add the loaded data to by constructing new CcdImages.
853 jointcalControl : `jointcal.JointcalControl`
854 Control object for C++ associations management.
855 camera : `lsst.afw.cameraGeom.Camera`
856 Camera object for detector geometry.
860 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
861 The original WCS of the input data, to aid in writing tests.
862 bands : `list` [`str`]
863 The filter bands of each input dataset.
867 load_cat_prof_file =
'jointcal_load_data.prof' if self.config.detailedProfile
else ''
868 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
871 catalogMap = {ref.dataId[
'visit']: i
for i, ref
in enumerate(inputSourceTableVisit)}
872 detectorDict = {detector.getId(): detector
for detector
in camera}
874 for visitSummaryRef
in inputVisitSummary:
875 visitSummary = visitSummaryRef.get()
876 visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId[
'visit']]].get()
877 selected = self.sourceSelector.
run(visitCatalog)
880 detectors = {id: index
for index, id
in enumerate(visitSummary[
'id'])}
881 for id, index
in detectors.items():
883 data = self.
_make_one_input_data_make_one_input_data(visitSummary[index], catalog, detectorDict)
884 result = self.
_build_ccdImage_build_ccdImage(data, associations, jointcalControl)
885 if result
is not None:
886 oldWcsList.append(result.wcs)
888 filters.append(data.filter)
889 if len(filters) == 0:
890 raise RuntimeError(
"No data to process: did source selector remove all sources?")
891 filters = collections.Counter(filters)
893 return oldWcsList, filters
895 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
896 """Return a data structure for this detector+visit."""
899 visitInfo=visitRecord.getVisitInfo(),
900 detector=detectorDict[visitRecord.getId()],
901 photoCalib=visitRecord.getPhotoCalib(),
902 wcs=visitRecord.getWcs(),
903 bbox=visitRecord.getBBox(),
907 physical=visitRecord[
'physical_filter']))
912 def _getMetadataName(self):
916 def _makeArgumentParser(cls):
917 """Create an argument parser"""
918 parser = pipeBase.ArgumentParser(name=cls.
_DefaultName_DefaultName)
919 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
920 ContainerClass=PerTractCcdDataIdContainer)
923 def _build_ccdImage(self, data, associations, jointcalControl):
925 Extract the necessary things from this catalog+metadata to add a new
930 data : `JointcalInputData`
931 The loaded input data.
932 associations : `lsst.jointcal.Associations`
933 Object to add the info to, to construct a new CcdImage
934 jointcalControl : `jointcal.JointcalControl`
935 Control object for associations management
941 The TAN WCS of this image, read from the calexp
942 (`lsst.afw.geom.SkyWcs`).
944 A key to identify this dataRef by its visit and ccd ids
947 if there are no sources in the loaded catalog.
949 if len(data.catalog) == 0:
950 self.log.warn(
"No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
953 associations.createCcdImage(data.catalog,
957 data.filter.physicalLabel,
961 data.detector.getId(),
964 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key'))
965 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
966 return Result(data.wcs, Key(data.visit, data.detector.getId()))
968 def _readDataId(self, butler, dataId):
969 """Read all of the data for one dataId from the butler. (gen2 version)"""
971 if "visit" in dataId.keys():
972 visit = dataId[
"visit"]
974 visit = butler.getButler().queryMetadata(
"calexp", (
"visit"), butler.dataId)[0]
975 detector = butler.get(
'calexp_detector', dataId=dataId)
977 catalog = butler.get(
'src',
978 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
980 goodSrc = self.sourceSelector.
run(catalog)
981 self.log.debug(
"%d sources selected in visit %d detector %d",
982 len(goodSrc.sourceCat),
986 catalog=goodSrc.sourceCat,
987 visitInfo=butler.get(
'calexp_visitInfo', dataId=dataId),
989 photoCalib=butler.get(
'calexp_photoCalib', dataId=dataId),
990 wcs=butler.get(
'calexp_wcs', dataId=dataId),
991 bbox=butler.get(
'calexp_bbox', dataId=dataId),
992 filter=butler.get(
'calexp_filterLabel', dataId=dataId))
994 def loadData(self, dataRefs, associations, jointcalControl):
995 """Read the data that jointcal needs to run. (Gen2 version)"""
996 visit_ccd_to_dataRef = {}
999 load_cat_prof_file =
'jointcal_loadData.prof' if self.config.detailedProfile
else ''
1000 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1002 camera = dataRefs[0].get(
'camera', immediate=
True)
1004 for dataRef
in dataRefs:
1005 data = self.
_readDataId_readDataId(dataRef.getButler(), dataRef.dataId)
1006 result = self.
_build_ccdImage_build_ccdImage(data, associations, jointcalControl)
1009 oldWcsList.append(result.wcs)
1010 visit_ccd_to_dataRef[result.key] = dataRef
1011 filters.append(data.filter)
1012 if len(filters) == 0:
1013 raise RuntimeError(
"No data to process: did source selector remove all sources?")
1014 filters = collections.Counter(filters)
1016 return oldWcsList, filters, visit_ccd_to_dataRef
1018 def _getDebugPath(self, filename):
1019 """Constructs a path to filename using the configured debug path.
1021 return os.path.join(self.config.debugOutputPath, filename)
1023 def _prep_sky(self, associations, filters):
1024 """Prepare on-sky and other data that must be computed after data has
1027 associations.computeCommonTangentPoint()
1029 boundingCircle = associations.computeBoundingCircle()
1031 radius =
lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1033 self.log.info(f
"Data has center={center} with radius={radius.asDegrees()} degrees.")
1036 defaultFilter = filters.most_common(1)[0][0]
1037 self.log.debug(
"Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1039 return boundingCircle, center, radius, defaultFilter
1041 @pipeBase.timeMethod
1044 Jointly calibrate the astrometry and photometry across a set of images.
1046 NOTE: this is for gen2 middleware only.
1050 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1051 List of data references to the exposures to be fit.
1055 result : `lsst.pipe.base.Struct`
1056 Struct of metadata from the fit, containing:
1059 The provided data references that were fit (with updated WCSs)
1061 The original WCS from each dataRef
1063 Dictionary of internally-computed metrics for testing/validation.
1065 if len(dataRefs) == 0:
1066 raise ValueError(
'Need a non-empty list of data references!')
1070 sourceFluxField =
"slot_%sFlux" % (self.config.sourceFluxType,)
1074 oldWcsList, filters, visit_ccd_to_dataRef = self.
loadDataloadData(dataRefs,
1078 boundingCircle, center, radius, defaultFilter = self.
_prep_sky_prep_sky(associations, filters)
1081 tract = dataRefs[0].dataId[
'tract']
1083 if self.config.doAstrometry:
1087 referenceSelector=self.astrometryReferenceSelector,
1095 if self.config.doPhotometry:
1099 referenceSelector=self.photometryReferenceSelector,
1103 reject_bad_fluxes=
True)
1108 return pipeBase.Struct(dataRefs=dataRefs,
1109 oldWcsList=oldWcsList,
1113 defaultFilter=defaultFilter,
1115 exitStatus=exitStatus)
1117 def _get_refcat_coordinate_error_override(self, refCat, name):
1118 """Check whether we should override the refcat coordinate errors, and
1119 return the overridden error if necessary.
1123 refCat : `lsst.afw.table.SimpleCatalog`
1124 The reference catalog to check for a ``coord_raErr`` field.
1126 Whether we are doing "astrometry" or "photometry".
1130 refCoordErr : `float`
1131 The refcat coordinate error to use, or NaN if we are not overriding
1136 lsst.pex.config.FieldValidationError
1137 Raised if the refcat does not contain coordinate errors and
1138 ``config.astrometryReferenceErr`` is not set.
1142 if name.lower() ==
"photometry":
1143 if 'coord_raErr' not in refCat.schema:
1148 if self.config.astrometryReferenceErr
is None and 'coord_raErr' not in refCat.schema:
1149 msg = (
"Reference catalog does not contain coordinate errors, "
1150 "and config.astrometryReferenceErr not supplied.")
1151 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1155 if self.config.astrometryReferenceErr
is not None and 'coord_raErr' in refCat.schema:
1156 self.log.warn(
"Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1157 self.config.astrometryReferenceErr)
1159 if self.config.astrometryReferenceErr
is None:
1162 return self.config.astrometryReferenceErr
1164 def _compute_proper_motion_epoch(self, ccdImageList):
1165 """Return the proper motion correction epoch of the provided images.
1169 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1170 The images to compute the appropriate epoch for.
1174 epoch : `astropy.time.Time`
1175 The date to use for proper motion corrections.
1177 mjds = [ccdImage.getMjd()
for ccdImage
in ccdImageList]
1178 return astropy.time.Time(np.mean(mjds), format=
'mjd', scale=
"tai")
1180 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1181 tract="", match_cut=3.0,
1182 reject_bad_fluxes=False, *,
1183 name="", refObjLoader=None, referenceSelector=None,
1184 fit_function=None, epoch=None):
1185 """Load reference catalog, perform the fit, and return the result.
1189 associations : `lsst.jointcal.Associations`
1190 The star/reference star associations to fit.
1191 defaultFilter : `lsst.afw.image.FilterLabel`
1192 filter to load from reference catalog.
1193 center : `lsst.geom.SpherePoint`
1194 ICRS center of field to load from reference catalog.
1195 radius : `lsst.geom.Angle`
1196 On-sky radius to load from reference catalog.
1198 Name of thing being fit: "astrometry" or "photometry".
1199 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1200 Reference object loader to use to load a reference catalog.
1201 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1202 Selector to use to pick objects from the loaded reference catalog.
1203 fit_function : callable
1204 Function to call to perform fit (takes Associations object).
1205 tract : `str`, optional
1206 Name of tract currently being fit.
1207 match_cut : `float`, optional
1208 Radius in arcseconds to find cross-catalog matches to during
1209 associations.associateCatalogs.
1210 reject_bad_fluxes : `bool`, optional
1211 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1212 epoch : `astropy.time.Time`, optional
1213 Epoch to which to correct refcat proper motion and parallax,
1214 or `None` to not apply such corrections.
1218 result : `Photometry` or `Astrometry`
1219 Result of `fit_function()`
1221 self.log.info(
"====== Now processing %s...", name)
1224 associations.associateCatalogs(match_cut)
1226 associations.fittedStarListSize())
1228 applyColorterms =
False if name.lower() ==
"astrometry" else self.config.applyColorTerms
1230 center, radius, defaultFilter,
1231 applyColorterms=applyColorterms,
1235 associations.collectRefStars(refCat,
1236 self.config.matchCut*lsst.geom.arcseconds,
1238 refCoordinateErr=refCoordErr,
1239 rejectBadFluxes=reject_bad_fluxes)
1241 associations.refStarListSize())
1243 associations.prepareFittedStars(self.config.minMeasurements)
1247 associations.nFittedStarsWithAssociatedRefStar())
1249 associations.fittedStarListSize())
1251 associations.nCcdImagesValidForFit())
1253 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if self.config.detailedProfile
else ''
1254 dataName =
"{}_{}".format(tract, defaultFilter.bandLabel)
1255 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1256 result = fit_function(associations, dataName)
1259 if self.config.writeChi2FilesInitialFinal:
1260 baseName = self.
_getDebugPath_getDebugPath(f
"{name}_final_chi2-{dataName}")
1261 result.fit.saveChi2Contributions(baseName+
"{type}")
1262 self.log.info(
"Wrote chi2 contributions files: %s", baseName)
1266 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1267 applyColorterms=False, epoch=None):
1268 """Load the necessary reference catalog sources, convert fluxes to
1269 correct units, and apply color term corrections if requested.
1273 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1274 The reference catalog loader to use to get the data.
1275 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1276 Source selector to apply to loaded reference catalog.
1277 center : `lsst.geom.SpherePoint`
1278 The center around which to load sources.
1279 radius : `lsst.geom.Angle`
1280 The radius around ``center`` to load sources in.
1281 filterLabel : `lsst.afw.image.FilterLabel`
1282 The camera filter to load fluxes for.
1283 applyColorterms : `bool`
1284 Apply colorterm corrections to the refcat for ``filterName``?
1285 epoch : `astropy.time.Time`, optional
1286 Epoch to which to correct refcat proper motion and parallax,
1287 or `None` to not apply such corrections.
1291 refCat : `lsst.afw.table.SimpleCatalog`
1292 The loaded reference catalog.
1294 The name of the reference catalog flux field appropriate for ``filterName``.
1296 skyCircle = refObjLoader.loadSkyCircle(center,
1298 filterLabel.bandLabel,
1301 selected = referenceSelector.run(skyCircle.refCat)
1303 if not selected.sourceCat.isContiguous():
1304 refCat = selected.sourceCat.copy(deep=
True)
1306 refCat = selected.sourceCat
1309 refCatName = refObjLoader.ref_dataset_name
1310 self.log.info(
"Applying color terms for physical filter=%r reference catalog=%s",
1311 filterLabel.physicalLabel, refCatName)
1312 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1316 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1317 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1319 refCat[skyCircle.fluxField+
'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1321 return refCat, skyCircle.fluxField
1323 def _check_star_lists(self, associations, name):
1325 if associations.nCcdImagesValidForFit() == 0:
1326 raise RuntimeError(
'No images in the ccdImageList!')
1327 if associations.fittedStarListSize() == 0:
1328 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
1329 if associations.refStarListSize() == 0:
1330 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
1332 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1333 """Compute chi2, log it, validate the model, and return chi2.
1337 associations : `lsst.jointcal.Associations`
1338 The star/reference star associations to fit.
1339 fit : `lsst.jointcal.FitterBase`
1340 The fitter to use for minimization.
1341 model : `lsst.jointcal.Model`
1342 The model being fit.
1344 Label to describe the chi2 (e.g. "Initialized", "Final").
1345 writeChi2Name : `str`, optional
1346 Filename prefix to write the chi2 contributions to.
1347 Do not supply an extension: an appropriate one will be added.
1351 chi2: `lsst.jointcal.Chi2Accumulator`
1352 The chi2 object for the current fitter and model.
1357 Raised if chi2 is infinite or NaN.
1359 Raised if the model is not valid.
1361 if writeChi2Name
is not None:
1363 fit.saveChi2Contributions(fullpath+
"{type}")
1364 self.log.info(
"Wrote chi2 contributions files: %s", fullpath)
1366 chi2 = fit.computeChi2()
1367 self.log.info(
"%s %s", chi2Label, chi2)
1369 if not np.isfinite(chi2.chi2):
1370 raise FloatingPointError(f
'{chi2Label} chi2 is invalid: {chi2}')
1371 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1372 raise ValueError(
"Model is not valid: check log messages for warnings.")
1375 def _fit_photometry(self, associations, dataName=None):
1377 Fit the photometric data.
1381 associations : `lsst.jointcal.Associations`
1382 The star/reference star associations to fit.
1384 Name of the data being processed (e.g. "1234_HSC-Y"), for
1385 identifying debugging files.
1389 fit_result : `namedtuple`
1390 fit : `lsst.jointcal.PhotometryFit`
1391 The photometric fitter used to perform the fit.
1392 model : `lsst.jointcal.PhotometryModel`
1393 The photometric model that was fit.
1395 self.log.info(
"=== Starting photometric fitting...")
1398 if self.config.photometryModel ==
"constrainedFlux":
1401 visitOrder=self.config.photometryVisitOrder,
1402 errorPedestal=self.config.photometryErrorPedestal)
1404 doLineSearch = self.config.allowLineSearch
1405 elif self.config.photometryModel ==
"constrainedMagnitude":
1408 visitOrder=self.config.photometryVisitOrder,
1409 errorPedestal=self.config.photometryErrorPedestal)
1411 doLineSearch = self.config.allowLineSearch
1412 elif self.config.photometryModel ==
"simpleFlux":
1414 errorPedestal=self.config.photometryErrorPedestal)
1415 doLineSearch =
False
1416 elif self.config.photometryModel ==
"simpleMagnitude":
1418 errorPedestal=self.config.photometryErrorPedestal)
1419 doLineSearch =
False
1424 if self.config.writeChi2FilesInitialFinal:
1425 baseName = f
"photometry_initial_chi2-{dataName}"
1428 if self.config.writeInitialModel:
1429 fullpath = self.
_getDebugPath_getDebugPath(
"initialPhotometryModel.txt")
1431 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialized", writeChi2Name=baseName)
1433 def getChi2Name(whatToFit):
1434 if self.config.writeChi2FilesOuterLoop:
1435 return f
"photometry_init-%s_chi2-{dataName}" % whatToFit
1441 dumpMatrixFile = self.
_getDebugPath_getDebugPath(
"photometry_preinit")
if self.config.writeInitMatrix
else ""
1442 if self.config.photometryModel.startswith(
"constrained"):
1445 fit.minimize(
"ModelVisit", dumpMatrixFile=dumpMatrixFile)
1446 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize ModelVisit",
1447 writeChi2Name=getChi2Name(
"ModelVisit"))
1450 fit.minimize(
"Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1452 writeChi2Name=getChi2Name(
"Model"))
1454 fit.minimize(
"Fluxes")
1455 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Fluxes",
1456 writeChi2Name=getChi2Name(
"Fluxes"))
1458 fit.minimize(
"Model Fluxes", doLineSearch=doLineSearch)
1459 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize ModelFluxes",
1460 writeChi2Name=getChi2Name(
"ModelFluxes"))
1462 model.freezeErrorTransform()
1463 self.log.debug(
"Photometry error scales are frozen.")
1467 self.config.maxPhotometrySteps,
1470 doRankUpdate=self.config.photometryDoRankUpdate,
1471 doLineSearch=doLineSearch,
1478 def _fit_astrometry(self, associations, dataName=None):
1480 Fit the astrometric data.
1484 associations : `lsst.jointcal.Associations`
1485 The star/reference star associations to fit.
1487 Name of the data being processed (e.g. "1234_HSC-Y"), for
1488 identifying debugging files.
1492 fit_result : `namedtuple`
1493 fit : `lsst.jointcal.AstrometryFit`
1494 The astrometric fitter used to perform the fit.
1495 model : `lsst.jointcal.AstrometryModel`
1496 The astrometric model that was fit.
1497 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1498 The model for the sky to tangent plane projection that was used in the fit.
1501 self.log.info(
"=== Starting astrometric fitting...")
1503 associations.deprojectFittedStars()
1510 if self.config.astrometryModel ==
"constrained":
1512 sky_to_tan_projection,
1513 chipOrder=self.config.astrometryChipOrder,
1514 visitOrder=self.config.astrometryVisitOrder)
1515 elif self.config.astrometryModel ==
"simple":
1517 sky_to_tan_projection,
1518 self.config.useInputWcs,
1520 order=self.config.astrometrySimpleOrder)
1525 if self.config.writeChi2FilesInitialFinal:
1526 baseName = f
"astrometry_initial_chi2-{dataName}"
1529 if self.config.writeInitialModel:
1530 fullpath = self.
_getDebugPath_getDebugPath(
"initialAstrometryModel.txt")
1532 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initial", writeChi2Name=baseName)
1534 def getChi2Name(whatToFit):
1535 if self.config.writeChi2FilesOuterLoop:
1536 return f
"astrometry_init-%s_chi2-{dataName}" % whatToFit
1540 dumpMatrixFile = self.
_getDebugPath_getDebugPath(
"astrometry_preinit")
if self.config.writeInitMatrix
else ""
1543 if self.config.astrometryModel ==
"constrained":
1544 fit.minimize(
"DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1545 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize DistortionsVisit",
1546 writeChi2Name=getChi2Name(
"DistortionsVisit"))
1549 fit.minimize(
"Distortions", dumpMatrixFile=dumpMatrixFile)
1550 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Distortions",
1551 writeChi2Name=getChi2Name(
"Distortions"))
1553 fit.minimize(
"Positions")
1554 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Positions",
1555 writeChi2Name=getChi2Name(
"Positions"))
1557 fit.minimize(
"Distortions Positions")
1558 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize DistortionsPositions",
1559 writeChi2Name=getChi2Name(
"DistortionsPositions"))
1563 self.config.maxAstrometrySteps,
1565 "Distortions Positions",
1566 doRankUpdate=self.config.astrometryDoRankUpdate,
1572 return Astrometry(fit, model, sky_to_tan_projection)
1574 def _check_stars(self, associations):
1575 """Count measured and reference stars per ccd and warn/log them."""
1576 for ccdImage
in associations.getCcdImageList():
1577 nMeasuredStars, nRefStars = ccdImage.countStars()
1578 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
1579 ccdImage.getName(), nMeasuredStars, nRefStars)
1580 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1581 self.log.warn(
"ccdImage %s has only %s measuredStars (desired %s)",
1582 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1583 if nRefStars < self.config.minRefStarsPerCcd:
1584 self.log.warn(
"ccdImage %s has only %s RefStars (desired %s)",
1585 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1587 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1590 doLineSearch=False):
1591 """Run fitter.minimize up to max_steps times, returning the final chi2.
1595 associations : `lsst.jointcal.Associations`
1596 The star/reference star associations to fit.
1597 fitter : `lsst.jointcal.FitterBase`
1598 The fitter to use for minimization.
1600 Maximum number of steps to run outlier rejection before declaring
1601 convergence failure.
1602 name : {'photometry' or 'astrometry'}
1603 What type of data are we fitting (for logs and debugging files).
1605 Passed to ``fitter.minimize()`` to define the parameters to fit.
1606 dataName : `str`, optional
1607 Descriptive name for this dataset (e.g. tract and filter),
1609 doRankUpdate : `bool`, optional
1610 Do an Eigen rank update during minimization, or recompute the full
1611 matrix and gradient?
1612 doLineSearch : `bool`, optional
1613 Do a line search for the optimum step during minimization?
1617 chi2: `lsst.jointcal.Chi2Statistic`
1618 The final chi2 after the fit converges, or is forced to end.
1623 Raised if the fitter fails with a non-finite value.
1625 Raised if the fitter fails for some other reason;
1626 log messages will provide further details.
1628 dumpMatrixFile = self.
_getDebugPath_getDebugPath(f
"{name}_postinit")
if self.config.writeInitMatrix
else ""
1630 oldChi2.chi2 = float(
"inf")
1631 for i
in range(max_steps):
1632 if self.config.writeChi2FilesOuterLoop:
1633 writeChi2Name = f
"{name}_iterate_{i}_chi2-{dataName}"
1635 writeChi2Name =
None
1636 result = fitter.minimize(whatToFit,
1637 self.config.outlierRejectSigma,
1638 doRankUpdate=doRankUpdate,
1639 doLineSearch=doLineSearch,
1640 dumpMatrixFile=dumpMatrixFile)
1642 chi2 = self.
_logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
1643 f
"Fit iteration {i}", writeChi2Name=writeChi2Name)
1645 if result == MinimizeResult.Converged:
1647 self.log.debug(
"fit has converged - no more outliers - redo minimization "
1648 "one more time in case we have lost accuracy in rank update.")
1650 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1651 chi2 = self.
_logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
"Fit completed")
1654 if chi2.chi2/chi2.ndof >= 4.0:
1655 self.log.error(
"Potentially bad fit: High chi-squared/ndof.")
1658 elif result == MinimizeResult.Chi2Increased:
1659 self.log.warn(
"Still some outliers remaining but chi2 increased - retry")
1661 chi2Ratio = chi2.chi2 / oldChi2.chi2
1663 self.log.warn(
'Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1664 chi2.chi2, oldChi2.chi2, chi2Ratio)
1671 msg = (
"Large chi2 increase between steps: fit likely cannot converge."
1672 " Try setting one or more of the `writeChi2*` config fields and looking"
1673 " at how individual star chi2-values evolve during the fit.")
1674 raise RuntimeError(msg)
1676 elif result == MinimizeResult.NonFinite:
1677 filename = self.
_getDebugPath_getDebugPath(
"{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1679 fitter.saveChi2Contributions(filename+
"{type}")
1680 msg =
"Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1681 raise FloatingPointError(msg.format(filename))
1682 elif result == MinimizeResult.Failed:
1683 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
1685 raise RuntimeError(
"Unxepected return code from minimize().")
1687 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
1691 def _make_output(self, ccdImageList, model, func):
1692 """Return the internal jointcal models converted to the afw
1693 structures that will be saved to disk.
1697 ccdImageList : `lsst.jointcal.CcdImageList`
1698 The list of CcdImages to get the output for.
1699 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1700 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1702 The name of the function to call on ``model`` to get the converted
1703 structure. Must accept an `lsst.jointcal.CcdImage`.
1707 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1708 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1709 The data to be saved, keyed on (visit, detector).
1712 for ccdImage
in ccdImageList:
1713 ccd = ccdImage.ccdId
1714 visit = ccdImage.visit
1715 self.log.debug(
"%s for visit: %d, ccd: %d", func, visit, ccd)
1716 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1719 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1721 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1725 associations : `lsst.jointcal.Associations`
1726 The star/reference star associations to fit.
1727 model : `lsst.jointcal.AstrometryModel`
1728 The astrometric model that was fit.
1729 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1730 Dict of ccdImage identifiers to dataRefs that were fit.
1732 ccdImageList = associations.getCcdImageList()
1733 output = self.
_make_output_make_output(ccdImageList, model,
"makeSkyWcs")
1734 for key, skyWcs
in output.items():
1735 dataRef = visit_ccd_to_dataRef[key]
1737 dataRef.put(skyWcs,
'jointcal_wcs')
1738 except pexExceptions.Exception
as e:
1739 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
1742 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1744 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1748 associations : `lsst.jointcal.Associations`
1749 The star/reference star associations to fit.
1750 model : `lsst.jointcal.PhotometryModel`
1751 The photoometric model that was fit.
1752 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1753 Dict of ccdImage identifiers to dataRefs that were fit.
1756 ccdImageList = associations.getCcdImageList()
1757 output = self.
_make_output_make_output(ccdImageList, model,
"toPhotoCalib")
1758 for key, photoCalib
in output.items():
1759 dataRef = visit_ccd_to_dataRef[key]
1761 dataRef.put(photoCalib,
'jointcal_photoCalib')
1762 except pexExceptions.Exception
as e:
1763 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
static Schema makeMinimalSchema()
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Class that handles the astrometric least squares problem.
Simple structure to accumulate chi2 and ndof.
A multi-component model, fitting mappings for sensors and visits simultaneously.
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
A model where there is one independent transform per CcdImage.
def getTargetList(parsedCmd, **kwargs)
def __init__(self, *config=None)
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
def _compute_proper_motion_epoch(self, ccdImageList)
def _get_refcat_coordinate_error_override(self, refCat, name)
def _getDebugPath(self, filename)
def _check_star_lists(self, associations, name)
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def _check_stars(self, associations)
def _prep_sky(self, associations, filters)
def _readDataId(self, butler, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
def _build_ccdImage(self, data, associations, jointcalControl)
def runDataRef(self, dataRefs)
def __init__(self, butler=None, initInputs=None, **kwargs)
def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId)
def _make_schema_table(self)
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
def _fit_photometry(self, associations, dataName=None)
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
def _make_output(self, ccdImageList, model, func)
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
def _fit_astrometry(self, associations, dataName=None)
def loadData(self, dataRefs, associations, jointcalControl)
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
def add_measurement(job, name, value)
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
def writeModel(model, filename, log)