lsst.fgcmcal  21.0.0-5-gb155db7+edafbab411
fgcmBuildStarsBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for BuildStars using src tables or sourceTable_visit tables.
22 """
23 
24 import os
25 import sys
26 import traceback
27 import abc
28 
29 import numpy as np
30 
31 import lsst.daf.persistence as dafPersist
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 import lsst.afw.table as afwTable
35 import lsst.geom as geom
36 from lsst.daf.base import PropertyList
37 from lsst.daf.base.dateTime import DateTime
38 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
39 
40 from .utilities import computeApertureRadiusFromDataRef
41 from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
42 
43 import fgcm
44 
45 REFSTARS_FORMAT_VERSION = 1
46 
47 __all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
48 
49 
50 class FgcmBuildStarsConfigBase(pexConfig.Config):
51  """Base config for FgcmBuildStars tasks"""
52 
53  instFluxField = pexConfig.Field(
54  doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55  "The associated flag will be implicitly included in badFlags"),
56  dtype=str,
57  default='slot_CalibFlux_instFlux',
58  )
59  minPerBand = pexConfig.Field(
60  doc="Minimum observations per band",
61  dtype=int,
62  default=2,
63  )
64  matchRadius = pexConfig.Field(
65  doc="Match radius (arcseconds)",
66  dtype=float,
67  default=1.0,
68  )
69  isolationRadius = pexConfig.Field(
70  doc="Isolation radius (arcseconds)",
71  dtype=float,
72  default=2.0,
73  )
74  densityCutNside = pexConfig.Field(
75  doc="Density cut healpix nside",
76  dtype=int,
77  default=128,
78  )
79  densityCutMaxPerPixel = pexConfig.Field(
80  doc="Density cut number of stars per pixel",
81  dtype=int,
82  default=1000,
83  )
84  matchNside = pexConfig.Field(
85  doc="Healpix Nside for matching",
86  dtype=int,
87  default=4096,
88  )
89  coarseNside = pexConfig.Field(
90  doc="Healpix coarse Nside for partitioning matches",
91  dtype=int,
92  default=8,
93  )
94  filterMap = pexConfig.DictField(
95  doc="Mapping from 'filterName' to band.",
96  keytype=str,
97  itemtype=str,
98  default={},
99  )
100  requiredBands = pexConfig.ListField(
101  doc="Bands required for each star",
102  dtype=str,
103  default=(),
104  )
105  primaryBands = pexConfig.ListField(
106  doc=("Bands for 'primary' star matches. "
107  "A star must be observed in one of these bands to be considered "
108  "as a calibration star."),
109  dtype=str,
110  default=None
111  )
112  visitDataRefName = pexConfig.Field(
113  doc="dataRef name for the 'visit' field, usually 'visit'.",
114  dtype=str,
115  default="visit"
116  )
117  ccdDataRefName = pexConfig.Field(
118  doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
119  dtype=str,
120  default="ccd"
121  )
122  doApplyWcsJacobian = pexConfig.Field(
123  doc="Apply the jacobian of the WCS to the star observations prior to fit?",
124  dtype=bool,
125  default=True
126  )
127  doModelErrorsWithBackground = pexConfig.Field(
128  doc="Model flux errors with background term?",
129  dtype=bool,
130  default=True
131  )
132  psfCandidateName = pexConfig.Field(
133  doc="Name of field with psf candidate flag for propagation",
134  dtype=str,
135  default="calib_psf_candidate"
136  )
137  doSubtractLocalBackground = pexConfig.Field(
138  doc=("Subtract the local background before performing calibration? "
139  "This is only supported for circular aperture calibration fluxes."),
140  dtype=bool,
141  default=False
142  )
143  localBackgroundFluxField = pexConfig.Field(
144  doc="Full name of the local background instFlux field to use.",
145  dtype=str,
146  default='base_LocalBackground_instFlux'
147  )
148  sourceSelector = sourceSelectorRegistry.makeField(
149  doc="How to select sources",
150  default="science"
151  )
152  apertureInnerInstFluxField = pexConfig.Field(
153  doc=("Full name of instFlux field that contains inner aperture "
154  "flux for aperture correction proxy"),
155  dtype=str,
156  default='base_CircularApertureFlux_12_0_instFlux'
157  )
158  apertureOuterInstFluxField = pexConfig.Field(
159  doc=("Full name of instFlux field that contains outer aperture "
160  "flux for aperture correction proxy"),
161  dtype=str,
162  default='base_CircularApertureFlux_17_0_instFlux'
163  )
164  doReferenceMatches = pexConfig.Field(
165  doc="Match reference catalog as additional constraint on calibration",
166  dtype=bool,
167  default=True,
168  )
169  fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
170  target=FgcmLoadReferenceCatalogTask,
171  doc="FGCM reference object loader",
172  )
173  nVisitsPerCheckpoint = pexConfig.Field(
174  doc="Number of visits read between checkpoints",
175  dtype=int,
176  default=500,
177  )
178 
179  def setDefaults(self):
180  sourceSelector = self.sourceSelectorsourceSelector["science"]
181  sourceSelector.setDefaults()
182 
183  sourceSelector.doFlags = True
184  sourceSelector.doUnresolved = True
185  sourceSelector.doSignalToNoise = True
186  sourceSelector.doIsolated = True
187 
188  sourceSelector.signalToNoise.minimum = 10.0
189  sourceSelector.signalToNoise.maximum = 1000.0
190 
191  # FGCM operates on unresolved sources, and this setting is
192  # appropriate for the current base_ClassificationExtendedness
193  sourceSelector.unresolved.maximum = 0.5
194 
195 
196 class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
197  """Subclass of TaskRunner for FgcmBuildStars tasks
198 
199  fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
200  arguments, one of which is the butler (for persistence and mapper data),
201  and a list of dataRefs extracted from the command line. Note that FGCM
202  runs on a large set of dataRefs, and not on single dataRef/tract/patch.
203  This class transforms the process arguments generated by the ArgumentParser
204  into the arguments expected by FgcmBuildStarsTask.run(). This runner does
205  not use any parallelization.
206  """
207  @staticmethod
208  def getTargetList(parsedCmd):
209  """
210  Return a list with one element: a tuple with the butler and
211  list of dataRefs
212  """
213  # we want to combine the butler with any (or no!) dataRefs
214  return [(parsedCmd.butler, parsedCmd.id.refList)]
215 
216  def __call__(self, args):
217  """
218  Parameters
219  ----------
220  args: `tuple` with (butler, dataRefList)
221 
222  Returns
223  -------
224  exitStatus: `list` with `lsst.pipe.base.Struct`
225  exitStatus (0: success; 1: failure)
226  """
227  butler, dataRefList = args
228 
229  task = self.TaskClass(config=self.config, log=self.log)
230 
231  exitStatus = 0
232  if self.doRaise:
233  task.runDataRef(butler, dataRefList)
234  else:
235  try:
236  task.runDataRef(butler, dataRefList)
237  except Exception as e:
238  exitStatus = 1
239  task.log.fatal("Failed: %s" % e)
240  if not isinstance(e, pipeBase.TaskError):
241  traceback.print_exc(file=sys.stderr)
242 
243  task.writeMetadata(butler)
244 
245  # The task does not return any results:
246  return [pipeBase.Struct(exitStatus=exitStatus)]
247 
248  def run(self, parsedCmd):
249  """
250  Run the task, with no multiprocessing
251 
252  Parameters
253  ----------
254  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
255  """
256 
257  resultList = []
258 
259  if self.precall(parsedCmd):
260  targetList = self.getTargetListgetTargetList(parsedCmd)
261  resultList = self(targetList[0])
262 
263  return resultList
264 
265 
266 class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
267  """
268  Base task to build stars for FGCM global calibration
269 
270  Parameters
271  ----------
272  butler : `lsst.daf.persistence.Butler`
273  """
274  def __init__(self, butler=None, initInputs=None, **kwargs):
275  super().__init__(**kwargs)
276 
277  self.makeSubtask("sourceSelector")
278  # Only log warning and fatal errors from the sourceSelector
279  self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
280 
281  # no saving of metadata for now
282  def _getMetadataName(self):
283  return None
284 
285  @pipeBase.timeMethod
286  def runDataRef(self, butler, dataRefs):
287  """
288  Cross-match and make star list for FGCM Input
289 
290  Parameters
291  ----------
292  butler: `lsst.daf.persistence.Butler`
293  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
294  Source data references for the input visits.
295 
296  Raises
297  ------
298  RuntimeErrror: Raised if `config.doReferenceMatches` is set and
299  an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
300  fails if the calibFlux is not a CircularAperture flux.
301  """
302  datasetType = dataRefs[0].butlerSubset.datasetType
303  self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
304 
305  if self.config.doReferenceMatches:
306  self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
307  # Ensure that we have a LUT
308  if not butler.datasetExists('fgcmLookUpTable'):
309  raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
310  # Compute aperture radius if necessary. This is useful to do now before
311  # any heavy lifting has happened (fail early).
312  calibFluxApertureRadius = None
313  if self.config.doSubtractLocalBackground:
314  try:
315  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
316  self.config.instFluxField)
317  except RuntimeError as e:
318  raise RuntimeError("Could not determine aperture radius from %s. "
319  "Cannot use doSubtractLocalBackground." %
320  (self.config.instFluxField)) from e
321 
322  camera = butler.get('camera')
323  groupedDataRefs = self._findAndGroupDataRefs_findAndGroupDataRefs(camera, dataRefs, butler=butler)
324 
325  # Make the visit catalog if necessary
326  # First check if the visit catalog is in the _current_ path
327  # We cannot use Gen2 datasetExists() because that checks all parent
328  # directories as well, which would make recovering from faults
329  # and fgcmcal reruns impossible.
330  visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
331  filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
332  if os.path.exists(filename):
333  # This file exists and we should continue processing
334  inVisitCat = visitCatDataRef.get()
335  if len(inVisitCat) != len(groupedDataRefs):
336  raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
337  "number of visits. Cannot continue.")
338  else:
339  inVisitCat = None
340 
341  visitCat = self.fgcmMakeVisitCatalogfgcmMakeVisitCatalog(camera, groupedDataRefs,
342  visitCatDataRef=visitCatDataRef,
343  inVisitCat=inVisitCat)
344 
345  # Persist the visitCat as a checkpoint file.
346  visitCatDataRef.put(visitCat)
347 
348  starObsDataRef = butler.dataRef('fgcmStarObservations')
349  filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
350  if os.path.exists(filename):
351  inStarObsCat = starObsDataRef.get()
352  else:
353  inStarObsCat = None
354 
355  rad = calibFluxApertureRadius
356  sourceSchemaDataRef = butler.dataRef('src_schema')
357  fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedDataRefs,
358  visitCat,
359  sourceSchemaDataRef,
360  camera,
361  calibFluxApertureRadius=rad,
362  starObsDataRef=starObsDataRef,
363  visitCatDataRef=visitCatDataRef,
364  inStarObsCat=inStarObsCat)
365  visitCatDataRef.put(visitCat)
366  starObsDataRef.put(fgcmStarObservationCat)
367 
368  # Always do the matching.
369  if self.config.doReferenceMatches:
370  lutDataRef = butler.dataRef('fgcmLookUpTable')
371  else:
372  lutDataRef = None
373  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStarsfgcmMatchStars(visitCat,
374  fgcmStarObservationCat,
375  lutDataRef=lutDataRef)
376 
377  # Persist catalogs via the butler
378  butler.put(fgcmStarIdCat, 'fgcmStarIds')
379  butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
380  if fgcmRefCat is not None:
381  butler.put(fgcmRefCat, 'fgcmReferenceStars')
382 
383  @abc.abstractmethod
384  def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
385  """
386  Find and group dataRefs (by visit). For Gen2 usage, set butler, and for
387  Gen3, use calexpDataRefDict
388 
389  Parameters
390  ----------
391  camera : `lsst.afw.cameraGeom.Camera`
392  Camera from the butler.
393  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` or
394  `lsst.daf.butler.DeferredDatasetHandle`
395  Data references for the input visits.
396  butler : `lsst.daf.persistence.Butler`, optional
397  Gen2 butler when used as CommandLineTask
398  calexpDataRefDict : `dict`, optional
399  Dictionary of Gen3 deferred data refs for calexps
400 
401  Returns
402  -------
403  groupedDataRefs : `OrderedDict` [`int`, `list`]
404  Dictionary with sorted visit keys, and `list`s of
405  `lsst.daf.persistence.ButlerDataRef` or
406  `lsst.daf.butler.DeferredDatasetHandle`
407 
408  Raises
409  ------
410  RuntimeError : Raised if neither or both of butler and dataRefDict are set.
411  """
412  raise NotImplementedError("_findAndGroupDataRefs not implemented.")
413 
414  @abc.abstractmethod
415  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
416  sourceSchemaDataRef,
417  camera,
418  calibFluxApertureRadius=None,
419  visitCatDataRef=None,
420  starObsDataRef=None,
421  inStarObsCat=None):
422  """
423  Compile all good star observations from visits in visitCat. Checkpoint files
424  will be stored if both visitCatDataRef and starObsDataRef are not None.
425 
426  Parameters
427  ----------
428  groupedDataRefs: `dict` of `list`s
429  Lists of `~lsst.daf.persistence.ButlerDataRef` or
430  `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
431  visitCat: `~afw.table.BaseCatalog`
432  Catalog with visit data for FGCM
433  sourceSchemaDataRef: `~lsst.daf.persistence.ButlerDataRef` or
434  `~lsst.daf.butler.DeferredDatasetHandle`
435  DataRef for the schema of the src catalogs.
436  camera: `~lsst.afw.cameraGeom.Camera`
437  calibFluxApertureRadius: `float`, optional
438  Aperture radius for calibration flux.
439  visitCatDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
440  Dataref to write visitCat for checkpoints
441  starObsDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
442  Dataref to write the star observation catalog for checkpoints.
443  inStarObsCat: `~afw.table.BaseCatalog`
444  Input observation catalog. If this is incomplete, observations
445  will be appended from when it was cut off.
446 
447  Returns
448  -------
449  fgcmStarObservations: `afw.table.BaseCatalog`
450  Full catalog of good observations.
451 
452  Raises
453  ------
454  RuntimeError: Raised if doSubtractLocalBackground is True and
455  calibFluxApertureRadius is not set.
456  """
457  raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
458 
459  def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
460  visitCatDataRef=None, inVisitCat=None):
461  """
462  Make a visit catalog with all the keys from each visit
463 
464  Parameters
465  ----------
466  camera: `lsst.afw.cameraGeom.Camera`
467  Camera from the butler
468  groupedDataRefs: `dict`
469  Dictionary with visit keys, and `list`s of
470  `lsst.daf.persistence.ButlerDataRef`
471  bkgDataRefDict: `dict`, optional
472  Dictionary of gen3 dataRefHandles for background info.
473  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
474  Dataref to write visitCat for checkpoints
475  inVisitCat: `afw.table.BaseCatalog`, optional
476  Input (possibly incomplete) visit catalog
477 
478  Returns
479  -------
480  visitCat: `afw.table.BaseCatalog`
481  """
482 
483  self.log.info("Assembling visitCatalog from %d %ss" %
484  (len(groupedDataRefs), self.config.visitDataRefName))
485 
486  nCcd = len(camera)
487 
488  if inVisitCat is None:
489  schema = self._makeFgcmVisitSchema_makeFgcmVisitSchema(nCcd)
490 
491  visitCat = afwTable.BaseCatalog(schema)
492  visitCat.reserve(len(groupedDataRefs))
493  visitCat.resize(len(groupedDataRefs))
494 
495  visitCat['visit'] = list(groupedDataRefs.keys())
496  visitCat['used'] = 0
497  visitCat['sources_read'] = False
498  else:
499  visitCat = inVisitCat
500 
501  # No matter what, fill the catalog. This will check if it was
502  # already read.
503  self._fillVisitCatalog_fillVisitCatalog(visitCat, groupedDataRefs,
504  bkgDataRefDict=bkgDataRefDict,
505  visitCatDataRef=visitCatDataRef)
506 
507  return visitCat
508 
509  def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
510  visitCatDataRef=None):
511  """
512  Fill the visit catalog with visit metadata
513 
514  Parameters
515  ----------
516  visitCat: `afw.table.BaseCatalog`
517  Catalog with schema from _makeFgcmVisitSchema()
518  groupedDataRefs: `dict`
519  Dictionary with visit keys, and `list`s of
520  `lsst.daf.persistence.ButlerDataRef`
521  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
522  Dataref to write visitCat for checkpoints
523  bkgDataRefDict: `dict`, optional
524  Dictionary of gen3 dataRefHandles for background info. FIXME
525  """
526  bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
527 
528  for i, visit in enumerate(groupedDataRefs):
529  # We don't use the bypasses since we need the psf info which does
530  # not have a bypass
531  # TODO: When DM-15500 is implemented in the Gen3 Butler, this
532  # can be fixed
533 
534  # Do not read those that have already been read
535  if visitCat['used'][i]:
536  continue
537 
538  if (i % self.config.nVisitsPerCheckpoint) == 0:
539  self.log.info("Retrieving metadata for %s %d (%d/%d)" %
540  (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
541  # Save checkpoint if desired
542  if visitCatDataRef is not None:
543  visitCatDataRef.put(visitCat)
544 
545  # Note that the reference ccd is first in the list (if available).
546 
547  # The first dataRef in the group will be the reference ccd (if available)
548  dataRef = groupedDataRefs[visit][0]
549  if isinstance(dataRef, dafPersist.ButlerDataRef):
550  exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
551  visitInfo = exp.getInfo().getVisitInfo()
552  f = exp.getFilter()
553  psf = exp.getPsf()
554  else:
555  visitInfo = dataRef.get(component='visitInfo')
556  f = dataRef.get(component='filter')
557  psf = dataRef.get(component='psf')
558 
559  rec = visitCat[i]
560  rec['visit'] = visit
561  rec['filtername'] = f.getName()
562  # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
563  # much easier to get the wcs's necessary to recompute the pointing
564  # ra/dec at the center of the camera.
565  radec = visitInfo.getBoresightRaDec()
566  rec['telra'] = radec.getRa().asDegrees()
567  rec['teldec'] = radec.getDec().asDegrees()
568  rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
569  rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
570  rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
571  rec['exptime'] = visitInfo.getExposureTime()
572  # convert from Pa to millibar
573  # Note that I don't know if this unit will need to be per-camera config
574  rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
575  # Flag to signify if this is a "deep" field. Not currently used
576  rec['deepFlag'] = 0
577  # Relative flat scaling (1.0 means no relative scaling)
578  rec['scaling'][:] = 1.0
579  # Median delta aperture, to be measured from stars
580  rec['deltaAper'] = 0.0
581 
582  rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
583 
584  if self.config.doModelErrorsWithBackground:
585  foundBkg = False
586  if isinstance(dataRef, dafPersist.ButlerDataRef):
587  det = dataRef.dataId[self.config.ccdDataRefName]
588  if dataRef.datasetExists(datasetType='calexpBackground'):
589  bgList = dataRef.get(datasetType='calexpBackground')
590  foundBkg = True
591  else:
592  det = dataRef.dataId.byName()['detector']
593  try:
594  bkgRef = bkgDataRefDict[(visit, det)]
595  bgList = bkgRef.get()
596  foundBkg = True
597  except KeyError:
598  pass
599 
600  if foundBkg:
601  bgStats = (bg[0].getStatsImage().getImage().array
602  for bg in bgList)
603  rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
604  else:
605  self.log.warn('Sky background not found for visit %d / ccd %d' %
606  (visit, det))
607  rec['skyBackground'] = -1.0
608  else:
609  rec['skyBackground'] = -1.0
610 
611  rec['used'] = 1
612 
613  def _makeSourceMapper(self, sourceSchema):
614  """
615  Make a schema mapper for fgcm sources
616 
617  Parameters
618  ----------
619  sourceSchema: `afwTable.Schema`
620  Default source schema from the butler
621 
622  Returns
623  -------
624  sourceMapper: `afwTable.schemaMapper`
625  Mapper to the FGCM source schema
626  """
627 
628  # create a mapper to the preferred output
629  sourceMapper = afwTable.SchemaMapper(sourceSchema)
630 
631  # map to ra/dec
632  sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
633  sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
634  sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
635  sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
636  # Add the mapping if the field exists in the input catalog.
637  # If the field does not exist, simply add it (set to False).
638  # This field is not required for calibration, but is useful
639  # to collate if available.
640  try:
641  sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
642  'psf_candidate')
643  except LookupError:
644  sourceMapper.editOutputSchema().addField(
645  "psf_candidate", type='Flag',
646  doc=("Flag set if the source was a candidate for PSF determination, "
647  "as determined by the star selector."))
648 
649  # and add the fields we want
650  sourceMapper.editOutputSchema().addField(
651  "visit", type=np.int32, doc="Visit number")
652  sourceMapper.editOutputSchema().addField(
653  "ccd", type=np.int32, doc="CCD number")
654  sourceMapper.editOutputSchema().addField(
655  "instMag", type=np.float32, doc="Instrumental magnitude")
656  sourceMapper.editOutputSchema().addField(
657  "instMagErr", type=np.float32, doc="Instrumental magnitude error")
658  sourceMapper.editOutputSchema().addField(
659  "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
660  sourceMapper.editOutputSchema().addField(
661  "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
662 
663  return sourceMapper
664 
665  def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
666  """
667  Use FGCM code to match observations into unique stars.
668 
669  Parameters
670  ----------
671  visitCat: `afw.table.BaseCatalog`
672  Catalog with visit data for fgcm
673  obsCat: `afw.table.BaseCatalog`
674  Full catalog of star observations for fgcm
675  lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
676  `lsst.daf.butler.DeferredDatasetHandle`, optional
677  Data reference to fgcm look-up table (used if matching reference stars).
678 
679  Returns
680  -------
681  fgcmStarIdCat: `afw.table.BaseCatalog`
682  Catalog of unique star identifiers and index keys
683  fgcmStarIndicesCat: `afwTable.BaseCatalog`
684  Catalog of unique star indices
685  fgcmRefCat: `afw.table.BaseCatalog`
686  Catalog of matched reference stars.
687  Will be None if `config.doReferenceMatches` is False.
688  """
689  # get filter names into a numpy array...
690  # This is the type that is expected by the fgcm code
691  visitFilterNames = np.zeros(len(visitCat), dtype='a10')
692  for i in range(len(visitCat)):
693  visitFilterNames[i] = visitCat[i]['filtername']
694 
695  # match to put filterNames with observations
696  visitIndex = np.searchsorted(visitCat['visit'],
697  obsCat['visit'])
698 
699  obsFilterNames = visitFilterNames[visitIndex]
700 
701  if self.config.doReferenceMatches:
702  # Get the reference filter names, using the LUT
703  lutCat = lutDataRef.get()
704 
705  stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
706  zip(lutCat[0]['filterNames'].split(','),
707  lutCat[0]['stdFilterNames'].split(','))}
708  stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
709  zip(lutCat[0]['stdFilterNames'].split(','),
710  lutCat[0]['lambdaStdFilter'])}
711 
712  del lutCat
713 
714  referenceFilterNames = self._getReferenceFilterNames_getReferenceFilterNames(visitCat,
715  stdFilterDict,
716  stdLambdaDict)
717  self.log.info("Using the following reference filters: %s" %
718  (', '.join(referenceFilterNames)))
719 
720  else:
721  # This should be an empty list
722  referenceFilterNames = []
723 
724  # make the fgcm starConfig dict
725 
726  starConfig = {'logger': self.log,
727  'filterToBand': self.config.filterMap,
728  'requiredBands': self.config.requiredBands,
729  'minPerBand': self.config.minPerBand,
730  'matchRadius': self.config.matchRadius,
731  'isolationRadius': self.config.isolationRadius,
732  'matchNSide': self.config.matchNside,
733  'coarseNSide': self.config.coarseNside,
734  'densNSide': self.config.densityCutNside,
735  'densMaxPerPixel': self.config.densityCutMaxPerPixel,
736  'primaryBands': self.config.primaryBands,
737  'referenceFilterNames': referenceFilterNames}
738 
739  # initialize the FgcmMakeStars object
740  fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
741 
742  # make the primary stars
743  # note that the ra/dec native Angle format is radians
744  # We determine the conversion from the native units (typically
745  # radians) to degrees for the first observation. This allows us
746  # to treate ra/dec as numpy arrays rather than Angles, which would
747  # be approximately 600x slower.
748  conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
749  fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
750  obsCat['dec'] * conv,
751  filterNameArray=obsFilterNames,
752  bandSelected=False)
753 
754  # and match all the stars
755  fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
756  obsCat['dec'] * conv,
757  obsFilterNames)
758 
759  if self.config.doReferenceMatches:
760  fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
761 
762  # now persist
763 
764  objSchema = self._makeFgcmObjSchema_makeFgcmObjSchema()
765 
766  # make catalog and records
767  fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
768  fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
769  for i in range(fgcmMakeStars.objIndexCat.size):
770  fgcmStarIdCat.addNew()
771 
772  # fill the catalog
773  fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
774  fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
775  fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
776  fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
777  fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
778 
779  obsSchema = self._makeFgcmObsSchema_makeFgcmObsSchema()
780 
781  fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
782  fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
783  for i in range(fgcmMakeStars.obsIndexCat.size):
784  fgcmStarIndicesCat.addNew()
785 
786  fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
787 
788  if self.config.doReferenceMatches:
789  refSchema = self._makeFgcmRefSchema_makeFgcmRefSchema(len(referenceFilterNames))
790 
791  fgcmRefCat = afwTable.BaseCatalog(refSchema)
792  fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
793 
794  for i in range(fgcmMakeStars.referenceCat.size):
795  fgcmRefCat.addNew()
796 
797  fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
798  fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
799  fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
800 
801  md = PropertyList()
802  md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
803  md.set("FILTERNAMES", referenceFilterNames)
804  fgcmRefCat.setMetadata(md)
805 
806  else:
807  fgcmRefCat = None
808 
809  return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
810 
811  def _makeFgcmVisitSchema(self, nCcd):
812  """
813  Make a schema for an fgcmVisitCatalog
814 
815  Parameters
816  ----------
817  nCcd: `int`
818  Number of CCDs in the camera
819 
820  Returns
821  -------
822  schema: `afwTable.Schema`
823  """
824 
825  schema = afwTable.Schema()
826  schema.addField('visit', type=np.int32, doc="Visit number")
827  # Note that the FGCM code currently handles filternames up to 2 characters long
828  schema.addField('filtername', type=str, size=10, doc="Filter name")
829  schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
830  schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
831  schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
832  schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
833  schema.addField('mjd', type=np.float64, doc="MJD of visit")
834  schema.addField('exptime', type=np.float32, doc="Exposure time")
835  schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
836  schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
837  schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
838  schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
839  # the following field is not used yet
840  schema.addField('deepFlag', type=np.int32, doc="Deep observation")
841  schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
842  size=nCcd)
843  schema.addField('used', type=np.int32, doc="This visit has been ingested.")
844  schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
845 
846  return schema
847 
848  def _makeFgcmObjSchema(self):
849  """
850  Make a schema for the objIndexCat from fgcmMakeStars
851 
852  Returns
853  -------
854  schema: `afwTable.Schema`
855  """
856 
857  objSchema = afwTable.Schema()
858  objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
859  # Will investigate making these angles...
860  objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
861  objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
862  objSchema.addField('obsArrIndex', type=np.int32,
863  doc='Index in obsIndexTable for first observation')
864  objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
865 
866  return objSchema
867 
868  def _makeFgcmObsSchema(self):
869  """
870  Make a schema for the obsIndexCat from fgcmMakeStars
871 
872  Returns
873  -------
874  schema: `afwTable.Schema`
875  """
876 
877  obsSchema = afwTable.Schema()
878  obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
879 
880  return obsSchema
881 
882  def _makeFgcmRefSchema(self, nReferenceBands):
883  """
884  Make a schema for the referenceCat from fgcmMakeStars
885 
886  Parameters
887  ----------
888  nReferenceBands: `int`
889  Number of reference bands
890 
891  Returns
892  -------
893  schema: `afwTable.Schema`
894  """
895 
896  refSchema = afwTable.Schema()
897  refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
898  refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
899  size=nReferenceBands)
900  refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
901  size=nReferenceBands)
902 
903  return refSchema
904 
905  def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
906  """
907  Get the reference filter names, in wavelength order, from the visitCat and
908  information from the look-up-table.
909 
910  Parameters
911  ----------
912  visitCat: `afw.table.BaseCatalog`
913  Catalog with visit data for FGCM
914  stdFilterDict: `dict`
915  Mapping of filterName to stdFilterName from LUT
916  stdLambdaDict: `dict`
917  Mapping of stdFilterName to stdLambda from LUT
918 
919  Returns
920  -------
921  referenceFilterNames: `list`
922  Wavelength-ordered list of reference filter names
923  """
924 
925  # Find the unique list of filter names in visitCat
926  filterNames = np.unique(visitCat.asAstropy()['filtername'])
927 
928  # Find the unique list of "standard" filters
929  stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
930 
931  # And sort these by wavelength
932  referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
933 
934  return referenceFilterNames
def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None)
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None)
def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None)
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None, inVisitCat=None)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, sourceSchemaDataRef, camera, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def __init__(self, butler=None, initInputs=None, **kwargs)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:799