lsst.fgcmcal  21.0.0-10-g8d1d15d+673cafdbcb
fgcmBuildStarsBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for BuildStars using src tables or sourceTable_visit tables.
22 """
23 
24 import os
25 import sys
26 import traceback
27 import abc
28 
29 import numpy as np
30 
31 import lsst.daf.persistence as dafPersist
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 import lsst.afw.table as afwTable
35 import lsst.geom as geom
36 from lsst.daf.base import PropertyList
37 from lsst.daf.base.dateTime import DateTime
38 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
39 
40 from .utilities import computeApertureRadiusFromDataRef
41 from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
42 
43 import fgcm
44 
45 REFSTARS_FORMAT_VERSION = 1
46 
47 __all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
48 
49 
50 class FgcmBuildStarsConfigBase(pexConfig.Config):
51  """Base config for FgcmBuildStars tasks"""
52 
53  instFluxField = pexConfig.Field(
54  doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55  "The associated flag will be implicitly included in badFlags"),
56  dtype=str,
57  default='slot_CalibFlux_instFlux',
58  )
59  minPerBand = pexConfig.Field(
60  doc="Minimum observations per band",
61  dtype=int,
62  default=2,
63  )
64  matchRadius = pexConfig.Field(
65  doc="Match radius (arcseconds)",
66  dtype=float,
67  default=1.0,
68  )
69  isolationRadius = pexConfig.Field(
70  doc="Isolation radius (arcseconds)",
71  dtype=float,
72  default=2.0,
73  )
74  densityCutNside = pexConfig.Field(
75  doc="Density cut healpix nside",
76  dtype=int,
77  default=128,
78  )
79  densityCutMaxPerPixel = pexConfig.Field(
80  doc="Density cut number of stars per pixel",
81  dtype=int,
82  default=1000,
83  )
84  matchNside = pexConfig.Field(
85  doc="Healpix Nside for matching",
86  dtype=int,
87  default=4096,
88  )
89  coarseNside = pexConfig.Field(
90  doc="Healpix coarse Nside for partitioning matches",
91  dtype=int,
92  default=8,
93  )
94  filterMap = pexConfig.DictField(
95  doc="Mapping from 'filterName' to band.",
96  keytype=str,
97  itemtype=str,
98  default={},
99  deprecated=("This field is no longer used, and has been deprecated by "
100  "DM-28088. It will be removed after v22. Use "
101  "physicalFilterMap instead.")
102  )
103  # The following config will not be necessary after Gen2 retirement.
104  # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
105  # which is easiest to access in the config file.
106  physicalFilterMap = pexConfig.DictField(
107  doc="Mapping from 'physicalFilter' to band.",
108  keytype=str,
109  itemtype=str,
110  default={},
111  )
112  requiredBands = pexConfig.ListField(
113  doc="Bands required for each star",
114  dtype=str,
115  default=(),
116  )
117  primaryBands = pexConfig.ListField(
118  doc=("Bands for 'primary' star matches. "
119  "A star must be observed in one of these bands to be considered "
120  "as a calibration star."),
121  dtype=str,
122  default=None
123  )
124  visitDataRefName = pexConfig.Field(
125  doc="dataRef name for the 'visit' field, usually 'visit'.",
126  dtype=str,
127  default="visit"
128  )
129  ccdDataRefName = pexConfig.Field(
130  doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
131  dtype=str,
132  default="ccd"
133  )
134  doApplyWcsJacobian = pexConfig.Field(
135  doc="Apply the jacobian of the WCS to the star observations prior to fit?",
136  dtype=bool,
137  default=True
138  )
139  doModelErrorsWithBackground = pexConfig.Field(
140  doc="Model flux errors with background term?",
141  dtype=bool,
142  default=True
143  )
144  psfCandidateName = pexConfig.Field(
145  doc="Name of field with psf candidate flag for propagation",
146  dtype=str,
147  default="calib_psf_candidate"
148  )
149  doSubtractLocalBackground = pexConfig.Field(
150  doc=("Subtract the local background before performing calibration? "
151  "This is only supported for circular aperture calibration fluxes."),
152  dtype=bool,
153  default=False
154  )
155  localBackgroundFluxField = pexConfig.Field(
156  doc="Full name of the local background instFlux field to use.",
157  dtype=str,
158  default='base_LocalBackground_instFlux'
159  )
160  sourceSelector = sourceSelectorRegistry.makeField(
161  doc="How to select sources",
162  default="science"
163  )
164  apertureInnerInstFluxField = pexConfig.Field(
165  doc=("Full name of instFlux field that contains inner aperture "
166  "flux for aperture correction proxy"),
167  dtype=str,
168  default='base_CircularApertureFlux_12_0_instFlux'
169  )
170  apertureOuterInstFluxField = pexConfig.Field(
171  doc=("Full name of instFlux field that contains outer aperture "
172  "flux for aperture correction proxy"),
173  dtype=str,
174  default='base_CircularApertureFlux_17_0_instFlux'
175  )
176  doReferenceMatches = pexConfig.Field(
177  doc="Match reference catalog as additional constraint on calibration",
178  dtype=bool,
179  default=True,
180  )
181  fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
182  target=FgcmLoadReferenceCatalogTask,
183  doc="FGCM reference object loader",
184  )
185  nVisitsPerCheckpoint = pexConfig.Field(
186  doc="Number of visits read between checkpoints",
187  dtype=int,
188  default=500,
189  )
190 
191  def setDefaults(self):
192  sourceSelector = self.sourceSelectorsourceSelector["science"]
193  sourceSelector.setDefaults()
194 
195  sourceSelector.doFlags = True
196  sourceSelector.doUnresolved = True
197  sourceSelector.doSignalToNoise = True
198  sourceSelector.doIsolated = True
199 
200  sourceSelector.signalToNoise.minimum = 10.0
201  sourceSelector.signalToNoise.maximum = 1000.0
202 
203  # FGCM operates on unresolved sources, and this setting is
204  # appropriate for the current base_ClassificationExtendedness
205  sourceSelector.unresolved.maximum = 0.5
206 
207 
208 class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
209  """Subclass of TaskRunner for FgcmBuildStars tasks
210 
211  fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
212  arguments, one of which is the butler (for persistence and mapper data),
213  and a list of dataRefs extracted from the command line. Note that FGCM
214  runs on a large set of dataRefs, and not on single dataRef/tract/patch.
215  This class transforms the process arguments generated by the ArgumentParser
216  into the arguments expected by FgcmBuildStarsTask.run(). This runner does
217  not use any parallelization.
218  """
219  @staticmethod
220  def getTargetList(parsedCmd):
221  """
222  Return a list with one element: a tuple with the butler and
223  list of dataRefs
224  """
225  # we want to combine the butler with any (or no!) dataRefs
226  return [(parsedCmd.butler, parsedCmd.id.refList)]
227 
228  def __call__(self, args):
229  """
230  Parameters
231  ----------
232  args: `tuple` with (butler, dataRefList)
233 
234  Returns
235  -------
236  exitStatus: `list` with `lsst.pipe.base.Struct`
237  exitStatus (0: success; 1: failure)
238  """
239  butler, dataRefList = args
240 
241  task = self.TaskClass(config=self.config, log=self.log)
242 
243  exitStatus = 0
244  if self.doRaise:
245  task.runDataRef(butler, dataRefList)
246  else:
247  try:
248  task.runDataRef(butler, dataRefList)
249  except Exception as e:
250  exitStatus = 1
251  task.log.fatal("Failed: %s" % e)
252  if not isinstance(e, pipeBase.TaskError):
253  traceback.print_exc(file=sys.stderr)
254 
255  task.writeMetadata(butler)
256 
257  # The task does not return any results:
258  return [pipeBase.Struct(exitStatus=exitStatus)]
259 
260  def run(self, parsedCmd):
261  """
262  Run the task, with no multiprocessing
263 
264  Parameters
265  ----------
266  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
267  """
268 
269  resultList = []
270 
271  if self.precall(parsedCmd):
272  targetList = self.getTargetListgetTargetList(parsedCmd)
273  resultList = self(targetList[0])
274 
275  return resultList
276 
277 
278 class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
279  """
280  Base task to build stars for FGCM global calibration
281 
282  Parameters
283  ----------
284  butler : `lsst.daf.persistence.Butler`
285  """
286  def __init__(self, butler=None, initInputs=None, **kwargs):
287  super().__init__(**kwargs)
288 
289  self.makeSubtask("sourceSelector")
290  # Only log warning and fatal errors from the sourceSelector
291  self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
292 
293  # no saving of metadata for now
294  def _getMetadataName(self):
295  return None
296 
297  @pipeBase.timeMethod
298  def runDataRef(self, butler, dataRefs):
299  """
300  Cross-match and make star list for FGCM Input
301 
302  Parameters
303  ----------
304  butler: `lsst.daf.persistence.Butler`
305  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
306  Source data references for the input visits.
307 
308  Raises
309  ------
310  RuntimeErrror: Raised if `config.doReferenceMatches` is set and
311  an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
312  fails if the calibFlux is not a CircularAperture flux.
313  """
314  datasetType = dataRefs[0].butlerSubset.datasetType
315  self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
316 
317  if self.config.doReferenceMatches:
318  self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
319  # Ensure that we have a LUT
320  if not butler.datasetExists('fgcmLookUpTable'):
321  raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
322  # Compute aperture radius if necessary. This is useful to do now before
323  # any heavy lifting has happened (fail early).
324  calibFluxApertureRadius = None
325  if self.config.doSubtractLocalBackground:
326  try:
327  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
328  self.config.instFluxField)
329  except RuntimeError as e:
330  raise RuntimeError("Could not determine aperture radius from %s. "
331  "Cannot use doSubtractLocalBackground." %
332  (self.config.instFluxField)) from e
333 
334  camera = butler.get('camera')
335  groupedDataRefs = self._findAndGroupDataRefsGen2_findAndGroupDataRefsGen2(butler, camera, dataRefs)
336 
337  # Make the visit catalog if necessary
338  # First check if the visit catalog is in the _current_ path
339  # We cannot use Gen2 datasetExists() because that checks all parent
340  # directories as well, which would make recovering from faults
341  # and fgcmcal reruns impossible.
342  visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
343  filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
344  if os.path.exists(filename):
345  # This file exists and we should continue processing
346  inVisitCat = visitCatDataRef.get()
347  if len(inVisitCat) != len(groupedDataRefs):
348  raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
349  "number of visits. Cannot continue.")
350  else:
351  inVisitCat = None
352 
353  visitCat = self.fgcmMakeVisitCatalogfgcmMakeVisitCatalog(camera, groupedDataRefs,
354  visitCatDataRef=visitCatDataRef,
355  inVisitCat=inVisitCat)
356 
357  # Persist the visitCat as a checkpoint file.
358  visitCatDataRef.put(visitCat)
359 
360  starObsDataRef = butler.dataRef('fgcmStarObservations')
361  filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
362  if os.path.exists(filename):
363  inStarObsCat = starObsDataRef.get()
364  else:
365  inStarObsCat = None
366 
367  rad = calibFluxApertureRadius
368  sourceSchemaDataRef = butler.dataRef('src_schema')
369  fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedDataRefs,
370  visitCat,
371  sourceSchemaDataRef,
372  camera,
373  calibFluxApertureRadius=rad,
374  starObsDataRef=starObsDataRef,
375  visitCatDataRef=visitCatDataRef,
376  inStarObsCat=inStarObsCat)
377  visitCatDataRef.put(visitCat)
378  starObsDataRef.put(fgcmStarObservationCat)
379 
380  # Always do the matching.
381  if self.config.doReferenceMatches:
382  lutDataRef = butler.dataRef('fgcmLookUpTable')
383  else:
384  lutDataRef = None
385  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStarsfgcmMatchStars(visitCat,
386  fgcmStarObservationCat,
387  lutDataRef=lutDataRef)
388 
389  # Persist catalogs via the butler
390  butler.put(fgcmStarIdCat, 'fgcmStarIds')
391  butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
392  if fgcmRefCat is not None:
393  butler.put(fgcmRefCat, 'fgcmReferenceStars')
394 
395  @abc.abstractmethod
396  def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
397  """
398  Find and group dataRefs (by visit); Gen2 only.
399 
400  Parameters
401  ----------
402  butler : `lsst.daf.persistence.Butler`
403  Gen2 butler.
404  camera : `lsst.afw.cameraGeom.Camera`
405  Camera from the butler.
406  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
407  Data references for the input visits.
408 
409  Returns
410  -------
411  groupedDataRefs : `dict` [`int`, `list`]
412  Dictionary with sorted visit keys, and `list`s of
413  `lsst.daf.persistence.ButlerDataRef`
414  """
415  raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.")
416 
417  @abc.abstractmethod
418  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
419  sourceSchemaDataRef,
420  camera,
421  calibFluxApertureRadius=None,
422  visitCatDataRef=None,
423  starObsDataRef=None,
424  inStarObsCat=None):
425  """
426  Compile all good star observations from visits in visitCat. Checkpoint files
427  will be stored if both visitCatDataRef and starObsDataRef are not None.
428 
429  Parameters
430  ----------
431  groupedDataRefs: `dict` of `list`s
432  Lists of `~lsst.daf.persistence.ButlerDataRef` or
433  `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
434  visitCat: `~afw.table.BaseCatalog`
435  Catalog with visit data for FGCM
436  sourceSchemaDataRef: `~lsst.daf.persistence.ButlerDataRef` or
437  `~lsst.daf.butler.DeferredDatasetHandle`
438  DataRef for the schema of the src catalogs.
439  camera: `~lsst.afw.cameraGeom.Camera`
440  calibFluxApertureRadius: `float`, optional
441  Aperture radius for calibration flux.
442  visitCatDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
443  Dataref to write visitCat for checkpoints
444  starObsDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
445  Dataref to write the star observation catalog for checkpoints.
446  inStarObsCat: `~afw.table.BaseCatalog`
447  Input observation catalog. If this is incomplete, observations
448  will be appended from when it was cut off.
449 
450  Returns
451  -------
452  fgcmStarObservations: `afw.table.BaseCatalog`
453  Full catalog of good observations.
454 
455  Raises
456  ------
457  RuntimeError: Raised if doSubtractLocalBackground is True and
458  calibFluxApertureRadius is not set.
459  """
460  raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
461 
462  def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
463  visitCatDataRef=None, inVisitCat=None):
464  """
465  Make a visit catalog with all the keys from each visit
466 
467  Parameters
468  ----------
469  camera: `lsst.afw.cameraGeom.Camera`
470  Camera from the butler
471  groupedDataRefs: `dict`
472  Dictionary with visit keys, and `list`s of
473  `lsst.daf.persistence.ButlerDataRef`
474  bkgDataRefDict: `dict`, optional
475  Dictionary of gen3 dataRefHandles for background info.
476  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
477  Dataref to write visitCat for checkpoints
478  inVisitCat: `afw.table.BaseCatalog`, optional
479  Input (possibly incomplete) visit catalog
480 
481  Returns
482  -------
483  visitCat: `afw.table.BaseCatalog`
484  """
485 
486  self.log.info("Assembling visitCatalog from %d %ss" %
487  (len(groupedDataRefs), self.config.visitDataRefName))
488 
489  nCcd = len(camera)
490 
491  if inVisitCat is None:
492  schema = self._makeFgcmVisitSchema_makeFgcmVisitSchema(nCcd)
493 
494  visitCat = afwTable.BaseCatalog(schema)
495  visitCat.reserve(len(groupedDataRefs))
496  visitCat.resize(len(groupedDataRefs))
497 
498  visitCat['visit'] = list(groupedDataRefs.keys())
499  visitCat['used'] = 0
500  visitCat['sources_read'] = False
501  else:
502  visitCat = inVisitCat
503 
504  # No matter what, fill the catalog. This will check if it was
505  # already read.
506  self._fillVisitCatalog_fillVisitCatalog(visitCat, groupedDataRefs,
507  bkgDataRefDict=bkgDataRefDict,
508  visitCatDataRef=visitCatDataRef)
509 
510  return visitCat
511 
512  def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
513  visitCatDataRef=None):
514  """
515  Fill the visit catalog with visit metadata
516 
517  Parameters
518  ----------
519  visitCat: `afw.table.BaseCatalog`
520  Catalog with schema from _makeFgcmVisitSchema()
521  groupedDataRefs: `dict`
522  Dictionary with visit keys, and `list`s of
523  `lsst.daf.persistence.ButlerDataRef`
524  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
525  Dataref to write visitCat for checkpoints
526  bkgDataRefDict: `dict`, optional
527  Dictionary of gen3 dataRefHandles for background info. FIXME
528  """
529  bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
530 
531  for i, visit in enumerate(groupedDataRefs):
532  # We don't use the bypasses since we need the psf info which does
533  # not have a bypass
534  # TODO: When DM-15500 is implemented in the Gen3 Butler, this
535  # can be fixed
536 
537  # Do not read those that have already been read
538  if visitCat['used'][i]:
539  continue
540 
541  if (i % self.config.nVisitsPerCheckpoint) == 0:
542  self.log.info("Retrieving metadata for %s %d (%d/%d)" %
543  (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
544  # Save checkpoint if desired
545  if visitCatDataRef is not None:
546  visitCatDataRef.put(visitCat)
547 
548  dataRef = groupedDataRefs[visit][0]
549  if isinstance(dataRef, dafPersist.ButlerDataRef):
550  # Gen2: calexp dataRef
551  # The first dataRef in the group will be the reference ccd (if available)
552  exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
553  visitInfo = exp.getInfo().getVisitInfo()
554  label = dataRef.get(datasetType='calexp_filterLabel')
555  physicalFilter = label.physicalLabel
556  psf = exp.getPsf()
557  psfSigma = psf.computeShape().getDeterminantRadius()
558  else:
559  # Gen3: use the visitSummary dataRef
560  summary = dataRef.get()
561 
562  summaryRow = summary.find(self.config.referenceCCD)
563  if summaryRow is None:
564  # Take the first available ccd if reference isn't available
565  summaryRow = summary[0]
566 
567  visitInfo = summaryRow.getVisitInfo()
568  physicalFilter = summaryRow['physical_filter']
569  # Compute the median psf sigma if possible
570  goodSigma, = np.where(summary['psfSigma'] > 0)
571  if goodSigma.size > 2:
572  psfSigma = np.median(summary['psfSigma'][goodSigma])
573  elif goodSigma > 0:
574  psfSigma = np.mean(summary['psfSigma'][goodSigma])
575  else:
576  psfSigma = 0.0
577 
578  rec = visitCat[i]
579  rec['visit'] = visit
580  rec['physicalFilter'] = physicalFilter
581  # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
582  # much easier to get the wcs's necessary to recompute the pointing
583  # ra/dec at the center of the camera.
584  radec = visitInfo.getBoresightRaDec()
585  rec['telra'] = radec.getRa().asDegrees()
586  rec['teldec'] = radec.getDec().asDegrees()
587  rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
588  rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
589  rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
590  rec['exptime'] = visitInfo.getExposureTime()
591  # convert from Pa to millibar
592  # Note that I don't know if this unit will need to be per-camera config
593  rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
594  # Flag to signify if this is a "deep" field. Not currently used
595  rec['deepFlag'] = 0
596  # Relative flat scaling (1.0 means no relative scaling)
597  rec['scaling'][:] = 1.0
598  # Median delta aperture, to be measured from stars
599  rec['deltaAper'] = 0.0
600  rec['psfSigma'] = psfSigma
601 
602  if self.config.doModelErrorsWithBackground:
603  foundBkg = False
604  if isinstance(dataRef, dafPersist.ButlerDataRef):
605  det = dataRef.dataId[self.config.ccdDataRefName]
606  if dataRef.datasetExists(datasetType='calexpBackground'):
607  bgList = dataRef.get(datasetType='calexpBackground')
608  foundBkg = True
609  else:
610  det = dataRef.dataId['detector']
611  try:
612  bkgRef = bkgDataRefDict[(visit, det)]
613  bgList = bkgRef.get()
614  foundBkg = True
615  except KeyError:
616  pass
617 
618  if foundBkg:
619  bgStats = (bg[0].getStatsImage().getImage().array
620  for bg in bgList)
621  rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
622  else:
623  self.log.warn('Sky background not found for visit %d / ccd %d' %
624  (visit, det))
625  rec['skyBackground'] = -1.0
626  else:
627  rec['skyBackground'] = -1.0
628 
629  rec['used'] = 1
630 
631  def _makeSourceMapper(self, sourceSchema):
632  """
633  Make a schema mapper for fgcm sources
634 
635  Parameters
636  ----------
637  sourceSchema: `afwTable.Schema`
638  Default source schema from the butler
639 
640  Returns
641  -------
642  sourceMapper: `afwTable.schemaMapper`
643  Mapper to the FGCM source schema
644  """
645 
646  # create a mapper to the preferred output
647  sourceMapper = afwTable.SchemaMapper(sourceSchema)
648 
649  # map to ra/dec
650  sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
651  sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
652  sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
653  sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
654  # Add the mapping if the field exists in the input catalog.
655  # If the field does not exist, simply add it (set to False).
656  # This field is not required for calibration, but is useful
657  # to collate if available.
658  try:
659  sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
660  'psf_candidate')
661  except LookupError:
662  sourceMapper.editOutputSchema().addField(
663  "psf_candidate", type='Flag',
664  doc=("Flag set if the source was a candidate for PSF determination, "
665  "as determined by the star selector."))
666 
667  # and add the fields we want
668  sourceMapper.editOutputSchema().addField(
669  "visit", type=np.int32, doc="Visit number")
670  sourceMapper.editOutputSchema().addField(
671  "ccd", type=np.int32, doc="CCD number")
672  sourceMapper.editOutputSchema().addField(
673  "instMag", type=np.float32, doc="Instrumental magnitude")
674  sourceMapper.editOutputSchema().addField(
675  "instMagErr", type=np.float32, doc="Instrumental magnitude error")
676  sourceMapper.editOutputSchema().addField(
677  "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
678  sourceMapper.editOutputSchema().addField(
679  "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
680 
681  return sourceMapper
682 
683  def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
684  """
685  Use FGCM code to match observations into unique stars.
686 
687  Parameters
688  ----------
689  visitCat: `afw.table.BaseCatalog`
690  Catalog with visit data for fgcm
691  obsCat: `afw.table.BaseCatalog`
692  Full catalog of star observations for fgcm
693  lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
694  `lsst.daf.butler.DeferredDatasetHandle`, optional
695  Data reference to fgcm look-up table (used if matching reference stars).
696 
697  Returns
698  -------
699  fgcmStarIdCat: `afw.table.BaseCatalog`
700  Catalog of unique star identifiers and index keys
701  fgcmStarIndicesCat: `afwTable.BaseCatalog`
702  Catalog of unique star indices
703  fgcmRefCat: `afw.table.BaseCatalog`
704  Catalog of matched reference stars.
705  Will be None if `config.doReferenceMatches` is False.
706  """
707  # get filter names into a numpy array...
708  # This is the type that is expected by the fgcm code
709  visitFilterNames = np.zeros(len(visitCat), dtype='a30')
710  for i in range(len(visitCat)):
711  visitFilterNames[i] = visitCat[i]['physicalFilter']
712 
713  # match to put filterNames with observations
714  visitIndex = np.searchsorted(visitCat['visit'],
715  obsCat['visit'])
716 
717  obsFilterNames = visitFilterNames[visitIndex]
718 
719  if self.config.doReferenceMatches:
720  # Get the reference filter names, using the LUT
721  lutCat = lutDataRef.get()
722 
723  stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
724  zip(lutCat[0]['physicalFilters'].split(','),
725  lutCat[0]['stdPhysicalFilters'].split(','))}
726  stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
727  zip(lutCat[0]['stdPhysicalFilters'].split(','),
728  lutCat[0]['lambdaStdFilter'])}
729 
730  del lutCat
731 
732  referenceFilterNames = self._getReferenceFilterNames_getReferenceFilterNames(visitCat,
733  stdFilterDict,
734  stdLambdaDict)
735  self.log.info("Using the following reference filters: %s" %
736  (', '.join(referenceFilterNames)))
737 
738  else:
739  # This should be an empty list
740  referenceFilterNames = []
741 
742  # make the fgcm starConfig dict
743  starConfig = {'logger': self.log,
744  'filterToBand': self.config.physicalFilterMap,
745  'requiredBands': self.config.requiredBands,
746  'minPerBand': self.config.minPerBand,
747  'matchRadius': self.config.matchRadius,
748  'isolationRadius': self.config.isolationRadius,
749  'matchNSide': self.config.matchNside,
750  'coarseNSide': self.config.coarseNside,
751  'densNSide': self.config.densityCutNside,
752  'densMaxPerPixel': self.config.densityCutMaxPerPixel,
753  'primaryBands': self.config.primaryBands,
754  'referenceFilterNames': referenceFilterNames}
755 
756  # initialize the FgcmMakeStars object
757  fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
758 
759  # make the primary stars
760  # note that the ra/dec native Angle format is radians
761  # We determine the conversion from the native units (typically
762  # radians) to degrees for the first observation. This allows us
763  # to treate ra/dec as numpy arrays rather than Angles, which would
764  # be approximately 600x slower.
765  conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
766  fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
767  obsCat['dec'] * conv,
768  filterNameArray=obsFilterNames,
769  bandSelected=False)
770 
771  # and match all the stars
772  fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
773  obsCat['dec'] * conv,
774  obsFilterNames)
775 
776  if self.config.doReferenceMatches:
777  fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
778 
779  # now persist
780 
781  objSchema = self._makeFgcmObjSchema_makeFgcmObjSchema()
782 
783  # make catalog and records
784  fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
785  fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
786  for i in range(fgcmMakeStars.objIndexCat.size):
787  fgcmStarIdCat.addNew()
788 
789  # fill the catalog
790  fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
791  fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
792  fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
793  fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
794  fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
795 
796  obsSchema = self._makeFgcmObsSchema_makeFgcmObsSchema()
797 
798  fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
799  fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
800  for i in range(fgcmMakeStars.obsIndexCat.size):
801  fgcmStarIndicesCat.addNew()
802 
803  fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
804 
805  if self.config.doReferenceMatches:
806  refSchema = self._makeFgcmRefSchema_makeFgcmRefSchema(len(referenceFilterNames))
807 
808  fgcmRefCat = afwTable.BaseCatalog(refSchema)
809  fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
810 
811  for i in range(fgcmMakeStars.referenceCat.size):
812  fgcmRefCat.addNew()
813 
814  fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
815  fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
816  fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
817 
818  md = PropertyList()
819  md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
820  md.set("FILTERNAMES", referenceFilterNames)
821  fgcmRefCat.setMetadata(md)
822 
823  else:
824  fgcmRefCat = None
825 
826  return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
827 
828  def _makeFgcmVisitSchema(self, nCcd):
829  """
830  Make a schema for an fgcmVisitCatalog
831 
832  Parameters
833  ----------
834  nCcd: `int`
835  Number of CCDs in the camera
836 
837  Returns
838  -------
839  schema: `afwTable.Schema`
840  """
841 
842  schema = afwTable.Schema()
843  schema.addField('visit', type=np.int32, doc="Visit number")
844  schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
845  schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
846  schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
847  schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
848  schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
849  schema.addField('mjd', type=np.float64, doc="MJD of visit")
850  schema.addField('exptime', type=np.float32, doc="Exposure time")
851  schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
852  schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
853  schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
854  schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
855  # the following field is not used yet
856  schema.addField('deepFlag', type=np.int32, doc="Deep observation")
857  schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
858  size=nCcd)
859  schema.addField('used', type=np.int32, doc="This visit has been ingested.")
860  schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
861 
862  return schema
863 
864  def _makeFgcmObjSchema(self):
865  """
866  Make a schema for the objIndexCat from fgcmMakeStars
867 
868  Returns
869  -------
870  schema: `afwTable.Schema`
871  """
872 
873  objSchema = afwTable.Schema()
874  objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
875  # Will investigate making these angles...
876  objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
877  objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
878  objSchema.addField('obsArrIndex', type=np.int32,
879  doc='Index in obsIndexTable for first observation')
880  objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
881 
882  return objSchema
883 
884  def _makeFgcmObsSchema(self):
885  """
886  Make a schema for the obsIndexCat from fgcmMakeStars
887 
888  Returns
889  -------
890  schema: `afwTable.Schema`
891  """
892 
893  obsSchema = afwTable.Schema()
894  obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
895 
896  return obsSchema
897 
898  def _makeFgcmRefSchema(self, nReferenceBands):
899  """
900  Make a schema for the referenceCat from fgcmMakeStars
901 
902  Parameters
903  ----------
904  nReferenceBands: `int`
905  Number of reference bands
906 
907  Returns
908  -------
909  schema: `afwTable.Schema`
910  """
911 
912  refSchema = afwTable.Schema()
913  refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
914  refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
915  size=nReferenceBands)
916  refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
917  size=nReferenceBands)
918 
919  return refSchema
920 
921  def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
922  """
923  Get the reference filter names, in wavelength order, from the visitCat and
924  information from the look-up-table.
925 
926  Parameters
927  ----------
928  visitCat: `afw.table.BaseCatalog`
929  Catalog with visit data for FGCM
930  stdFilterDict: `dict`
931  Mapping of filterName to stdFilterName from LUT
932  stdLambdaDict: `dict`
933  Mapping of stdFilterName to stdLambda from LUT
934 
935  Returns
936  -------
937  referenceFilterNames: `list`
938  Wavelength-ordered list of reference filter names
939  """
940 
941  # Find the unique list of filter names in visitCat
942  filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
943 
944  # Find the unique list of "standard" filters
945  stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
946 
947  # And sort these by wavelength
948  referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
949 
950  return referenceFilterNames
def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None)
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None)
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None, inVisitCat=None)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, sourceSchemaDataRef, camera, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs)
def __init__(self, butler=None, initInputs=None, **kwargs)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:799