lsst.fgcmcal  20.0.0-4-ge48a6ca+11
fgcmBuildStarsBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for BuildStars using src tables or sourceTable_visit tables.
22 """
23 
24 import os
25 import sys
26 import traceback
27 import abc
28 
29 import numpy as np
30 
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.table as afwTable
34 import lsst.geom as geom
35 from lsst.daf.base import PropertyList
36 from lsst.daf.base.dateTime import DateTime
37 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
38 
39 from .utilities import computeApertureRadiusFromDataRef
40 from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
41 
42 import fgcm
43 
44 REFSTARS_FORMAT_VERSION = 1
45 
46 __all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
47 
48 
49 class FgcmBuildStarsConfigBase(pexConfig.Config):
50  """Base config for FgcmBuildStars tasks"""
51 
52  instFluxField = pexConfig.Field(
53  doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
54  "The associated flag will be implicitly included in badFlags"),
55  dtype=str,
56  default='slot_CalibFlux_instFlux',
57  )
58  minPerBand = pexConfig.Field(
59  doc="Minimum observations per band",
60  dtype=int,
61  default=2,
62  )
63  matchRadius = pexConfig.Field(
64  doc="Match radius (arcseconds)",
65  dtype=float,
66  default=1.0,
67  )
68  isolationRadius = pexConfig.Field(
69  doc="Isolation radius (arcseconds)",
70  dtype=float,
71  default=2.0,
72  )
73  densityCutNside = pexConfig.Field(
74  doc="Density cut healpix nside",
75  dtype=int,
76  default=128,
77  )
78  densityCutMaxPerPixel = pexConfig.Field(
79  doc="Density cut number of stars per pixel",
80  dtype=int,
81  default=1000,
82  )
83  matchNside = pexConfig.Field(
84  doc="Healpix Nside for matching",
85  dtype=int,
86  default=4096,
87  )
88  coarseNside = pexConfig.Field(
89  doc="Healpix coarse Nside for partitioning matches",
90  dtype=int,
91  default=8,
92  )
93  filterMap = pexConfig.DictField(
94  doc="Mapping from 'filterName' to band.",
95  keytype=str,
96  itemtype=str,
97  default={},
98  )
99  requiredBands = pexConfig.ListField(
100  doc="Bands required for each star",
101  dtype=str,
102  default=(),
103  )
104  primaryBands = pexConfig.ListField(
105  doc=("Bands for 'primary' star matches. "
106  "A star must be observed in one of these bands to be considered "
107  "as a calibration star."),
108  dtype=str,
109  default=None
110  )
111  visitDataRefName = pexConfig.Field(
112  doc="dataRef name for the 'visit' field, usually 'visit'.",
113  dtype=str,
114  default="visit"
115  )
116  ccdDataRefName = pexConfig.Field(
117  doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
118  dtype=str,
119  default="ccd"
120  )
121  doApplyWcsJacobian = pexConfig.Field(
122  doc="Apply the jacobian of the WCS to the star observations prior to fit?",
123  dtype=bool,
124  default=True
125  )
126  psfCandidateName = pexConfig.Field(
127  doc="Name of field with psf candidate flag for propagation",
128  dtype=str,
129  default="calib_psf_candidate"
130  )
131  doSubtractLocalBackground = pexConfig.Field(
132  doc=("Subtract the local background before performing calibration? "
133  "This is only supported for circular aperture calibration fluxes."),
134  dtype=bool,
135  default=False
136  )
137  localBackgroundFluxField = pexConfig.Field(
138  doc="Full name of the local background instFlux field to use.",
139  dtype=str,
140  default='base_LocalBackground_instFlux'
141  )
142  sourceSelector = sourceSelectorRegistry.makeField(
143  doc="How to select sources",
144  default="science"
145  )
146  apertureInnerInstFluxField = pexConfig.Field(
147  doc=("Full name of instFlux field that contains inner aperture "
148  "flux for aperture correction proxy"),
149  dtype=str,
150  default='base_CircularApertureFlux_12_0_instFlux'
151  )
152  apertureOuterInstFluxField = pexConfig.Field(
153  doc=("Full name of instFlux field that contains outer aperture "
154  "flux for aperture correction proxy"),
155  dtype=str,
156  default='base_CircularApertureFlux_17_0_instFlux'
157  )
158  doReferenceMatches = pexConfig.Field(
159  doc="Match reference catalog as additional constraint on calibration",
160  dtype=bool,
161  default=True,
162  )
163  fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
164  target=FgcmLoadReferenceCatalogTask,
165  doc="FGCM reference object loader",
166  )
167  nVisitsPerCheckpoint = pexConfig.Field(
168  doc="Number of visits read between checkpoints",
169  dtype=int,
170  default=500,
171  )
172 
173  def setDefaults(self):
174  sourceSelector = self.sourceSelector["science"]
175  sourceSelector.setDefaults()
176 
177  sourceSelector.doFlags = True
178  sourceSelector.doUnresolved = True
179  sourceSelector.doSignalToNoise = True
180  sourceSelector.doIsolated = True
181 
182  sourceSelector.signalToNoise.minimum = 10.0
183  sourceSelector.signalToNoise.maximum = 1000.0
184 
185  # FGCM operates on unresolved sources, and this setting is
186  # appropriate for the current base_ClassificationExtendedness
187  sourceSelector.unresolved.maximum = 0.5
188 
189 
190 class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
191  """Subclass of TaskRunner for FgcmBuildStars tasks
192 
193  fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
194  arguments, one of which is the butler (for persistence and mapper data),
195  and a list of dataRefs extracted from the command line. Note that FGCM
196  runs on a large set of dataRefs, and not on single dataRef/tract/patch.
197  This class transforms the process arguments generated by the ArgumentParser
198  into the arguments expected by FgcmBuildStarsTask.run(). This runner does
199  not use any parallelization.
200  """
201  @staticmethod
202  def getTargetList(parsedCmd):
203  """
204  Return a list with one element: a tuple with the butler and
205  list of dataRefs
206  """
207  # we want to combine the butler with any (or no!) dataRefs
208  return [(parsedCmd.butler, parsedCmd.id.refList)]
209 
210  def __call__(self, args):
211  """
212  Parameters
213  ----------
214  args: `tuple` with (butler, dataRefList)
215 
216  Returns
217  -------
218  exitStatus: `list` with `lsst.pipe.base.Struct`
219  exitStatus (0: success; 1: failure)
220  """
221  butler, dataRefList = args
222 
223  task = self.TaskClass(config=self.config, log=self.log)
224 
225  exitStatus = 0
226  if self.doRaise:
227  task.runDataRef(butler, dataRefList)
228  else:
229  try:
230  task.runDataRef(butler, dataRefList)
231  except Exception as e:
232  exitStatus = 1
233  task.log.fatal("Failed: %s" % e)
234  if not isinstance(e, pipeBase.TaskError):
235  traceback.print_exc(file=sys.stderr)
236 
237  task.writeMetadata(butler)
238 
239  # The task does not return any results:
240  return [pipeBase.Struct(exitStatus=exitStatus)]
241 
242  def run(self, parsedCmd):
243  """
244  Run the task, with no multiprocessing
245 
246  Parameters
247  ----------
248  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
249  """
250 
251  resultList = []
252 
253  if self.precall(parsedCmd):
254  targetList = self.getTargetList(parsedCmd)
255  resultList = self(targetList[0])
256 
257  return resultList
258 
259 
260 class FgcmBuildStarsBaseTask(pipeBase.CmdLineTask, abc.ABC):
261  """
262  Base task to build stars for FGCM global calibration
263 
264  Parameters
265  ----------
266  butler : `lsst.daf.persistence.Butler`
267  """
268  def __init__(self, butler=None, **kwargs):
269  pipeBase.CmdLineTask.__init__(self, **kwargs)
270  self.makeSubtask("sourceSelector")
271  # Only log warning and fatal errors from the sourceSelector
272  self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
273 
274  # no saving of metadata for now
275  def _getMetadataName(self):
276  return None
277 
278  @pipeBase.timeMethod
279  def runDataRef(self, butler, dataRefs):
280  """
281  Cross-match and make star list for FGCM Input
282 
283  Parameters
284  ----------
285  butler: `lsst.daf.persistence.Butler`
286  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
287  Source data references for the input visits.
288 
289  Raises
290  ------
291  RuntimeErrror: Raised if `config.doReferenceMatches` is set and
292  an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
293  fails if the calibFlux is not a CircularAperture flux.
294  """
295  datasetType = dataRefs[0].butlerSubset.datasetType
296  self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
297 
298  if self.config.doReferenceMatches:
299  # Ensure that we have a LUT
300  if not butler.datasetExists('fgcmLookUpTable'):
301  raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
302  # Compute aperture radius if necessary. This is useful to do now before
303  # any heavy lifting has happened (fail early).
304  calibFluxApertureRadius = None
305  if self.config.doSubtractLocalBackground:
306  try:
307  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
308  self.config.instFluxField)
309  except RuntimeError as e:
310  raise RuntimeError("Could not determine aperture radius from %s. "
311  "Cannot use doSubtractLocalBackground." %
312  (self.config.instFluxField)) from e
313 
314  groupedDataRefs = self.findAndGroupDataRefs(butler, dataRefs)
315 
316  camera = butler.get('camera')
317 
318  # Make the visit catalog if necessary
319  # First check if the visit catalog is in the _current_ path
320  # We cannot use Gen2 datasetExists() because that checks all parent
321  # directories as well, which would make recovering from faults
322  # and fgcmcal reruns impossible.
323  visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
324  filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
325  if os.path.exists(filename):
326  # This file exists and we should continue processing
327  inVisitCat = visitCatDataRef.get()
328  if len(inVisitCat) != len(groupedDataRefs):
329  raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
330  "number of visits. Cannot continue.")
331  else:
332  inVisitCat = None
333 
334  visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
335  visitCatDataRef=visitCatDataRef,
336  inVisitCat=inVisitCat)
337 
338  # Persist the visitCat as a checkpoint file.
339  visitCatDataRef.put(visitCat)
340 
341  starObsDataRef = butler.dataRef('fgcmStarObservations')
342  filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
343  if os.path.exists(filename):
344  inStarObsCat = starObsDataRef.get()
345  else:
346  inStarObsCat = None
347 
348  rad = calibFluxApertureRadius
349  fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
350  visitCat,
351  calibFluxApertureRadius=rad,
352  starObsDataRef=starObsDataRef,
353  visitCatDataRef=visitCatDataRef,
354  inStarObsCat=inStarObsCat)
355  visitCatDataRef.put(visitCat)
356  starObsDataRef.put(fgcmStarObservationCat)
357 
358  # Always do the matching.
359  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(butler,
360  visitCat,
361  fgcmStarObservationCat)
362 
363  # Persist catalogs via the butler
364  butler.put(fgcmStarIdCat, 'fgcmStarIds')
365  butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
366  if fgcmRefCat is not None:
367  butler.put(fgcmRefCat, 'fgcmReferenceStars')
368 
369  @abc.abstractmethod
370  def findAndGroupDataRefs(self, butler, dataRefs):
371  """
372  Find and group dataRefs (by visit).
373 
374  Parameters
375  ----------
376  butler: `lsst.daf.persistence.Butler`
377  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
378  Data references for the input visits.
379 
380  Returns
381  -------
382  groupedDataRefs: `dict` [`int`, `list`]
383  Dictionary with visit keys, and `list`s of `lsst.daf.persistence.ButlerDataRef`
384  """
385  raise NotImplementedError("findAndGroupDataRefs not implemented.")
386 
387  @abc.abstractmethod
388  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
389  calibFluxApertureRadius=None,
390  visitCatDataRef=None,
391  starObsDataRef=None,
392  inStarObsCat=None):
393  """
394  Compile all good star observations from visits in visitCat. Checkpoint files
395  will be stored if both visitCatDataRef and starObsDataRef are not None.
396 
397  Parameters
398  ----------
399  groupedDataRefs: `dict` of `list`s
400  Lists of `lsst.daf.persistence.ButlerDataRef`, grouped by visit.
401  visitCat: `afw.table.BaseCatalog`
402  Catalog with visit data for FGCM
403  calibFluxApertureRadius: `float`, optional
404  Aperture radius for calibration flux.
405  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
406  Dataref to write visitCat for checkpoints
407  starObsDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
408  Dataref to write the star observation catalog for checkpoints.
409  inStarObsCat: `afw.table.BaseCatalog`
410  Input observation catalog. If this is incomplete, observations
411  will be appended from when it was cut off.
412 
413  Returns
414  -------
415  fgcmStarObservations: `afw.table.BaseCatalog`
416  Full catalog of good observations.
417 
418  Raises
419  ------
420  RuntimeError: Raised if doSubtractLocalBackground is True and
421  calibFluxApertureRadius is not set.
422  """
423  raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
424 
425  def fgcmMakeVisitCatalog(self, camera, groupedDataRefs,
426  visitCatDataRef=None, inVisitCat=None):
427  """
428  Make a visit catalog with all the keys from each visit
429 
430  Parameters
431  ----------
432  camera: `lsst.afw.cameraGeom.Camera`
433  Camera from the butler
434  groupedDataRefs: `dict`
435  Dictionary with visit keys, and `list`s of
436  `lsst.daf.persistence.ButlerDataRef`
437  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
438  Dataref to write visitCat for checkpoints
439  inVisitCat: `afw.table.BaseCatalog`
440  Input (possibly incomplete) visit catalog
441 
442  Returns
443  -------
444  visitCat: `afw.table.BaseCatalog`
445  """
446 
447  self.log.info("Assembling visitCatalog from %d %ss" %
448  (len(groupedDataRefs), self.config.visitDataRefName))
449 
450  nCcd = len(camera)
451 
452  if inVisitCat is None:
453  schema = self._makeFgcmVisitSchema(nCcd)
454 
455  visitCat = afwTable.BaseCatalog(schema)
456  visitCat.reserve(len(groupedDataRefs))
457 
458  for i, visit in enumerate(sorted(groupedDataRefs)):
459  rec = visitCat.addNew()
460  rec['visit'] = visit
461  rec['used'] = 0
462  rec['sources_read'] = 0
463  else:
464  visitCat = inVisitCat
465 
466  # No matter what, fill the catalog. This will check if it was
467  # already read.
468  self._fillVisitCatalog(visitCat, groupedDataRefs,
469  visitCatDataRef=visitCatDataRef)
470 
471  return visitCat
472 
473  def _fillVisitCatalog(self, visitCat, groupedDataRefs,
474  visitCatDataRef=None):
475  """
476  Fill the visit catalog with visit metadata
477 
478  Parameters
479  ----------
480  visitCat: `afw.table.BaseCatalog`
481  Catalog with schema from _makeFgcmVisitSchema()
482  groupedDataRefs: `dict`
483  Dictionary with visit keys, and `list`s of
484  `lsst.daf.persistence.ButlerDataRef
485  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
486  Dataref to write visitCat for checkpoints
487  """
488 
489  bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
490 
491  for i, visit in enumerate(sorted(groupedDataRefs)):
492  # We don't use the bypasses since we need the psf info which does
493  # not have a bypass
494  # TODO: When DM-15500 is implemented in the Gen3 Butler, this
495  # can be fixed
496 
497  # Do not read those that have already been read
498  if visitCat['used'][i]:
499  continue
500 
501  if (i % self.config.nVisitsPerCheckpoint) == 0:
502  self.log.info("Retrieving metadata for %s %d (%d/%d)" %
503  (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
504  # Save checkpoint if desired
505  if visitCatDataRef is not None:
506  visitCatDataRef.put(visitCat)
507 
508  # Note that the reference ccd is first in the list (if available).
509 
510  # The first dataRef in the group will be the reference ccd (if available)
511  dataRef = groupedDataRefs[visit][0]
512 
513  exp = dataRef.get(datasetType='calexp_sub', bbox=bbox,
514  flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
515 
516  visitInfo = exp.getInfo().getVisitInfo()
517  f = exp.getFilter()
518  psf = exp.getPsf()
519 
520  rec = visitCat[i]
521  rec['visit'] = visit
522  rec['filtername'] = f.getName()
523  radec = visitInfo.getBoresightRaDec()
524  rec['telra'] = radec.getRa().asDegrees()
525  rec['teldec'] = radec.getDec().asDegrees()
526  rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
527  rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
528  rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
529  rec['exptime'] = visitInfo.getExposureTime()
530  # convert from Pa to millibar
531  # Note that I don't know if this unit will need to be per-camera config
532  rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
533  # Flag to signify if this is a "deep" field. Not currently used
534  rec['deepFlag'] = 0
535  # Relative flat scaling (1.0 means no relative scaling)
536  rec['scaling'][:] = 1.0
537  # Median delta aperture, to be measured from stars
538  rec['deltaAper'] = 0.0
539 
540  rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
541 
542  if dataRef.datasetExists(datasetType='calexpBackground'):
543  # Get background for reference CCD
544  # This approximation is good enough for now
545  bgStats = (bg[0].getStatsImage().getImage().array
546  for bg in dataRef.get(datasetType='calexpBackground'))
547  rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
548  else:
549  self.log.warn('Sky background not found for visit %d / ccd %d' %
550  (visit, dataRef.dataId[self.config.ccdDataRefName]))
551  rec['skyBackground'] = -1.0
552 
553  rec['used'] = 1
554 
555  def _makeSourceMapper(self, sourceSchema):
556  """
557  Make a schema mapper for fgcm sources
558 
559  Parameters
560  ----------
561  sourceSchema: `afwTable.Schema`
562  Default source schema from the butler
563 
564  Returns
565  -------
566  sourceMapper: `afwTable.schemaMapper`
567  Mapper to the FGCM source schema
568  """
569 
570  # create a mapper to the preferred output
571  sourceMapper = afwTable.SchemaMapper(sourceSchema)
572 
573  # map to ra/dec
574  sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
575  sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
576  sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
577  sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
578  # Add the mapping if the field exists in the input catalog.
579  # If the field does not exist, simply add it (set to False).
580  # This field is not required for calibration, but is useful
581  # to collate if available.
582  try:
583  sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
584  'psf_candidate')
585  except LookupError:
586  sourceMapper.editOutputSchema().addField(
587  "psf_candidate", type='Flag',
588  doc=("Flag set if the source was a candidate for PSF determination, "
589  "as determined by the star selector."))
590 
591  # and add the fields we want
592  sourceMapper.editOutputSchema().addField(
593  "visit", type=np.int32, doc="Visit number")
594  sourceMapper.editOutputSchema().addField(
595  "ccd", type=np.int32, doc="CCD number")
596  sourceMapper.editOutputSchema().addField(
597  "instMag", type=np.float32, doc="Instrumental magnitude")
598  sourceMapper.editOutputSchema().addField(
599  "instMagErr", type=np.float32, doc="Instrumental magnitude error")
600  sourceMapper.editOutputSchema().addField(
601  "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
602 
603  return sourceMapper
604 
605  def fgcmMatchStars(self, butler, visitCat, obsCat):
606  """
607  Use FGCM code to match observations into unique stars.
608 
609  Parameters
610  ----------
611  butler: `lsst.daf.persistence.Butler`
612  visitCat: `afw.table.BaseCatalog`
613  Catalog with visit data for fgcm
614  obsCat: `afw.table.BaseCatalog`
615  Full catalog of star observations for fgcm
616 
617  Returns
618  -------
619  fgcmStarIdCat: `afw.table.BaseCatalog`
620  Catalog of unique star identifiers and index keys
621  fgcmStarIndicesCat: `afwTable.BaseCatalog`
622  Catalog of unique star indices
623  fgcmRefCat: `afw.table.BaseCatalog`
624  Catalog of matched reference stars.
625  Will be None if `config.doReferenceMatches` is False.
626  """
627 
628  if self.config.doReferenceMatches:
629  # Make a subtask for reference loading
630  self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
631 
632  # get filter names into a numpy array...
633  # This is the type that is expected by the fgcm code
634  visitFilterNames = np.zeros(len(visitCat), dtype='a10')
635  for i in range(len(visitCat)):
636  visitFilterNames[i] = visitCat[i]['filtername']
637 
638  # match to put filterNames with observations
639  visitIndex = np.searchsorted(visitCat['visit'],
640  obsCat['visit'])
641 
642  obsFilterNames = visitFilterNames[visitIndex]
643 
644  if self.config.doReferenceMatches:
645  # Get the reference filter names, using the LUT
646  lutCat = butler.get('fgcmLookUpTable')
647 
648  stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
649  zip(lutCat[0]['filterNames'].split(','),
650  lutCat[0]['stdFilterNames'].split(','))}
651  stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
652  zip(lutCat[0]['stdFilterNames'].split(','),
653  lutCat[0]['lambdaStdFilter'])}
654 
655  del lutCat
656 
657  referenceFilterNames = self._getReferenceFilterNames(visitCat,
658  stdFilterDict,
659  stdLambdaDict)
660  self.log.info("Using the following reference filters: %s" %
661  (', '.join(referenceFilterNames)))
662 
663  else:
664  # This should be an empty list
665  referenceFilterNames = []
666 
667  # make the fgcm starConfig dict
668 
669  starConfig = {'logger': self.log,
670  'filterToBand': self.config.filterMap,
671  'requiredBands': self.config.requiredBands,
672  'minPerBand': self.config.minPerBand,
673  'matchRadius': self.config.matchRadius,
674  'isolationRadius': self.config.isolationRadius,
675  'matchNSide': self.config.matchNside,
676  'coarseNSide': self.config.coarseNside,
677  'densNSide': self.config.densityCutNside,
678  'densMaxPerPixel': self.config.densityCutMaxPerPixel,
679  'primaryBands': self.config.primaryBands,
680  'referenceFilterNames': referenceFilterNames}
681 
682  # initialize the FgcmMakeStars object
683  fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
684 
685  # make the primary stars
686  # note that the ra/dec native Angle format is radians
687  # We determine the conversion from the native units (typically
688  # radians) to degrees for the first observation. This allows us
689  # to treate ra/dec as numpy arrays rather than Angles, which would
690  # be approximately 600x slower.
691  conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
692  fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
693  obsCat['dec'] * conv,
694  filterNameArray=obsFilterNames,
695  bandSelected=False)
696 
697  # and match all the stars
698  fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
699  obsCat['dec'] * conv,
700  obsFilterNames)
701 
702  if self.config.doReferenceMatches:
703  fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
704 
705  # now persist
706 
707  objSchema = self._makeFgcmObjSchema()
708 
709  # make catalog and records
710  fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
711  fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
712  for i in range(fgcmMakeStars.objIndexCat.size):
713  fgcmStarIdCat.addNew()
714 
715  # fill the catalog
716  fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
717  fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
718  fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
719  fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
720  fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
721 
722  obsSchema = self._makeFgcmObsSchema()
723 
724  fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
725  fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
726  for i in range(fgcmMakeStars.obsIndexCat.size):
727  fgcmStarIndicesCat.addNew()
728 
729  fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
730 
731  if self.config.doReferenceMatches:
732  refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
733 
734  fgcmRefCat = afwTable.BaseCatalog(refSchema)
735  fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
736 
737  for i in range(fgcmMakeStars.referenceCat.size):
738  fgcmRefCat.addNew()
739 
740  fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
741  fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
742  fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
743 
744  md = PropertyList()
745  md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
746  md.set("FILTERNAMES", referenceFilterNames)
747  fgcmRefCat.setMetadata(md)
748 
749  else:
750  fgcmRefCat = None
751 
752  return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
753 
754  def _makeFgcmVisitSchema(self, nCcd):
755  """
756  Make a schema for an fgcmVisitCatalog
757 
758  Parameters
759  ----------
760  nCcd: `int`
761  Number of CCDs in the camera
762 
763  Returns
764  -------
765  schema: `afwTable.Schema`
766  """
767 
768  schema = afwTable.Schema()
769  schema.addField('visit', type=np.int32, doc="Visit number")
770  # Note that the FGCM code currently handles filternames up to 2 characters long
771  schema.addField('filtername', type=str, size=10, doc="Filter name")
772  schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
773  schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
774  schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
775  schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
776  schema.addField('mjd', type=np.float64, doc="MJD of visit")
777  schema.addField('exptime', type=np.float32, doc="Exposure time")
778  schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
779  schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
780  schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
781  schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
782  # the following field is not used yet
783  schema.addField('deepFlag', type=np.int32, doc="Deep observation")
784  schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
785  size=nCcd)
786  schema.addField('used', type=np.int32, doc="This visit has been ingested.")
787  schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
788 
789  return schema
790 
791  def _makeFgcmObjSchema(self):
792  """
793  Make a schema for the objIndexCat from fgcmMakeStars
794 
795  Returns
796  -------
797  schema: `afwTable.Schema`
798  """
799 
800  objSchema = afwTable.Schema()
801  objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
802  # Will investigate making these angles...
803  objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
804  objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
805  objSchema.addField('obsArrIndex', type=np.int32,
806  doc='Index in obsIndexTable for first observation')
807  objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
808 
809  return objSchema
810 
811  def _makeFgcmObsSchema(self):
812  """
813  Make a schema for the obsIndexCat from fgcmMakeStars
814 
815  Returns
816  -------
817  schema: `afwTable.Schema`
818  """
819 
820  obsSchema = afwTable.Schema()
821  obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
822 
823  return obsSchema
824 
825  def _makeFgcmRefSchema(self, nReferenceBands):
826  """
827  Make a schema for the referenceCat from fgcmMakeStars
828 
829  Parameters
830  ----------
831  nReferenceBands: `int`
832  Number of reference bands
833 
834  Returns
835  -------
836  schema: `afwTable.Schema`
837  """
838 
839  refSchema = afwTable.Schema()
840  refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
841  refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
842  size=nReferenceBands)
843  refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
844  size=nReferenceBands)
845 
846  return refSchema
847 
848  def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
849  """
850  Get the reference filter names, in wavelength order, from the visitCat and
851  information from the look-up-table.
852 
853  Parameters
854  ----------
855  visitCat: `afw.table.BaseCatalog`
856  Catalog with visit data for FGCM
857  stdFilterDict: `dict`
858  Mapping of filterName to stdFilterName from LUT
859  stdLambdaDict: `dict`
860  Mapping of stdFilterName to stdLambda from LUT
861 
862  Returns
863  -------
864  referenceFilterNames: `list`
865  Wavelength-ordered list of reference filter names
866  """
867 
868  # Find the unique list of filter names in visitCat
869  filterNames = np.unique(visitCat.asAstropy()['filtername'])
870 
871  # Find the unique list of "standard" filters
872  stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
873 
874  # And sort these by wavelength
875  referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
876 
877  return referenceFilterNames
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsRunner
Definition: fgcmBuildStarsBase.py:190
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.setDefaults
def setDefaults(self)
Definition: fgcmBuildStarsBase.py:173
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask
Definition: fgcmBuildStarsBase.py:260
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeFgcmVisitSchema
def _makeFgcmVisitSchema(self, nCcd)
Definition: fgcmBuildStarsBase.py:754
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.fgcmMatchStars
def fgcmMatchStars(self, butler, visitCat, obsCat)
Definition: fgcmBuildStarsBase.py:605
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeFgcmObsSchema
def _makeFgcmObsSchema(self)
Definition: fgcmBuildStarsBase.py:811
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeFgcmRefSchema
def _makeFgcmRefSchema(self, nReferenceBands)
Definition: fgcmBuildStarsBase.py:825
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsRunner.run
def run(self, parsedCmd)
Definition: fgcmBuildStarsBase.py:242
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsRunner.getTargetList
def getTargetList(parsedCmd)
Definition: fgcmBuildStarsBase.py:202
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.fgcmMakeAllStarObservations
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
Definition: fgcmBuildStarsBase.py:388
lsst.fgcmcal.utilities.computeApertureRadiusFromDataRef
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:776
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._getReferenceFilterNames
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
Definition: fgcmBuildStarsBase.py:848
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.__init__
def __init__(self, butler=None, **kwargs)
Definition: fgcmBuildStarsBase.py:268
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.fgcmMakeVisitCatalog
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, visitCatDataRef=None, inVisitCat=None)
Definition: fgcmBuildStarsBase.py:425
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._fillVisitCatalog
def _fillVisitCatalog(self, visitCat, groupedDataRefs, visitCatDataRef=None)
Definition: fgcmBuildStarsBase.py:473
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.findAndGroupDataRefs
def findAndGroupDataRefs(self, butler, dataRefs)
Definition: fgcmBuildStarsBase.py:370
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.runDataRef
def runDataRef(self, butler, dataRefs)
Definition: fgcmBuildStarsBase.py:279
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeFgcmObjSchema
def _makeFgcmObjSchema(self)
Definition: fgcmBuildStarsBase.py:791
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.sourceSelector
sourceSelector
Definition: fgcmBuildStarsBase.py:142
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsRunner.__call__
def __call__(self, args)
Definition: fgcmBuildStarsBase.py:210
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase
Definition: fgcmBuildStarsBase.py:49