70 addIsPhotometric=False, addIsResolved=False,
71 addIsVariable=False, fullPositionInformation=False):
72 """Make a standard schema for reference object catalogs.
76 filterNameList : `list` of `str`
77 List of filter names. Used to create <filterName>_flux fields.
79 If True then add fields "centroid" and "hasCentroid".
80 addIsPhotometric : `bool`
81 If True then add field "photometric".
82 addIsResolved : `bool`
83 If True then add field "resolved".
84 addIsVariable : `bool`
85 If True then add field "variable".
86 fullPositionInformation : `bool`
87 If True then add epoch, proper motion, and parallax, along with the
88 full five-dimensional covariance between ra and dec coordinates,
89 proper motion in ra and dec, and parallax.
93 schema : `lsst.afw.table.Schema`
94 Schema for reference catalog, an
95 `lsst.afw.table.SimpleCatalog`.
99 lsst.afw.table.Point2DKey.addFields(
102 "centroid on an exposure, if relevant",
108 doc=
"is position known?",
110 for filterName
in filterNameList:
112 field=
"%s_flux" % (filterName,),
114 doc=
"flux in filter %s" % (filterName,),
117 for filterName
in filterNameList:
119 field=
"%s_fluxErr" % (filterName,),
121 doc=
"flux uncertainty in filter %s" % (filterName,),
128 doc=
"set if the object can be used for photometric calibration",
134 doc=
"set if the object is spatially resolved",
140 doc=
"set if the object has variable brightness",
142 lsst.afw.table.CovarianceMatrix2fKey.addFields(
146 units=[
"rad",
"rad"],
150 if fullPositionInformation:
154 doc=
"date of observation (TAI, MJD)",
160 doc=
"proper motion in the right ascension direction = dra/dt * cos(dec)",
166 doc=
"proper motion in the declination direction",
169 lsst.afw.table.CovarianceMatrix2fKey.addFields(
173 units=[
"rad/year",
"rad/year"],
179 doc=
"Set if proper motion or proper motion error is bad",
190 doc=
"uncertainty in parallax",
194 field=
"parallax_flag",
196 doc=
"Set if parallax or parallax error is bad",
199 fields = [
"coord_ra",
"coord_dec",
"pm_ra",
"pm_dec",
"parallax"]
200 units = [
"rad",
"rad",
"rad/year",
"rad/year",
"rad"]
201 for field, unit
in zip(itertools.combinations(fields, r=2), itertools.combinations(units, r=2)):
206 formatted_unit =
"rad^2"
207 if (
"year" in i_unit)
and (
"year" in j_unit):
208 formatted_unit +=
"/year^2"
209 elif (
"year" in i_unit)
or (
"year" in j_unit):
210 formatted_unit +=
"/year"
212 field=f
"{i_field}_{j_field}_Cov",
214 doc=f
"Covariance between {i_field} and {j_field}",
221 """Add extra columns to a schema from a numpy dtype.
223 Note that schema and key_map will be modified in place.
227 schema : `lsst.afw.table.Schema`
228 Schema to append extra columns.
229 dtype : `numpy.dtype`
230 Numpy record array dtype.
231 extra_col_names : `list` [`str`]
232 Extra column names to convert from dtype into schema.
233 key_map : `dict` [`str`, `lsst.afw.table.Key`]
234 Mapping from column name to table key.
237 if dtype[name].kind ==
'U':
239 at_size = dtype[name].itemsize
240 return schema.addField(name, type=str, size=at_size)
241 elif dtype[name].kind ==
'b':
243 return schema.addField(name, type=
"Flag")
245 at_type = dtype[name].type
246 return schema.addField(name, at_type)
248 for col
in extra_col_names:
249 key_map[col] = addField(col)
275 dataset_config = pexConfig.ConfigField(
277 doc=
"Configuration for reading the ingested data",
279 n_processes = pexConfig.Field(
281 doc=(
"Number of python processes to use when ingesting."),
284 manager = pexConfig.ConfigurableField(
286 doc=
"Multiprocessing manager to perform the actual conversion of values, file-by-file."
288 file_reader = pexConfig.ConfigurableField(
289 target=ReadTextCatalogTask,
290 doc=
'Task to use to read the files. Default is to expect text files.'
292 ra_name = pexConfig.Field(
294 doc=
"Name of RA column (values in decimal degrees)",
296 dec_name = pexConfig.Field(
298 doc=
"Name of Dec column (values in decimal degrees)",
300 ra_err_name = pexConfig.Field(
302 doc=
"Name of RA error column",
305 dec_err_name = pexConfig.Field(
307 doc=
"Name of Dec error column",
310 coord_err_unit = pexConfig.Field(
312 doc=
"Unit of RA/Dec error fields (astropy.unit.Unit compatible)",
315 mag_column_list = pexConfig.ListField(
317 doc=
"The values in the reference catalog are assumed to be in AB magnitudes. "
318 "List of column names to use for photometric information. At least one entry is required."
320 mag_err_column_map = pexConfig.DictField(
324 doc=
"A map of magnitude column name (key) to magnitude error column (value)."
326 is_photometric_name = pexConfig.Field(
329 doc=
'Name of column stating if satisfactory for photometric calibration (optional).'
331 is_resolved_name = pexConfig.Field(
334 doc=
'Name of column stating if the object is resolved (optional).'
336 is_variable_name = pexConfig.Field(
339 doc=
'Name of column stating if the object is measured to be variable (optional).'
341 id_name = pexConfig.Field(
344 doc=
'Name of column to use as an identifier (optional).'
346 pm_ra_name = pexConfig.Field(
348 doc=
"Name of proper motion RA column",
351 pm_dec_name = pexConfig.Field(
353 doc=
"Name of proper motion Dec column",
356 pm_ra_err_name = pexConfig.Field(
358 doc=
"Name of proper motion RA error column",
361 pm_dec_err_name = pexConfig.Field(
363 doc=
"Name of proper motion Dec error column",
366 pm_scale = pexConfig.Field(
368 doc=
"Scale factor by which to multiply proper motion values to obtain units of milliarcsec/year",
371 parallax_name = pexConfig.Field(
373 doc=
"Name of parallax column",
376 parallax_err_name = pexConfig.Field(
378 doc=
"Name of parallax error column",
381 parallax_scale = pexConfig.Field(
383 doc=
"Scale factor by which to multiply parallax values to obtain units of milliarcsec",
386 full_position_information = pexConfig.Field(
388 doc=
"Include epoch, proper motions, parallax, and covariances between sky coordinates, proper motion,"
389 " and parallax in the schema. If true, a custom ``ConvertRefcatManager`` class must exist to"
390 " compute the output covariances.",
393 epoch_name = pexConfig.Field(
395 doc=
"Name of epoch column",
398 epoch_format = pexConfig.Field(
400 doc=
"Format of epoch column: any value accepted by astropy.time.Time, e.g. 'iso' or 'unix'",
403 epoch_scale = pexConfig.Field(
405 doc=
"Scale of epoch column: any value accepted by astropy.time.Time, e.g. 'utc'",
408 extra_col_names = pexConfig.ListField(
411 doc=
'Extra columns to add to the reference catalog.'
423 def assertAllOrNone(*names):
424 """Raise ValueError unless all the named fields are set or are
427 setNames = [name
for name
in names
if bool(getattr(self, name))]
428 if len(setNames)
in (len(names), 0):
430 prefix =
"Both or neither" if len(names) == 2
else "All or none"
431 raise ValueError(
"{} of {} must be set, but only {} are set".format(
432 prefix,
", ".join(names),
", ".join(setNames)))
436 "ra_name and dec_name and at least one entry in mag_column_list must be supplied.")
439 "mag_err_column_map specified, but keys do not match mag_column_list: {} != {}".format(
441 assertAllOrNone(
"ra_err_name",
"dec_err_name",
"coord_err_unit")
443 result = astropy.units.Unit(self.
coord_err_unit, parse_strict=
'silent')
444 if isinstance(result, astropy.units.UnrecognizedUnit):
445 msg = f
"{self.coord_err_unit} is not a valid astropy unit string."
446 raise pexConfig.FieldValidationError(ConvertReferenceCatalogConfig.coord_err_unit, self, msg)
448 assertAllOrNone(
"epoch_name",
"epoch_format",
"epoch_scale")
449 assertAllOrNone(
"pm_ra_name",
"pm_dec_name")
450 assertAllOrNone(
"pm_ra_err_name",
"pm_dec_err_name")
451 assertAllOrNone(
"parallax_name",
"parallax_err_name")
453 raise ValueError(
'"pm_ra/dec_name" must be specified if "pm_ra/dec_err_name" are specified')
456 '"epoch_name" must be specified if "pm_ra/dec_name" or "parallax_name" are specified')
462 assertAllOrNone(
"full_position_information",
463 "ra_err_name",
"dec_err_name",
"coord_err_unit",
464 "epoch_name",
"epoch_format",
"epoch_scale",
465 "pm_ra_name",
"pm_dec_name",
466 "pm_ra_err_name",
"pm_dec_err_name",
467 "parallax_name",
"parallax_err_name"
666 """Run `ConvertReferenceCatalogTask` on the input arguments.
671 Path to write the output files to.
673 File specifying the ``ConvertReferenceCatalogConfig`` fields.
675 Quoted glob for the files to be read in and converted.
678 logging.basicConfig(level=logging.INFO, format=
"{name} {levelname}: {message}", style=
"{")
680 config = ConvertReferenceCatalogTask.ConfigClass()
681 config.load(configFile)
683 files = glob.glob(fileglob)
685 with open(os.path.join(outputDir,
"convertReferenceCatalogConfig.py"),
"w")
as outfile:
686 converter.config.saveToStream(outfile)
687 msg = (
"Completed refcat conversion.\n\n"
688 "Ingest the resulting files with the following commands, substituting the path\n"
689 "to your butler repo for `REPO`, and the ticket number you are tracking this\n"
690 "ingest on for `DM-NNNNN`:\n"
691 f
"\n butler register-dataset-type REPO {config.dataset_config.ref_dataset_name} "
693 "\n butler ingest-files -t direct REPO gaia_dr2 refcats/DM-NNNNN "
694 f
"{converter.ingest_table_file}"
695 "\n butler collection-chain REPO --mode extend refcats refcats/DM-NNNNN")