23Convert an external reference catalog into the hierarchical triangular mesh
24(HTM) sharded LSST-style format, to be ingested into the butler.
27__all__ = [
"ConvertReferenceCatalogTask"]
37from .
import ConvertReferenceCatalogBase
41 """Class for producing HTM-indexed reference catalogs from external
47 The path to write the output files to, in a subdirectory defined by
48 ``DatasetConfig.ref_dataset_name``.
50 _DefaultName = 'ConvertReferenceCatalogTask'
52 def __init__(self, *, output_dir=None, **kwargs):
54 if output_dir
is None:
55 raise RuntimeError(
"Must specify output_dir.")
57 self.
output_dir = os.path.join(output_dir, self.config.dataset_config.ref_dataset_name)
63 pathlib.Path(self.
output_dir).mkdir(exist_ok=
False)
65 def _postRun(self, result):
67 dimension = f
"htm{self.config.dataset_config.indexer.active.depth}"
68 table = astropy.table.Table(names=(
"filename", dimension), dtype=(
'str',
'int'))
70 table.add_row((result[key], key))
73 def _persistConfig(self):
74 filename = os.path.join(self.
output_dir,
"config.py")
75 with open(filename,
'w')
as file:
76 self.config.dataset_config.saveToStream(file)
78 def _getOnePixelFilename(self, start):
79 return os.path.join(self.
output_dir, f
"{self.indexer.htm}.fits")
81 def _writeMasterSchema(self, catalog):
82 filename = os.path.join(self.
output_dir,
"master_schema.fits")
83 catalog.writeFits(filename)
85 def _reduce_kwargs(self):
87 kwargs = super()._reduce_kwargs()
93 """Construct an argument parser for the ``convertReferenceCatalog`` script.
97 argparser : `argparse.ArgumentParser`
98 The argument parser that defines the ``convertReferenceCatalog``
99 command-line interface.
101 parser = argparse.ArgumentParser(
103 formatter_class=argparse.RawDescriptionHelpFormatter,
104 epilog='More information is available at https://pipelines.lsst.io.'
106 parser.add_argument(
"outputDir",
107 help=
"Path to write the output shard files, configs, and `ingest-files` table to.")
108 parser.add_argument(
"configFile",
109 help=
"File containing the ConvertReferenceCatalogConfig fields.")
112 parser.add_argument(
"fileglob", nargs=
"+",
113 help=
"Quoted glob for the files to be read in and converted."
114 " Example (note required quotes to prevent shell expansion):"
115 ' "gaia_source/csv/GaiaSource*"')
120 """Run `ConvertReferenceCatalogTask` on the input arguments.
125 Path to write the output files to.
127 File specifying the ``ConvertReferenceCatalogConfig`` fields.
129 Quoted glob for the files to be read
in and converted.
132 logging.basicConfig(level=logging.INFO, format=
"{name} {levelname}: {message}", style=
"{")
134 config = ConvertReferenceCatalogTask.ConfigClass()
135 config.load(configFile)
138 files = glob.glob(fileglob)
140 with open(os.path.join(outputDir,
"convertReferenceCatalogConfig.py"),
"w")
as outfile:
141 converter.config.saveToStream(outfile)
142 msg = (
"Completed refcat conversion."
143 " Ingest the resulting files with the following commands,"
144 " substituting the path to your butler repo for REPO:"
145 f
"\n butler register-dataset-type REPO {config.dataset_config.ref_dataset_name} "
147 f
"\n butler ingest-files -t direct REPO gaia_dr2 refcats {converter.ingest_table_file}")
153 if len(args.fileglob) > 1:
154 raise RuntimeError(
"Final argument must be a quoted file glob, not a shell-expanded list of files.")
156 run_convert(args.outputDir, args.configFile, args.fileglob[0])
def __init__(self, *output_dir=None, **kwargs)
def run_convert(outputDir, configFile, fileglob)