4 from dateutil
import parser
7 from lsst.pex.config
import Config, Field, ListField, ConfigurableField
12 def _convertToDate(dateString):
13 """Convert a string into a date object"""
14 return parser.parse(dateString).date()
18 """Task that will parse the filename and/or its contents to get the
19 required information to populate the calibration registry."""
22 """Return a a known calibration dataset type using
23 the observation type in the header keyword OBSTYPE
25 @param filename: Input filename
27 md = readMetadata(filename, self.config.hdu)
28 if not md.exists(
"OBSTYPE"):
29 raise RuntimeError(
"Unable to find the required header keyword OBSTYPE in %s, hdu %d" %
30 (filename, self.config.hdu))
31 obstype = md.getScalar(
"OBSTYPE").strip().lower()
34 elif "zero" in obstype
or "bias" in obstype:
36 elif "dark" in obstype:
38 elif "fringe" in obstype:
40 elif "sky" in obstype:
42 elif "illumcor" in obstype:
44 elif "defects" in obstype:
46 elif "qe_curve" in obstype:
51 """Get destination for the file
53 @param butler Data butler
54 @param info File properties, used as dataId for the butler
55 @param filename Input filename
56 @return Destination filename
61 tempinfo = {k: v
for (k, v)
in info.items()
if v
is not None}
63 raw = butler.get(calibType +
"_filename", tempinfo)[0]
72 """Configuration for the CalibsRegisterTask"""
73 tables = ListField(dtype=str, default=[
"bias",
"dark",
"flat",
"fringe",
"sky",
"defects",
"qe_curve"],
74 doc=
"Names of tables")
75 calibDate = Field(dtype=str, default=
"calibDate", doc=
"Name of column for calibration date")
76 validStart = Field(dtype=str, default=
"validStart", doc=
"Name of column for validity start")
77 validEnd = Field(dtype=str, default=
"validEnd", doc=
"Name of column for validity stop")
78 detector = ListField(dtype=str, default=[
"filter",
"ccd"],
79 doc=
"Columns that identify individual detectors")
80 validityUntilSuperseded = ListField(dtype=str, default=[
"defects",
"qe_curve"],
81 doc=
"Tables for which to set validity for a calib from when it is "
82 "taken until it is superseded by the next; validity in other tables "
83 "is calculated by applying the validity range.")
87 """Task that will generate the calibration registry for the Mapper"""
88 ConfigClass = CalibsRegisterConfig
90 def openRegistry(self, directory, create=False, dryrun=False, name="calibRegistry.sqlite3"):
91 """Open the registry and return the connection handle"""
92 return RegisterTask.openRegistry(self, directory, create, dryrun, name)
95 """Create the registry tables"""
96 for table
in self.config.tables:
97 RegisterTask.createTable(self, conn, table=table, forceCreateTables=forceCreateTables)
99 def addRow(self, conn, info, *args, **kwargs):
100 """Add a row to the file table"""
101 info[self.config.validStart] =
None
102 info[self.config.validEnd] =
None
103 RegisterTask.addRow(self, conn, info, *args, **kwargs)
106 """Loop over all tables, filters, and ccdnums,
107 and update the validity ranges in the registry.
109 @param conn: Database connection
110 @param validity: Validity range (days)
112 conn.row_factory = sqlite3.Row
113 cursor = conn.cursor()
115 tables = self.config.tables
117 sql =
"SELECT DISTINCT %s FROM %s" % (
", ".join(self.config.detector), table)
119 rows = cursor.fetchall()
124 """Update the validity ranges among selected rows in the registry.
126 For defects and qe_curve, the products are valid from their start date until
127 they are superseded by subsequent defect data.
128 For other calibration products, the validity ranges are checked and
129 if there are overlaps, a midpoint is used to fix the overlaps,
130 so that the calibration data with whose date is nearest the date
131 of the observation is used.
133 @param conn: Database connection
134 @param table: Name of table to be selected
135 @param detectorData: Values identifying a detector (from columns in self.config.detector)
136 @param validity: Validity range (days)
138 columns =
", ".join([self.config.calibDate, self.config.validStart, self.config.validEnd])
139 sql =
"SELECT id, %s FROM %s" % (columns, table)
140 sql +=
" WHERE " +
" AND ".join(col +
"=?" for col
in self.config.detector)
141 sql +=
" ORDER BY " + self.config.calibDate
142 cursor = conn.cursor()
143 cursor.execute(sql, detectorData)
144 rows = cursor.fetchall()
147 valids = collections.OrderedDict([(_convertToDate(row[self.config.calibDate]), [
None,
None])
for
150 det =
" ".join(
"%s=%s" % (k, v)
for k, v
in zip(self.config.detector, detectorData))
152 self.log.warn(str(
"Skipped setting the validity overlaps for %s %s: missing calibration dates" %
155 dates = list(valids.keys())
156 if table
in self.config.validityUntilSuperseded:
158 for thisDate, nextDate
in zip(dates[:-1], dates[1:]):
159 valids[thisDate][0] = thisDate
160 valids[thisDate][1] = nextDate - datetime.timedelta(1)
161 valids[dates[-1]][0] = dates[-1]
162 valids[dates[-1]][1] = _convertToDate(
"2037-12-31")
166 valids[dd] = [dd - datetime.timedelta(validity), dd + datetime.timedelta(validity)]
169 midpoints = [t1 + (t2 - t1)//2
for t1, t2
in zip(dates[:-1], dates[1:])]
170 for i, (date, midpoint)
in enumerate(zip(dates[:-1], midpoints)):
171 if valids[date][1] > midpoint:
172 nextDate = dates[i + 1]
173 valids[nextDate][0] = midpoint + datetime.timedelta(1)
174 valids[date][1] = midpoint
179 calibDate = _convertToDate(row[self.config.calibDate])
180 validStart = valids[calibDate][0].isoformat()
181 validEnd = valids[calibDate][1].isoformat()
182 sql =
"UPDATE %s" % table
183 sql +=
" SET %s=?, %s=?" % (self.config.validStart, self.config.validEnd)
185 conn.execute(sql, (validStart, validEnd, row[
"id"]))
189 """Argument parser to support ingesting calibration images into the repository"""
192 InputOnlyArgumentParser.__init__(self, *args, **kwargs)
193 self.add_argument(
"-n",
"--dry-run", dest=
"dryrun", action=
"store_true",
194 default=
False, help=
"Don't perform any action?")
195 self.add_argument(
"--mode", choices=[
"move",
"copy",
"link",
"skip"], default=
"move",
196 help=
"Mode of delivering the files to their destination")
197 self.add_argument(
"--create", action=
"store_true", help=
"Create new registry?")
198 self.add_argument(
"--validity", type=int, required=
True, help=
"Calibration validity period (days)")
199 self.add_argument(
"--ignore-ingested", dest=
"ignoreIngested", action=
"store_true",
200 help=
"Don't register files that have already been registered")
201 self.add_argument(
"files", nargs=
"+", help=
"Names of file")
205 """Configuration for IngestCalibsTask"""
206 parse = ConfigurableField(target=CalibsParseTask, doc=
"File parsing")
207 register = ConfigurableField(target=CalibsRegisterTask, doc=
"Registry entry")
208 allowError = Field(dtype=bool, default=
False, doc=
"Allow error in ingestion?")
209 clobber = Field(dtype=bool, default=
False, doc=
"Clobber existing file?")
213 """Task that generates registry for calibration images"""
214 ConfigClass = IngestCalibsConfig
215 ArgumentParser = IngestCalibsArgumentParser
216 _DefaultName =
"ingestCalibs"
219 """Ingest all specified files and add them to the registry"""
220 calibRoot = args.calib
if args.calib
is not None else args.output
222 with self.register.openRegistry(calibRoot, create=args.create, dryrun=args.dryrun)
as registry:
224 for infile
in filenameList:
225 fileInfo, hduInfoList = self.parse.getInfo(infile)
226 calibType = self.parse.getCalibType(infile)
227 if calibType
not in self.register.config.tables:
228 self.log.warn(str(
"Skipped adding %s of observation type '%s' to registry "
229 "(must be one of %s)" %
230 (infile, calibType,
", ".join(self.register.config.tables))))
232 calibTypes.add(calibType)
233 if args.mode !=
'skip':
234 outfile = self.parse.getDestination(args.butler, fileInfo, infile)
235 ingested = self.
ingest(infile, outfile, mode=args.mode, dryrun=args.dryrun)
237 self.log.warn(str(
"Failed to ingest %s of observation type '%s'" %
238 (infile, calibType)))
240 if self.register.check(registry, fileInfo, table=calibType):
241 if args.ignoreIngested:
244 self.log.warn(
"%s: already ingested: %s" % (infile, fileInfo))
245 for info
in hduInfoList:
246 self.register.addRow(registry, info, dryrun=args.dryrun,
247 create=args.create, table=calibType)
249 self.register.updateValidityRanges(registry, args.validity, tables=calibTypes)
251 self.log.info(
"Would update validity ranges here, but dryrun")