4from dateutil
import parser
12def _convertToDate(dateString):
13 """Convert a string into a date object"""
14 return parser.parse(dateString).date()
18 """Task that will parse the filename and/or its contents to get the
19 required information to populate the calibration registry."""
22 """Return a a known calibration dataset type using
23 the observation type in the header keyword OBSTYPE
25 @param filename: Input filename
27 md = readMetadata(filename, self.config.hdu)
28 if not md.exists(
"OBSTYPE"):
29 raise RuntimeError(
"Unable to find the required header keyword OBSTYPE in %s, hdu %d" %
30 (filename, self.config.hdu))
31 obstype = md.getScalar(
"OBSTYPE").strip().lower()
34 elif "zero" in obstype
or "bias" in obstype:
36 elif "dark" in obstype:
38 elif "fringe" in obstype:
40 elif "sky" in obstype:
42 elif "illumcor" in obstype:
44 elif "defects" in obstype:
46 elif "qe_curve" in obstype:
48 elif "linearizer" in obstype:
49 obstype =
"linearizer"
50 elif "crosstalk" in obstype:
52 elif "BFK" in obstype:
57 """Get destination for the file
59 @param butler Data butler
60 @param info File properties, used
as dataId
for the butler
61 @param filename Input filename
62 @return Destination filename
67 tempinfo = {k: v
for (k, v)
in info.items()
if v
is not None}
69 raw = butler.get(calibType +
"_filename", tempinfo)[0]
78 """Configuration for the CalibsRegisterTask"""
79 tables = ListField(dtype=str, default=[
"bias",
"dark",
"flat",
"fringe",
"sky",
"defects",
"qe_curve",
80 "linearizer",
"crosstalk",
"bfk"], doc=
"Names of tables")
81 calibDate = Field(dtype=str, default=
"calibDate", doc=
"Name of column for calibration date")
82 validStart = Field(dtype=str, default=
"validStart", doc=
"Name of column for validity start")
83 validEnd = Field(dtype=str, default=
"validEnd", doc=
"Name of column for validity stop")
84 detector = ListField(dtype=str, default=[
"filter",
"ccd"],
85 doc=
"Columns that identify individual detectors")
86 validityUntilSuperseded = ListField(dtype=str, default=[
"defects",
"qe_curve",
"linearizer",
"crosstalk",
88 doc=
"Tables for which to set validity for a calib from when it is "
89 "taken until it is superseded by the next; validity in other tables "
90 "is calculated by applying the validity range.")
91 incrementValidEnd = Field(
94 doc=
"Fix the off-by-one error by incrementing validEnd. See "
95 "fixSubsetValidity for more details.",
100 """Task that will generate the calibration registry for the Mapper"""
101 ConfigClass = CalibsRegisterConfig
103 def openRegistry(self, directory, create=False, dryrun=False, name="calibRegistry.sqlite3"):
104 """Open the registry and return the connection handle"""
105 return RegisterTask.openRegistry(self, directory, create, dryrun, name)
108 """Create the registry tables"""
109 for table
in self.config.tables:
110 RegisterTask.createTable(self, conn, table=table, forceCreateTables=forceCreateTables)
112 def addRow(self, conn, info, *args, **kwargs):
113 """Add a row to the file table"""
114 info[self.config.validStart] =
None
115 info[self.config.validEnd] =
None
116 RegisterTask.addRow(self, conn, info, *args, **kwargs)
119 """Loop over all tables, filters, and ccdnums,
120 and update the validity ranges
in the registry.
122 @param conn: Database connection
123 @param validity: Validity range (days)
125 conn.row_factory = sqlite3.Row
126 cursor = conn.cursor()
128 tables = self.config.tables
130 sql =
"SELECT DISTINCT %s FROM %s" % (
", ".join(self.config.detector), table)
132 rows = cursor.fetchall()
137 """Update the validity ranges among selected rows in the registry.
139 For defects and qe_curve, the products are valid
from their start date until
140 they are superseded by subsequent defect data.
141 For other calibration products, the validity ranges are checked
and
142 if there are overlaps, a midpoint
is used to fix the overlaps,
143 so that the calibration data
with whose date
is nearest the date
144 of the observation
is used.
146 DM generated calibrations contain a CALIB_ID header
147 keyword. These calibrations likely require the
148 incrementValidEnd configuration option set to
True. Other
149 calibrations generate the calibDate via the DATE-OBS header
150 keyword,
and likely require incrementValidEnd=
False.
152 @param conn: Database connection
153 @param table: Name of table to be selected
154 @param detectorData: Values identifying a detector (
from columns
in self.config.detector)
155 @param validity: Validity range (days)
157 columns = ", ".join([self.config.calibDate, self.config.validStart, self.config.validEnd])
158 sql =
"SELECT id, %s FROM %s" % (columns, table)
159 sql +=
" WHERE " +
" AND ".join(col +
"=?" for col
in self.config.detector)
160 sql +=
" ORDER BY " + self.config.calibDate
161 cursor = conn.cursor()
162 cursor.execute(sql, detectorData)
163 rows = cursor.fetchall()
166 valids = collections.OrderedDict([(_convertToDate(row[self.config.calibDate]), [
None,
None])
for
169 det =
" ".join(
"%s=%s" % (k, v)
for k, v
in zip(self.config.detector, detectorData))
170 self.log.warning(
"Skipped setting the validity overlaps for %s %s: missing calibration dates",
173 dates = list(valids.keys())
174 if table
in self.config.validityUntilSuperseded:
176 for thisDate, nextDate
in zip(dates[:-1], dates[1:]):
177 valids[thisDate][0] = thisDate
178 valids[thisDate][1] = nextDate
179 valids[dates[-1]][0] = dates[-1]
180 valids[dates[-1]][1] = _convertToDate(
"2037-12-31")
184 valids[dd] = [dd - datetime.timedelta(validity), dd + datetime.timedelta(validity)]
187 midpoints = [t1 + (t2 - t1)//2
for t1, t2
in zip(dates[:-1], dates[1:])]
188 for i, (date, midpoint)
in enumerate(zip(dates[:-1], midpoints)):
189 if valids[date][1] > midpoint:
190 nextDate = dates[i + 1]
191 valids[nextDate][0] = midpoint + datetime.timedelta(1)
192 if self.config.incrementValidEnd:
193 valids[date][1] = midpoint + datetime.timedelta(1)
195 valids[date][1] = midpoint
200 calibDate = _convertToDate(row[self.config.calibDate])
201 validStart = valids[calibDate][0].isoformat()
202 validEnd = valids[calibDate][1].isoformat()
203 sql =
"UPDATE %s" % table
204 sql +=
" SET %s=?, %s=?" % (self.config.validStart, self.config.validEnd)
206 conn.execute(sql, (validStart, validEnd, row[
"id"]))
210 """Argument parser to support ingesting calibration images into the repository"""
213 InputOnlyArgumentParser.__init__(self, *args, **kwargs)
214 self.add_argument(
"-n",
"--dry-run", dest=
"dryrun", action=
"store_true",
215 default=
False, help=
"Don't perform any action?")
216 self.add_argument(
"--mode", choices=[
"move",
"copy",
"link",
"skip"], default=
"move",
217 help=
"Mode of delivering the files to their destination")
218 self.add_argument(
"--create", action=
"store_true", help=
"Create new registry?")
219 self.add_argument(
"--validity", type=int, required=
True, help=
"Calibration validity period (days)")
220 self.add_argument(
"--ignore-ingested", dest=
"ignoreIngested", action=
"store_true",
221 help=
"Don't register files that have already been registered")
222 self.add_argument(
"files", nargs=
"+", help=
"Names of file")
226 """Configuration for IngestCalibsTask"""
227 parse = ConfigurableField(target=CalibsParseTask, doc=
"File parsing")
228 register = ConfigurableField(target=CalibsRegisterTask, doc=
"Registry entry")
229 allowError = Field(dtype=bool, default=
False, doc=
"Allow error in ingestion?")
230 clobber = Field(dtype=bool, default=
False, doc=
"Clobber existing file?")
234 """Task that generates registry for calibration images"""
235 ConfigClass = IngestCalibsConfig
236 ArgumentParser = IngestCalibsArgumentParser
237 _DefaultName =
"ingestCalibs"
240 """Ingest all specified files and add them to the registry"""
241 calibRoot = args.calib
if args.calib
is not None else args.output
243 with self.register.openRegistry(calibRoot, create=args.create, dryrun=args.dryrun)
as registry:
245 for infile
in filenameList:
246 fileInfo, hduInfoList = self.parse.getInfo(infile)
247 calibType = self.parse.getCalibType(infile)
248 if calibType
not in self.register.config.tables:
249 self.log.warning(
"Skipped adding %s of observation type '%s' to registry "
250 "(must be one of %s)",
251 infile, calibType,
", ".join(self.register.config.tables))
253 calibTypes.add(calibType)
254 if args.mode !=
'skip':
255 outfile = self.parse.getDestination(args.butler, fileInfo, infile)
256 ingested = self.
ingest(infile, outfile, mode=args.mode, dryrun=args.dryrun)
258 self.log.warning(
"Failed to ingest %s of observation type '%s'",
261 if self.register.check(registry, fileInfo, table=calibType):
262 if args.ignoreIngested:
265 self.log.warning(
"%s: already ingested: %s", infile, fileInfo)
266 for info
in hduInfoList:
267 self.register.addRow(registry, info, dryrun=args.dryrun,
268 create=args.create, table=calibType)
270 self.register.updateValidityRanges(registry, args.validity, tables=calibTypes)
272 self.log.info(
"Would update validity ranges here, but dryrun")
def expandFiles(self, fileNameList)
Expand a set of filenames and globs, returning a list of filenames.
def ingest(self, infile, outfile, mode="move", dryrun=False)
def getCalibType(self, filename)
def getDestination(self, butler, info, filename)
def fixSubsetValidity(self, conn, table, detectorData, validity)
def addRow(self, conn, info, *args, **kwargs)
def openRegistry(self, directory, create=False, dryrun=False, name="calibRegistry.sqlite3")
def createTable(self, conn, forceCreateTables=False)
def updateValidityRanges(self, conn, validity, tables=None)
def __init__(self, *args, **kwargs)