12 #include "boost/multi_index_container.hpp"
13 #include "boost/multi_index/sequenced_index.hpp"
14 #include "boost/multi_index/ordered_index.hpp"
15 #include "boost/multi_index/hashed_index.hpp"
16 #include "boost/multi_index/member.hpp"
35 template <std::
string FitsSchemaItem::*Member>
36 struct SetFitsSchemaString {
37 void operator()(FitsSchemaItem &item) { item.*Member = _v; }
38 explicit SetFitsSchemaString(
std::string const &v) : _v(v) {}
52 using InputContainer = boost::multi_index_container<FitsSchemaItem, boost::multi_index::indexed_by<boost::multi_index::ordered_non_unique<boost::multi_index::member<FitsSchemaItem, int, &FitsSchemaItem::column>>, boost::multi_index::ordered_non_unique<boost::multi_index::member<FitsSchemaItem, int, &FitsSchemaItem::bit>>, boost::multi_index::hashed_unique<boost::multi_index::member<FitsSchemaItem, std::string, &FitsSchemaItem::ttype>>, boost::multi_index::sequenced<>>>;
55 using SetTTYPE = SetFitsSchemaString<&FitsSchemaItem::ttype>;
56 using SetTFORM = SetFitsSchemaString<&FitsSchemaItem::tform>;
57 using SetTCCLS = SetFitsSchemaString<&FitsSchemaItem::tccls>;
58 using SetTUNIT = SetFitsSchemaString<&FitsSchemaItem::tunit>;
59 using SetDoc = SetFitsSchemaString<&FitsSchemaItem::doc>;
91 : _impl(
std::make_shared<
Impl>()) {
94 if (!metadata.
exists(
"AFW_TYPE")) {
97 _impl->version = metadata.
get(
"AFW_TABLE_VERSION", _impl->version);
98 _impl->type = metadata.
get(
"AFW_TYPE", _impl->type);
100 metadata.
remove(
"AFW_TABLE_VERSION");
103 metadata.
remove(
"AFW_TYPE");
107 _impl->archiveHdu = metadata.
get(
"AR_HDU", -1);
108 if (_impl->archiveHdu > 0) {
111 metadata.
remove(
"AR_HDU");
118 for (
auto const &rawAliase : rawAliases) {
120 if (pos == std::string::npos) {
122 (boost::format(
"Malformed alias definition: '%s'") % rawAliase).str());
124 _impl->schema.getAliasMap()->set(rawAliase.substr(0, pos), rawAliase.substr(pos + 1, std::string::npos));
133 if (_impl->version == 0) {
142 for (
auto const &oldSlotKey : oldSlotKeys) {
145 _impl->schema.getAliasMap()->set(oldSlotKey.second,
target);
147 metadata.
remove(oldSlotKey.first);
148 metadata.
remove(oldSlotKey.first +
"_ERR_SLOT");
149 metadata.
remove(oldSlotKey.first +
"_FLAG_SLOT");
157 for (
auto const &key : keyList) {
158 if (key.compare(0, 5,
"TTYPE") == 0) {
159 int column =
std::stoi(key.substr(5)) - 1;
160 auto iter = _impl->byColumn().lower_bound(column);
161 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
166 if (
iter->doc.empty()) {
172 }
else if (key.compare(0, 5,
"TFLAG") == 0) {
174 auto iter = _impl->byBit().lower_bound(bit);
175 if (
iter == _impl->byBit().end() ||
iter->bit != bit) {
180 if (
iter->doc.empty()) {
186 }
else if (key.compare(0, 4,
"TDOC") == 0) {
187 int column =
std::stoi(key.substr(4)) - 1;
188 auto iter = _impl->byColumn().lower_bound(column);
189 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
196 }
else if (key.compare(0, 5,
"TFDOC") == 0) {
198 auto iter = _impl->byBit().lower_bound(bit);
199 if (
iter == _impl->byBit().end() ||
iter->bit != bit) {
206 }
else if (key.compare(0, 5,
"TUNIT") == 0) {
207 int column =
std::stoi(key.substr(5)) - 1;
208 auto iter = _impl->byColumn().lower_bound(column);
209 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
216 }
else if (key.compare(0, 5,
"TCCLS") == 0) {
217 int column =
std::stoi(key.substr(5)) - 1;
218 auto iter = _impl->byColumn().lower_bound(column);
219 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
226 }
else if (key.compare(0, 5,
"TFORM") == 0) {
227 int column =
std::stoi(key.substr(5)) - 1;
228 auto iter = _impl->byColumn().lower_bound(column);
229 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
236 }
else if (key.compare(0, 5,
"TZERO") == 0) {
240 }
else if (key.compare(0, 5,
"TSCAL") == 0) {
244 }
else if (key.compare(0, 5,
"TNULL") == 0) {
248 }
else if (key.compare(0, 5,
"TDISP") == 0) {
256 _impl->flagColumn = metadata.
get(
"FLAGCOL", 0);
257 if (_impl->flagColumn > 0) {
259 metadata.
remove(
"FLAGCOL");
262 auto iter = _impl->byColumn().find(_impl->flagColumn);
263 if (
iter == _impl->byColumn().end()) {
266 (boost::format(
"Column for flag data not found; FLAGCOL=%d") % _impl->flagColumn).str());
271 static std::regex const regex(
"(\\d+)?X\\(?(\\d)*\\)?");
276 (boost::format(
"Invalid TFORM key for flags column: '%s'") %
iter->tform).str());
282 _impl->flagKeys.resize(
nFlags);
283 _impl->flagWorkspace.reset(
new bool[
nFlags]);
286 _impl->byColumn().erase(
iter);
299 int oldHdu =
fits.getHdu();
300 if (_impl->archiveHdu < 0) _impl->archiveHdu = oldHdu + 1;
302 fits.setHdu(_impl->archiveHdu);
309 _impl->archiveHdu = -1;
317 auto iter = _impl->byName().find(ttype);
318 if (
iter == _impl->byName().end()) {
325 auto iter = _impl->byColumn().lower_bound(column);
326 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
333 auto iter = _impl->byColumn().lower_bound(item->
column);
334 assert(
iter != _impl->byColumn().end() &&
iter->column == item->
column);
335 _impl->byColumn().erase(
iter);
339 auto iter = _impl->byName().find(ttype);
340 if (
iter != _impl->byName().end() &&
iter->ttype == ttype) {
341 _impl->byName().erase(
iter);
346 auto iter = _impl->byColumn().lower_bound(column);
347 if (
iter != _impl->byColumn().end() &&
iter->column == column) {
348 _impl->byColumn().erase(
iter);
355 _impl->readers.push_back(
std::move(reader));
360 template <
typename T>
369 : _column(item.column), _key(
schema.addField<T>(item.ttype, item.doc, item.tunit, base)),
370 _cache(), _cacheFirstRow(0)
377 if (_key.getElementCount() == 1u) {
378 std::size_t nElements = nRows*_key.getElementCount();
379 _cache.resize(nElements);
380 _cacheFirstRow = firstRow;
381 fits.readTableArray(firstRow, _column, nElements, &_cache.front());
385 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
387 if (_cache.empty()) {
388 fits.readTableArray(row, _column, _key.getElementCount(), record.getElement(_key));
390 assert(row >= _cacheFirstRow);
392 assert(offset < _cache.size());
393 std::copy_n(_cache.begin() + offset, _key.getElementCount(), record.getElement(_key));
405 class AngleReader :
public FitsColumnReader {
408 Schema &
schema, FitsSchemaItem
const &item,
409 FieldBase<lsst::geom::Angle>
const &base = FieldBase<lsst::geom::Angle>()) {
413 AngleReader(Schema &
schema, FitsSchemaItem
const &item, FieldBase<lsst::geom::Angle>
const &base)
414 : _column(item.column), _key(
schema.addField<
lsst::geom::
Angle>(item.ttype, item.doc,
"", base)) {
419 if (!item.tunit.empty() && item.tunit !=
"rad") {
421 "Angle fields must be persisted in radians (TUNIT='rad').");
426 assert(_key.getElementCount() == 1u);
427 _cache.resize(nRows);
428 _cacheFirstRow = firstRow;
429 fits.readTableArray(firstRow, _column, nRows, &_cache.front());
432 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
434 if (_cache.empty()) {
436 fits.readTableScalar(row, _column, tmp);
439 assert(row >= _cacheFirstRow);
441 assert(offset < _cache.size());
448 Key<lsst::geom::Angle> _key;
453 class StringReader :
public FitsColumnReader {
459 StringReader(Schema &
schema, FitsSchemaItem
const &item,
int size)
460 : _column(item.column),
461 _key(
schema.addField<
std::string>(item.ttype, item.doc, item.tunit, size)),
462 _isVariableLength(size == 0) {}
464 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
467 fits.readTableScalar(row, _column, s, _isVariableLength);
473 Key<std::string> _key;
474 bool _isVariableLength;
477 template <
typename T>
478 class VariableLengthArrayReader :
public FitsColumnReader {
484 VariableLengthArrayReader(Schema &
schema, FitsSchemaItem
const &item)
485 : _column(item.column), _key(
schema.addField<Array<T>>(item.ttype, item.doc, item.tunit, 0)) {}
487 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
489 int size =
fits.getTableArraySize(row, _column);
490 ndarray::Array<T, 1, 1> array = ndarray::allocate(size);
491 fits.readTableArray(row, _column, size, array.getData());
492 record.set(_key, array);
502 template <
typename T>
503 class PointConversionReader :
public FitsColumnReader {
509 PointConversionReader(Schema &
schema, FitsSchemaItem
const &item)
510 : _column(item.column), _key(PointKey<T>::addFields(
schema, item.ttype, item.doc, item.tunit)) {}
512 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
515 fits.readTableArray(row, _column, 2, buffer.
data());
526 class CoordConversionReader :
public FitsColumnReader {
532 CoordConversionReader(Schema &
schema, FitsSchemaItem
const &item)
533 : _column(item.column), _key(CoordKey::addFields(
schema, item.ttype, item.doc)) {}
535 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
538 fits.readTableArray(row, _column, 2, buffer.
data());
549 class MomentsConversionReader :
public FitsColumnReader {
555 MomentsConversionReader(Schema &
schema, FitsSchemaItem
const &item)
556 : _column(item.column),
559 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
562 fits.readTableArray(row, _column, 3, buffer.
data());
563 record.set(_key, geom::ellipses::Quadrupole(buffer[0], buffer[1], buffer[2],
false));
574 template <
typename T,
int N>
575 class CovarianceConversionReader :
public FitsColumnReader {
578 static std::regex const regex(
"(.*)(\\^(\\d+))?");
593 CovarianceConversionReader(Schema &
schema, FitsSchemaItem
const &item,
595 : _column(item.column),
597 _key(CovarianceMatrixKey<T, N>::addFields(
schema, item.ttype, names, guessUnits(item.tunit))),
600 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
603 for (
int i = 0; i < _size; ++i) {
604 for (
int j = i; j < _size; ++j) {
613 CovarianceMatrixKey<T, N> _key;
621 static std::regex const regex(
"(\\d+)?([PQ])?([A-Z])\\(?(\\d)*\\)?");
631 char code =
m[3].str()[0];
641 if (item.tccls ==
"Array") {
642 return StandardReader<Array<std::uint8_t>>::make(
schema, item, size);
644 return StandardReader<std::uint8_t>::make(
schema, item);
647 return VariableLengthArrayReader<std::uint8_t>::make(
schema, item);
649 return StandardReader<Array<std::uint8_t>>::make(
schema, item, size);
654 if (item.tccls ==
"Array") {
655 return StandardReader<Array<std::uint16_t>>::make(
schema, item, size);
657 return StandardReader<std::uint16_t>::make(
schema, item);
660 return VariableLengthArrayReader<std::uint16_t>::make(
schema, item);
662 return StandardReader<Array<std::uint16_t>>::make(
schema, item, size);
665 return VariableLengthArrayReader<std::int32_t>::make(
schema, item);
667 if (item.tccls ==
"Point") {
668 return PointConversionReader<std::int32_t>::make(
schema, item);
670 if (size > 1 || item.tccls ==
"Array") {
671 return StandardReader<Array<std::int32_t>>::make(
schema, item, size);
673 return StandardReader<std::int32_t>::make(
schema, item);
676 return StandardReader<std::int64_t>::make(
schema, item);
680 return VariableLengthArrayReader<float>::make(
schema, item);
683 if (item.tccls ==
"Array") {
684 return StandardReader<Array<float>>::make(
schema, item, 1);
688 return StandardReader<float>::make(
schema, item);
690 if (size == 3 && item.tccls ==
"Covariance(Point)") {
692 return CovarianceConversionReader<float, 2>::make(
schema, item, names);
694 if (size == 6 && item.tccls ==
"Covariance(Moments)") {
696 return CovarianceConversionReader<float, 3>::make(
schema, item, names);
698 if (item.tccls ==
"Covariance") {
699 double v = 0.5 * (
std::sqrt(1 + 8 * size) - 1);
701 if (n * (n + 1) != size * 2) {
702 throw LSST_EXCEPT(afw::fits::FitsError,
"Covariance field has invalid size.");
708 return CovarianceConversionReader<float, Eigen::Dynamic>::make(
schema, item, names);
710 return StandardReader<Array<float>>::make(
schema, item, size);
713 return VariableLengthArrayReader<double>::make(
schema, item);
716 if (item.tccls ==
"Angle") {
717 return AngleReader::make(
schema, item);
719 if (item.tccls ==
"Array") {
720 return StandardReader<Array<double>>::make(
schema, item, 1);
722 return StandardReader<double>::make(
schema, item);
725 if (item.tccls ==
"Point") {
726 return PointConversionReader<double>::make(
schema, item);
728 if (item.tccls ==
"Coord") {
729 return CoordConversionReader::make(
schema, item);
732 if (size == 3 && item.tccls ==
"Moments") {
733 return MomentsConversionReader::make(
schema, item);
735 return StandardReader<Array<double>>::make(
schema, item, size);
738 return StringReader::make(
schema, item, size);
748 bool isInstFlux(FitsSchemaItem
const & item) {
753 if (!
includes(item.ttype,
"flux"))
return false;
754 if (
includes(item.ttype,
"modelfit_CModel") && item.tunit.empty()) {
761 std::transform(units.begin(), units.end(), units.begin(), [](
char c) { return std::tolower(c); } );
773 if (_impl->version == 0) {
774 AliasMap &aliases = *_impl->schema.getAliasMap();
775 for (
auto iter = _impl->asList().begin();
iter != _impl->asList().end(); ++
iter) {
777 if (flagPos != std::string::npos) {
786 ttype.
replace(flagPos, 5,
"flag");
793 }
else if (isInstFlux(*
iter)) {
795 if (endswith(
iter->ttype,
"_err")) {
796 aliases.
set(replace(
iter->ttype,
"_err",
"_instFluxErr"),
iter->ttype);
798 aliases.
set(
iter->ttype +
"_instFlux",
iter->ttype);
800 }
else if (endswith(
iter->ttype,
"_err")) {
806 if (
iter->tccls ==
"Covariance(Point)") {
807 aliases.
set(replace(
iter->ttype,
"_err",
"_yErr"),
iter->ttype +
"_yErr");
808 aliases.
set(replace(
iter->ttype,
"_err",
"_xErr"),
iter->ttype +
"_xErr");
809 aliases.
set(replace(
iter->ttype,
"_err",
"_x_y_Cov"),
iter->ttype +
"_x_y_Cov");
810 }
else if (
iter->tccls ==
"Covariance(Moments)") {
811 aliases.
set(replace(
iter->ttype,
"_err",
"_xxErr"),
iter->ttype +
"_xxErr");
812 aliases.
set(replace(
iter->ttype,
"_err",
"_yyErr"),
iter->ttype +
"_yyErr");
813 aliases.
set(replace(
iter->ttype,
"_err",
"_xyErr"),
iter->ttype +
"_xyErr");
814 aliases.
set(replace(
iter->ttype,
"_err",
"_xx_yy_Cov"),
iter->ttype +
"_xx_yy_Cov");
815 aliases.
set(replace(
iter->ttype,
"_err",
"_xx_xy_Cov"),
iter->ttype +
"_xx_xy_Cov");
816 aliases.
set(replace(
iter->ttype,
"_err",
"_yy_xy_Cov"),
iter->ttype +
"_yy_xy_Cov");
820 }
else if (_impl->version < 3) {
824 AliasMap &aliases = *_impl->schema.getAliasMap();
825 for (
auto iter = _impl->asList().begin();
iter != _impl->asList().end(); ++
iter) {
827 if (_impl->version < 2 && endswith(
name,
"Sigma")) {
830 if (_impl->version < 3 && isInstFlux(*
iter)) {
838 for (
auto iter = _impl->asList().begin();
iter != _impl->asList().end(); ++
iter) {
842 _impl->readers.push_back(
std::move(reader));
844 LOGLS_WARN(
"afw.FitsSchemaInputMapper",
"Format " <<
iter->tform <<
" for column "
846 <<
" not supported; skipping.");
851 (boost::format(
"Flag field '%s' is is in bit %d (0-indexed) of only %d") %
852 iter->ttype %
iter->bit % _impl->flagKeys.size())
855 _impl->flagKeys[
iter->bit] = _impl->schema.addField<Flag>(
iter->ttype,
iter->doc);
858 _impl->asList().clear();
859 if (_impl->schema.getRecordSize() <= 0) {
862 (boost::format(
"Non-positive record size: %d; file is corrupt or invalid.") %
863 _impl->schema.getRecordSize()).str()
867 return _impl->schema;
871 if (!_impl->flagKeys.empty()) {
872 fits.readTableArray<
bool>(row, _impl->flagColumn, _impl->flagKeys.size(), _impl->flagWorkspace.get());
873 for (
std::size_t bit = 0; bit < _impl->flagKeys.size(); ++bit) {
874 record.
set(_impl->flagKeys[bit], _impl->flagWorkspace[bit]);
877 if (_impl->nRowsToPrep != 1 && row % _impl->nRowsToPrep == 0) {
881 for (
auto const &reader : _impl->readers) {
882 reader->prepRead(row, size,
fits);
885 for (
auto const & reader : _impl->readers) {
886 reader->readCell(record, row,
fits, _impl->archive);
table::Key< std::string > name
Key< Flag > const & target
#define LSST_EXCEPT(type,...)
#define LOGLS_WARN(logger, message)
An exception thrown when problems are found when reading or writing FITS files.
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Mapping class that holds aliases for a Schema.
void set(std::string const &alias, std::string const &target)
Add an alias to the schema or replace an existing one.
Base class for all records.
void set(Key< T > const &key, U const &value)
Set value of a field for the given key.
Defines the fields and offsets for a table.
Polymorphic reader interface used to read different kinds of objects from one or more FITS binary tab...
T get(std::string const &name) const
std::string const & getComment(std::string const &name) const
std::vector< T > getArray(std::string const &name) const
std::vector< std::string > getOrderedNames() const
virtual void remove(std::string const &name)
bool exists(std::string const &name) const
std::size_t computeCovariancePackedSize(std::size_t size)
Defines the packed size of a covariance matrices.
std::size_t indexCovariance(std::size_t i, std::size_t j)
Defines the ordering of packed covariance matrices.
CoordinateType
Enum used to set units for geometric FunctorKeys.
constexpr AngleUnit radians
A base class for image defects.
Field base class default implementation (used for numeric scalars and lsst::geom::Angle).
A structure that describes a field as a collection of related strings read from the FITS header.