14int ttk::TopologicalCompression::CompressWithZFP(
16 const bool decompress,
17 std::vector<double> &array,
21 const double zfpTolerance)
const {
24 bool const is2D = nx == 1 || ny == 1 || nz == 1;
26 if(nx + ny == 2 || ny + nz == 2 || nx + nz == 2) {
27 this->printErr(
"One-dimensional arrays not supported.");
31 n1 = nx != 1 ? nx : ny;
32 n2 = nx != 1 && ny != 1 ? ny : nz;
36 const auto type = zfp_type_double;
41 field = zfp_field_2d(array.data(), type, n1, n2);
43 field = zfp_field_3d(array.data(), type, nx, ny, nz);
47 auto *zfp = zfp_stream_open(
nullptr);
52 zfp_stream_set_accuracy(zfp, zfpTolerance);
55 const auto bufsize = zfp_stream_maximum_size(zfp, field);
56 std::vector<unsigned char> buffer(bufsize);
59 auto *stream = stream_open(buffer.data(), bufsize);
60 zfp_stream_set_bit_stream(zfp, stream);
61 zfp_stream_rewind(zfp);
72 zfpsize += fread(buffer.data(), 1, bufsize, file);
75 const auto res = zfp_read_header(zfp, field, ZFP_HEADER_FULL);
77 this->printErr(
"Could not read ZFP header");
81 if(!zfp_decompress(zfp, field)) {
82 this->printErr(
"Decompression failed");
87 const auto res = zfp_write_header(zfp, field, ZFP_HEADER_FULL);
89 this->printErr(
"Could not write ZFP header");
94 zfpsize += zfp_compress(zfp, field);
96 this->printErr(
"Compression failed");
99 fwrite(buffer.data(), 1, zfpsize, file);
103 zfp_field_free(field);
104 zfp_stream_close(zfp);
105 stream_close(stream);
108 this->printErr(
"Encountered a problem with ZFP.");
116#ifdef TTK_ENABLE_ZLIB
120unsigned long ttk::TopologicalCompression::GetZlibDestLen(
121 const unsigned long sourceLen)
const {
122 return compressBound(sourceLen);
125void ttk::TopologicalCompression::CompressWithZlib(
128 unsigned long &destLen,
129 const unsigned char *
const source,
130 const unsigned long sourceLen)
const {
133 uncompress(dest, &destLen, source, sourceLen);
135 compress(dest, &destLen, source, sourceLen);
145 unsigned int ret = 0;
157 std::vector<int> &segmentation,
158 int &numberOfVertices,
159 int &numberOfSegments)
const {
161 int numberOfBytesRead = 0;
163 numberOfBytesRead +=
sizeof(int);
164 numberOfVertices = Read<int32_t>(fm);
166 numberOfBytesRead +=
sizeof(int);
167 numberOfSegments = Read<int32_t>(fm);
169 unsigned int const numberOfBitsPerSegment = log2(numberOfSegments) + 1;
171#ifndef TTK_ENABLE_KAMIKAZE
173 if(numberOfBitsPerSegment == 0) {
179 if(numberOfBitsPerSegment > 32)
186 int oldCompressedInt = 0;
188 while(currentCell < numberOfVertices) {
191 numberOfBytesRead +=
sizeof(int);
192 compressedInt = Read<int32_t>(fm);
194 while(offset + numberOfBitsPerSegment <= 32) {
197 int currentSegment = compressedInt;
199 if(maskerRank == 0) {
200 currentSegment <<= (32 - offset - numberOfBitsPerSegment);
202 if(currentSegment < 0) {
203 currentSegment &= 2147483647;
204 currentSegment >>= (32 - numberOfBitsPerSegment);
205 currentSegment |= (1 << (numberOfBitsPerSegment - 1));
207 currentSegment >>= (32 - numberOfBitsPerSegment);
210 offset += numberOfBitsPerSegment;
215 currentSegment <<= (32 - offset);
216 if(currentSegment < 0) {
217 currentSegment &= 2147483647;
218 currentSegment >>= (32 - numberOfBitsPerSegment);
219 currentSegment |= (1 << (numberOfBitsPerSegment - 1));
221 currentSegment >>= (32 - numberOfBitsPerSegment);
224 int const nextSegment = currentSegment;
226 if(oldCompressedInt < 0) {
227 oldCompressedInt &= 2147483647;
228 oldCompressedInt >>= maskerRank;
229 oldCompressedInt |= (1 << (32 - maskerRank - 1));
231 oldCompressedInt >>= maskerRank;
234 currentSegment = nextSegment | oldCompressedInt;
238 segmentation.push_back(currentSegment);
244 oldCompressedInt = compressedInt;
249 offset += numberOfBitsPerSegment;
255 return numberOfBytesRead;
261 const std::vector<int> &segmentation,
262 int numberOfVertices,
263 int numberOfSegments)
const {
265 int numberOfBytesWritten = 0;
269 unsigned int const numberOfBitsPerSegment = log2(numberOfSegments) + 1;
272 if(numberOfBitsPerSegment > 32)
279 while(currentCell < numberOfVertices) {
282 int compressedInt = 0;
286 while(offset + numberOfBitsPerSegment <= 32) {
290 int currentSegment = segmentation[currentCell];
293 if(maskerRank != 0) {
295 currentSegment = currentSegment >> (numberOfBitsPerSegment - offset);
298 compressedInt |= currentSegment;
301 int const cursor = (currentSegment << offset);
302 compressedInt |= cursor;
303 offset += numberOfBitsPerSegment;
310 if(currentCell >= numberOfVertices) {
311 numberOfBytesWritten +=
sizeof(int);
312 Write<int32_t>(fm, compressedInt);
319 int const currentSegment = segmentation[currentCell];
324 int const cursor = (currentSegment << offset);
325 compressedInt = compressedInt | cursor;
327 maskerRank = 32 - offset;
328 offset += numberOfBitsPerSegment;
335 numberOfBytesWritten +=
sizeof(int);
336 Write<int32_t>(fm, compressedInt);
339 return numberOfBytesWritten;
344 std::vector<std::tuple<double, int>> &mappings,
345 std::vector<std::tuple<double, int>> &mappingsSortedPerValue,
346 std::vector<std::tuple<int, double, int>> &constraints,
349 int &nbConstraints)
const {
351 int numberOfBytesRead = 0;
355 numberOfBytesRead +=
sizeof(int);
356 mappingSize = Read<int32_t>(fm);
358 for(
int i = 0; i < mappingSize; ++i) {
360 numberOfBytesRead +=
sizeof(int);
361 idv = Read<int32_t>(fm);
364 numberOfBytesRead +=
sizeof(double);
365 value = Read<double>(fm);
367 mappings.emplace_back(value, idv);
368 mappingsSortedPerValue.emplace_back(value, idv);
372 std::sort(mappings.begin(), mappings.end(), cmp);
373 std::sort(mappingsSortedPerValue.begin(), mappingsSortedPerValue.end(), cmp2);
376 numberOfBytesRead +=
sizeof(int);
377 nbConstraints = Read<int32_t>(fm);
379 for(
int i = 0; i < nbConstraints; ++i) {
384 numberOfBytesRead +=
sizeof(int);
385 idVertex = Read<int32_t>(fm);
387 numberOfBytesRead +=
sizeof(double);
388 value = Read<double>(fm);
390 numberOfBytesRead +=
sizeof(int);
391 vertexType = Read<int32_t>(fm);
403 constraints.emplace_back(idVertex, value, vertexType);
406 return numberOfBytesRead;
411 std::vector<std::tuple<double, int>> &mapping,
412 std::vector<std::tuple<int, double, int>> &constraints)
const {
414 int numberOfBytesWritten = 0;
417 auto mappingSize = (int)mapping.size();
418 numberOfBytesWritten +=
sizeof(int);
419 Write<int32_t>(fm, mappingSize);
422 for(
int i = 0; i < mappingSize; ++i) {
423 std::tuple<double, int> t = mapping[i];
424 int const idv = std::get<1>(t);
425 numberOfBytesWritten +=
sizeof(int);
426 Write<int32_t>(fm, idv);
428 auto value = std::get<0>(t);
429 numberOfBytesWritten +=
sizeof(double);
430 Write<double>(fm, value);
433 auto nbConstraints = (int)constraints.size();
434 numberOfBytesWritten +=
sizeof(int);
435 Write<int32_t>(fm, nbConstraints);
437 for(
int i = 0; i < nbConstraints; ++i) {
438 std::tuple<int, double, int> t = constraints[i];
439 int const idVertex = std::get<0>(t);
440 auto value = std::get<1>(t);
441 int const vertexType = std::get<2>(t);
443 numberOfBytesWritten +=
sizeof(int);
444 Write<int32_t>(fm, idVertex);
446 numberOfBytesWritten +=
sizeof(double);
447 Write<double>(fm, value);
449 numberOfBytesWritten +=
sizeof(int);
450 Write<int32_t>(fm, vertexType);
453 return numberOfBytesWritten;
456int ttk::TopologicalCompression::ComputeTotalSizeForPersistenceDiagram(
457 std::vector<std::tuple<double, int>> &mapping,
458 std::vector<std::tuple<int, double, int>> &criticalConstraints,
462 double zfpTolerance)
const {
468 int const numberOfBitsPerSegment = log2(nSegments) + 1;
469 double const nbCharPerSegment = (double)numberOfBitsPerSegment / 8.0;
470 totalSize += (
sizeof(int) * 2 + std::ceil(nbCharPerSegment * nVertices));
473 auto mappingSize = (int)mapping.size();
474 auto constraintsSize = (int)criticalConstraints.size();
475 totalSize += (mappingSize) * (
sizeof(
int) +
sizeof(double)) +
sizeof(int);
477 += (constraintsSize) * (2 *
sizeof(
int) +
sizeof(double)) +
sizeof(int);
481 totalSize += zfpTolerance > 0.0 ? nVertices *
sizeof(double) : 0 + 2;
486int ttk::TopologicalCompression::WritePersistenceTopology(FILE *fm) {
487 int numberOfBytesWritten = 0;
489 int const numberOfVertices = getNbVertices();
490 int const numberOfSegments = getNbSegments();
493 if(numberOfSegments < 1)
496 numberOfBytesWritten +=
sizeof(int);
497 Write<int32_t>(fm, numberOfVertices);
499 numberOfBytesWritten +=
sizeof(int);
500 Write<int32_t>(fm, numberOfSegments);
502 numberOfBytesWritten += WriteCompactSegmentation(
503 fm, getSegmentation(), numberOfVertices, numberOfSegments);
505 rawFileLength += numberOfBytesWritten;
510int ttk::TopologicalCompression::WritePersistenceGeometry(FILE *fm,
514 double *toCompress) {
515 int numberOfBytesWritten = 0;
521 += WritePersistenceIndex(fm, mapping_, criticalConstraints_);
524 this->
printMsg(
"Wrote raw geometry.");
526 if(zfpTolerance >= 0.0) {
529 int const nx = 1 + dataExtent[1] - dataExtent[0];
530 int const ny = 1 + dataExtent[3] - dataExtent[2];
531 int const nz = 1 + dataExtent[5] - dataExtent[4];
533 std::vector<double> dataVector(toCompress, toCompress + (nx * ny * nz));
535 += CompressWithZFP(fm,
false, dataVector, nx, ny, nz, zfpTolerance);
541 this->printErr(
"Attempted to write with ZFP but ZFP is not installed.");
546 rawFileLength += numberOfBytesWritten;
551int ttk::TopologicalCompression::ReadPersistenceTopology(FILE *fm) {
552 int numberOfSegments;
553 int numberOfVertices;
555 int const numberOfBytesRead = ReadCompactSegmentation(
556 fm, segmentation_, numberOfVertices, numberOfSegments);
558 this->rawFileLength += numberOfBytesRead;
567int ttk::TopologicalCompression::ComputeTotalSizeForOther()
const {
580int ttk::TopologicalCompression::WriteOtherTopology(
582 this->printWrn(
"Writing Other index / topology.");
587int ttk::TopologicalCompression::WriteOtherGeometry(
589 this->printWrn(
"Writing Other buffer / geometry.");
594int ttk::TopologicalCompression::ReadOtherTopology(FILE *
ttkNotUsed(fm))
const {
595 this->printWrn(
"Reading Other index / topology.");
600int ttk::TopologicalCompression::ReadOtherGeometry(FILE *
ttkNotUsed(fm))
const {
601 this->printWrn(
"Reading Other buffer / geometry.");
613 const char *sqMethod,
621 const std::string &dataArrayName) {
623 WriteMetaData(fp, compressionType, zfpOnly, sqMethod, dataType, dataExtent,
624 dataSpacing, dataOrigin, tolerance, zfpTolerance,
627#ifdef TTK_ENABLE_ZLIB
628 Write<uint8_t>(fp,
true);
630 Write<uint8_t>(fp,
false);
633 bool const usePersistence
637 int numberOfVertices = 1;
638 for(
int i = 0; i < 3; ++i)
639 numberOfVertices *= (1 + dataExtent[2 * i + 1] - dataExtent[2 * i]);
640 NbVertices = numberOfVertices;
642 int const totalSize = usePersistence ? ComputeTotalSizeForPersistenceDiagram(
643 getMapping(), getCriticalConstraints(), zfpOnly,
644 getNbSegments(), getNbVertices(), zfpTolerance)
645 : useOther ? ComputeTotalSizeForOther()
648 std::vector<char> bbuf(totalSize);
649 char *buf = bbuf.data();
650 size_t const len = (size_t)totalSize;
655 const std::string s = fileName + std::string(
".temp");
656 const char *ffn = s.c_str();
657 FILE *fm = fopen(ffn,
"wb");
663 WritePersistenceTopology(fm);
665 WriteOtherTopology(fm);
668 this->
printMsg(
"Topology successfully written to buffer.");
674 = WritePersistenceGeometry(fm, dataExtent, zfpOnly, zfpTolerance, data);
676 status = WriteOtherGeometry(fm);
680 fm = fopen(ffn,
"rb");
681 int const ret = fread(buf, len,
sizeof(
char), fm);
687 this->
printMsg(
"Geometry successfully written to buffer.");
689 this->printErr(
"Geometry was not successfully written to buffer.");
696 if(totalSize < rawFileLength) {
697 this->printErr(
"Invalid total size (" + std::to_string(totalSize) +
" vs "
698 + std::to_string(rawFileLength) +
").");
701#ifdef TTK_ENABLE_ZLIB
703 auto sourceLen =
static_cast<unsigned long>(rawFileLength);
704 const auto source =
reinterpret_cast<unsigned char *
>(buf);
705 auto destLen = GetZlibDestLen(sourceLen);
706 std::vector<unsigned char> ddest(destLen);
707 CompressWithZlib(
false, ddest.data(), destLen, source, sourceLen);
708 this->
printMsg(
"Data successfully compressed.");
711 Write<uint64_t>(fp, destLen);
712 Write<uint64_t>(fp, sourceLen);
713 WriteByteArray(fp, ddest.data(), destLen);
714 this->
printMsg(
"Data successfully written to filesystem.");
717 this->
printMsg(
"ZLIB not found, writing raw file.");
718 unsigned char *source =
reinterpret_cast<unsigned char *
>(buf);
719 unsigned long sourceLen = (
unsigned long)rawFileLength;
720 unsigned long destLen = (
unsigned long)rawFileLength;
722 Write<uint64_t>(fp, destLen);
723 Write<uint64_t>(fp, sourceLen);
724 WriteByteArray(fp, source, destLen);
737 const char *sqMethod,
744 const std::string &dataArrayName) {
747 WriteByteArray(fp, magicBytes_, std::strlen(magicBytes_));
750 Write<uint64_t>(fp, formatVersion_);
753 Write<int32_t>(fp, compressionType);
756 Write<uint8_t>(fp, zfpOnly);
759 const char *sq = sqMethod;
760 int const sqType = (strcmp(sq,
"") == 0) ? 0
761 : (strcmp(sq,
"r") == 0 || strcmp(sq,
"R") == 0) ? 1
762 : (strcmp(sq,
"d") == 0 || strcmp(sq,
"D") == 0) ? 2
765 Write<int32_t>(fp, sqType);
768 Write<int32_t>(fp, dataType);
771 for(
int i = 0; i < 6; ++i)
772 Write<int32_t>(fp, dataExtent[i]);
774 for(
int i = 0; i < 3; ++i)
775 Write<double>(fp, dataSpacing[i]);
777 for(
int i = 0; i < 3; ++i)
778 Write<double>(fp, dataOrigin[i]);
781 Write<double>(fp, tolerance);
784 Write<double>(fp, zfpTolerance);
788 Write<uint64_t>(fp, dataArrayName.size());
791 WriteByteArray(fp, dataArrayName.c_str(), dataArrayName.size());
793 this->
printMsg(
"Metadata successfully written.");
801 const auto magicBytesLen{std::strlen(magicBytes_)};
802 std::vector<char> mBytes(magicBytesLen + 1);
803 mBytes[magicBytesLen] =
'\0';
804 ReadByteArray(fm, mBytes.data(), magicBytesLen);
807 const bool hasMagicBytes = strcmp(mBytes.data(), magicBytes_) == 0;
809 this->printErr(
"Could not find magic bytes in input file!");
814 const auto fileVersion = Read<uint64_t>(fm);
815 if(fileVersion < this->formatVersion_) {
816 this->printErr(
"Old format version detected (" + std::to_string(fileVersion)
817 +
" vs. " + std::to_string(this->formatVersion_) +
").");
818 this->printErr(
"Older formats are not supported!");
820 }
else if(fileVersion > this->formatVersion_) {
821 this->printErr(
"Newer format version detected ("
822 + std::to_string(fileVersion) +
" vs. "
823 + std::to_string(this->formatVersion_) +
").");
824 this->printErr(
"Cannot read file with current TTK, try with to update.");
829 compressionType_ = Read<int32_t>(fm);
832 ZFPOnly = Read<uint8_t>(fm);
835 SQMethodInt = Read<int32_t>(fm);
838 dataScalarType_ = Read<int32_t>(fm);
842 for(
int i = 0; i < 6; ++i)
843 dataExtent_[i] = Read<int32_t>(fm);
845 for(
int i = 0; i < 3; ++i)
846 dataSpacing_[i] = Read<double>(fm);
848 for(
int i = 0; i < 3; ++i)
849 dataOrigin_[i] = Read<double>(fm);
852 Tolerance = Read<double>(fm);
855 ZFPTolerance = Read<double>(fm);
858 size_t const dataArrayNameLength = Read<uint64_t>(fm);
861 dataArrayName_.resize(dataArrayNameLength + 1);
862 dataArrayName_[dataArrayNameLength] =
'\0';
863 ReadByteArray(fm, dataArrayName_.data(), dataArrayNameLength);
#define TTK_FORCE_USE(x)
Force the compiler to use the function/method parameter.
#define ttkNotUsed(x)
Mark function/method parameters that are not used in the function body at all.
void setDebugMsgPrefix(const std::string &prefix)
int ReadPersistenceIndex(FILE *fm, std::vector< std::tuple< double, int > > &mappings, std::vector< std::tuple< double, int > > &mappingsSortedPerValue, std::vector< std::tuple< int, double, int > > &constraints, double &min, double &max, int &nbConstraints) const
int WriteCompactSegmentation(FILE *fm, const std::vector< int > &segmentation, int numberOfVertices, int numberOfSegments) const
int ReadMetaData(FILE *fm)
int WritePersistenceIndex(FILE *fm, std::vector< std::tuple< double, int > > &mapping, std::vector< std::tuple< int, double, int > > &constraints) const
int WriteToFile(FILE *fp, int compressionType, bool zfpOnly, const char *sqMethod, int dataType, int *dataExtent, double *dataSpacing, double *dataOrigin, double *data, double tolerance, double zfpTolerance, const std::string &dataArrayName)
int ReadCompactSegmentation(FILE *fm, std::vector< int > &segmentation, int &numberOfVertices, int &numberOfSegments) const
int WriteMetaData(FILE *fp, int compressionType, bool zfpOnly, const char *sqMethod, int dataType, int *dataExtent, double *dataSpacing, double *dataOrigin, double tolerance, double zfpTolerance, const std::string &dataArrayName)
static unsigned int log2(int val)
printMsg(debug::output::BOLD+" | | | | | . \\ | | (__| | / __/| |_| / __/|__ _|"+debug::output::ENDCOLOR, debug::Priority::PERFORMANCE, debug::LineMode::NEW, stream)