BitMagic-C++
bmsparsevec_serial.h
Go to the documentation of this file.
1 #ifndef BMSPARSEVEC_SERIAL__H__INCLUDED__
2 #define BMSPARSEVEC_SERIAL__H__INCLUDED__
3 /*
4 Copyright(c) 2002-2017 Anatoliy Kuznetsov(anatoliy_kuznetsov at yahoo.com)
5 
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
9 
10  http://www.apache.org/licenses/LICENSE-2.0
11 
12 Unless required by applicable law or agreed to in writing, software
13 distributed under the License is distributed on an "AS IS" BASIS,
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 See the License for the specific language governing permissions and
16 limitations under the License.
17 
18 For more information please visit: http://bitmagic.io
19 */
20 
21 /*! \file bmsparsevec_serial.h
22  \brief Serialization for sparse_vector<>
23 */
24 
25 
26 #ifndef BM__H__INCLUDED__
27 // BitMagic utility headers do not include main "bm.h" declaration
28 // #include "bm.h" or "bm64.h" explicitly
29 # error missing include (bm.h or bm64.h)
30 #endif
31 
32 #include "bmsparsevec.h"
33 #include "bmserial.h"
34 #include "bmbuffer.h"
35 #include "bmdef.h"
36 
37 namespace bm
38 {
39 
40 /** \defgroup svserial Sparse vector serialization
41  Sparse vector serialization
42  \ingroup svector
43  */
44 
45 
46 /*!
47  \brief layout class for serialization buffer structure
48 
49  Class keeps a memory block sized for the target sparse vector BLOB.
50  This class also provides acess to bit-plane memory, so it becomes possible
51  to use parallel storage methods to save bit-plains into
52  different storage shards.
53 
54  \ingroup svserial
55 */
56 template<class SV>
58 {
59  typedef typename SV::value_type value_type;
60  typedef typename SV::bvector_type bvector_type;
62 
64 
66 
67  /*!
68  \brief resize capacity
69  \param capacity - new capacity
70  \return new buffer or 0 if failed
71  */
72  unsigned char* reserve(size_t capacity)
73  {
74  if (capacity == 0)
75  {
76  freemem();
77  return 0;
78  }
79  buf_.reinit(capacity);
80  return buf_.data();
81  }
82 
83  /// return current serialized size
84  size_t size() const { return buf_.size(); }
85 
86  /// Set new serialized size
87  void resize(size_t ssize) { buf_.resize(ssize); }
88 
89  /// return serialization buffer capacity
90  size_t capacity() const { return buf_.capacity(); }
91 
92  /// free memory
93  void freemem() { buf_.release(); }
94 
95  /// Set plain output pointer and size
96  void set_plain(unsigned i, unsigned char* ptr, size_t buf_size)
97  {
98  plain_ptrs_[i] = ptr;
99  plane_size_[i] = buf_size;
100  }
101 
102  /// Get plain pointer
103  const unsigned char* get_plain(unsigned i) const { return plain_ptrs_[i]; }
104 
105  /// Return serialization buffer pointer
106  const unsigned char* buf() const { return buf_.buf(); }
107  /// Return serialization buffer pointer
108  const unsigned char* data() const { return buf_.buf(); }
109 
110 private:
112  void operator=(const sparse_vector_serial_layout&);
113 protected:
114  buffer_type buf_; ///< serialization buffer
115  unsigned char* plain_ptrs_[SV::sv_plains]; ///< pointers on serialized bit-plains
116  size_t plane_size_[SV::sv_plains]; ///< serialized plain size
117 };
118 
119 // -------------------------------------------------------------------------
120 
121 /*!
122  \brief Serialize sparse vector into a memory buffer(s) structure
123 
124  Serialization format:
125 
126  | HEADER | BIT-VECTORS ... | REMAP_MATRIX
127 
128  Header structure:
129  -----------------
130  BYTE+BYTE: Magic-signature 'BM' or 'BC' (c-compressed)
131  BYTE : Byte order ( 0 - Big Endian, 1 - Little Endian)
132  {
133  BYTE : Number of Bit-vector plains (total) (non-zero when < 255 plains)
134  |
135  BYTE: zero - flag of large plain matrix
136  INT64: Nnmber of bit-vector plains
137  }
138  INT64: Vector size
139  INT64: Offset of plain 0 from the header start (value 0 means plain is empty)
140  INT64: Offset of plain 1 from
141  ...
142  INT32: reserved
143 
144 Bit-vectors:
145 ------------
146  Based on current bit-vector serialization
147 
148 Remap Matrix:
149  SubHeader | Matrix BLOB
150 
151  sub-header:
152  BYTE: 'R' (remapping) or 'N' (no remapping)
153  N - means no other info is saved on the stream
154  INT64: remap matrix size
155 
156  \ingroup svector
157  \ingroup svserial
158 */
159 template<typename SV>
161 {
162 public:
163  typedef typename SV::bvector_type bvector_type;
166  typedef typename SV::value_type value_type;
167  typedef typename SV::size_type size_type;
170 
171 public:
173 
174  /**
175  Add skip-markers for faster range deserialization
176 
177  @param enable - TRUE searilization will add bookmark codes
178  @param bm_interval - bookmark interval in (number of blocks)
179  (suggested between 4 and 512)
180  smaller interval means more bookmarks added to the skip list thus
181  more increasing the BLOB size
182  */
183  void set_bookmarks(bool enable, unsigned bm_interval = 256)
184  { bvs_.set_bookmarks(enable, bm_interval); }
185 
186  /// Turn ON and OFF XOR compression of sparse vectors
187  void set_xor_ref(bool is_enabled) { is_xor_ref_ = is_enabled; }
188 
189  /// Get XOR reference compression status (enabled/disabled)
190  bool is_xor_ref() const { return is_xor_ref_; }
191 
192  /*!
193  \brief Serialize sparse vector into a memory buffer(s) structure
194 
195  \param sv - sparse vector to serialize
196  \param sv_layout - buffer structure to keep the result
197  as defined in bm::serialization_flags
198  */
199  void serialize(const SV& sv,
201 
202 protected:
203  typedef typename
205 
206  void build_xor_ref_vector(const SV& sv);
207 
208 private:
210  sparse_vector_serializer& operator=(const sparse_vector_serializer&) = delete;
211 protected:
213 
216 };
217 
218 /**
219  sparse vector de-serializer
220 
221 */
222 template<typename SV>
224 {
225 public:
226  typedef typename SV::bvector_type bvector_type;
229  typedef typename SV::value_type value_type;
230  typedef typename SV::size_type size_type;
232 
233 public:
236 
237  /*!
238  Deserialize sparse vector
239 
240  @param sv - [out] target sparse vector to populate
241  @param buf - source memory pointer
242  */
243  void deserialize(SV& sv, const unsigned char* buf)
244  { idx_range_set_ = false; deserialize_sv(sv, buf, 0); }
245 
246  /*!
247  Deserialize sparse vector for the range [from, to]
248 
249  @param sv - [out] target sparse vector to populate
250  @param buf - source memory pointer
251  @param from - start vector index for deserialization range
252  @param to - end vector index for deserialization range
253  */
254  void deserialize_range(SV& sv, const unsigned char* buf,
255  size_type from, size_type to);
256 
257  void deserialize(SV& sv, const unsigned char* buf,
258  size_type from, size_type to)
259  {
260  deserialize_range(sv, buf, from, to);
261  }
262 
263 
264  /*!
265  Deserialize sparse vector using address mask vector
266  Address mask defines (by set bits) which vector elements to be extracted
267  from the compressed BLOB
268 
269  @param sv - [out] target sparse vector to populate
270  @param buf - source memory pointer
271  @param mask_bv - AND mask bit-vector (address gather vector)
272  */
273  void deserialize(SV& sv,
274  const unsigned char* buf,
275  const bvector_type& mask_bv)
276  { idx_range_set_ = false; deserialize_sv(sv, buf, &mask_bv); }
277 
278 
279 protected:
282 
283 
284  /// Deserialize header/version and other common info
285  ///
286  /// @return number of bit-plains
287  ///
288  unsigned load_header(bm::decoder& dec, SV& sv, unsigned char& matr_s_ser);
289 
290  void deserialize_sv(SV& sv, const unsigned char* buf,
291  const bvector_type* mask_bv);
292 
293 
294  /// deserialize bit-vector plains
295  void deserialize_plains(SV& sv, unsigned plains,
296  const unsigned char* buf,
297  const bvector_type* mask_bv = 0);
298 
299  /// load offset table
300  void load_plains_off_table(bm::decoder& dec, unsigned plains);
301 
302  /// load NULL bit-plain (returns new plains count)
303  int load_null_plain(SV& sv,
304  int plains,
305  const unsigned char* buf,
306  const bvector_type* mask_bv);
307 
308  /// load string remap dict
309  static void load_remap(SV& sv, const unsigned char* remap_buf_ptr);
310 
311  /// throw error on incorrect deserialization
312  static void raise_invalid_header();
313  /// throw error on incorrect deserialization
314  static void raise_invalid_64bit();
315  /// throw error on incorrect deserialization
316  static void raise_invalid_bitdepth();
317 
318 private:
320  sparse_vector_deserializer& operator=(const sparse_vector_deserializer&) = delete;
321 protected:
322  const unsigned char* remap_buf_ptr_;
331  bm::heap_vector<size_t, alloc_type, true> off_vect_;
332 
334 
338 };
339 
340 
341 
342 /*!
343  \brief Serialize sparse vector into a memory buffer(s) structure
344 
345  \param sv - sparse vector to serialize
346  \param sv_layout - buffer structure to keep the result
347  \param temp_block - temporary buffer
348  (allocate with BM_DECLARE_TEMP_BLOCK(x) for speed)
349 
350  \ingroup svserial
351 
352  @sa serialization_flags
353  @sa sparse_vector_deserializer
354 */
355 template<class SV>
357  const SV& sv,
359  bm::word_t* temp_block = 0)
360 {
361  (void)temp_block;
362  bm::sparse_vector_serializer<SV> sv_serializer;
363  sv_serializer.serialize(sv, sv_layout);
364 }
365 
366 // -------------------------------------------------------------------------
367 
368 /*!
369  \brief Deserialize sparse vector
370  \param sv - target sparse vector
371  \param buf - source memory buffer
372  \param temp_block - temporary block buffer to avoid re-allocations
373 
374  \return 0 (error processing via std::logic_error)
375 
376  \ingroup svserial
377  @sa sparse_vector_deserializer
378 */
379 template<class SV>
381  const unsigned char* buf,
382  bm::word_t* temp_block=0)
383 {
384  (void)temp_block;
385  bm::sparse_vector_deserializer<SV> sv_deserializer;
386  sv_deserializer.deserialize(sv, buf);
387  return 0;
388 }
389 
390 // -------------------------------------------------------------------------
391 
392 /**
393  Seriaizer for compressed collections
394 */
395 template<class CBC>
397 {
398 public:
400  typedef typename CBC::bvector_type bvector_type;
401  typedef typename CBC::buffer_type buffer_type;
402  typedef typename CBC::statistics statistics_type;
403  typedef typename CBC::address_resolver_type address_resolver_type;
404 
405 public:
406  void serialize(const CBC& buffer_coll,
407  buffer_type& buf,
408  bm::word_t* temp_block = 0);
409 };
410 
411 /**
412  Deseriaizer for compressed collections
413 */
414 template<class CBC>
416 {
417 public:
419  typedef typename CBC::bvector_type bvector_type;
421  typedef typename CBC::buffer_type buffer_type;
422  typedef typename CBC::statistics statistics_type;
423  typedef typename CBC::address_resolver_type address_resolver_type;
424  typedef typename CBC::container_type container_type;
425 
426 public:
427  int deserialize(CBC& buffer_coll,
428  const unsigned char* buf,
429  bm::word_t* temp_block=0);
430 };
431 
432 
433 // -------------------------------------------------------------------------
434 
435 /**
436  \brief Serialize compressed collection into memory buffer
437 
438 Serialization format:
439 
440 
441 <pre>
442  | MAGIC_HEADER | ADDRESS_BITVECTROR | LIST_OF_BUFFER_SIZES | BUFFER(s)
443 
444  MAGIC_HEADER:
445  BYTE+BYTE: Magic-signature 'BM' or 'BC'
446  BYTE : Byte order ( 0 - Big Endian, 1 - Little Endian)
447 
448  ADDRESS_BITVECTROR:
449  INT64: address bit-vector size
450  [memblock]: serialized address bit-vector
451 
452  LIST_OF_BUFFER_SIZES:
453  INT64 - buffer sizes count
454  INT32 - buffer size 0
455  INT32 - buffer size 1
456  ...
457 
458  BUFFERS:
459  [memblock]: block0
460  [memblock]: block1
461  ...
462 
463 </pre>
464 */
465 
466 template<class CBC>
468  buffer_type& buf,
469  bm::word_t* temp_block)
470 {
471  statistics_type st;
472  buffer_coll.calc_stat(&st);
473 
474  buf.resize(st.max_serialize_mem);
475 
476  // ptr where bit-plains start
477  unsigned char* buf_ptr = buf.data();
478 
479  bm::encoder enc(buf.data(), buf.capacity());
481  enc.put_8('B');
482  enc.put_8('C');
483  enc.put_8((unsigned char)bo);
484 
485  unsigned char* mbuf1 = enc.get_pos(); // bookmark position
486  enc.put_64(0); // address vector size (reservation)
487 
488  buf_ptr = enc.get_pos();
489 
490  const address_resolver_type& addr_res = buffer_coll.resolver();
491  const bvector_type& bv = addr_res.get_bvector();
492  {
493  bm::serializer<bvector_type > bvs(temp_block);
494  bvs.gap_length_serialization(false);
495 
496  size_t addr_bv_size = bvs.serialize(bv, buf_ptr, buf.size());
497  buf_ptr += addr_bv_size;
498 
499  enc.set_pos(mbuf1); // rewind to bookmark
500  enc.put_64(addr_bv_size); // save the address vector size
501  }
502  enc.set_pos(buf_ptr); // restore stream position
503  size_t coll_size = buffer_coll.size();
504 
505  enc.put_64(coll_size);
506 
507  // pass 1 (save buffer sizes)
508  {
509  for (unsigned i = 0; i < buffer_coll.size(); ++i)
510  {
511  const buffer_type& cbuf = buffer_coll.get(i);
512  size_t sz = cbuf.size();
513  enc.put_64(sz);
514  } // for i
515  }
516  // pass 2 (save buffers)
517  {
518  for (unsigned i = 0; i < buffer_coll.size(); ++i)
519  {
520  const buffer_type& cbuf = buffer_coll.get(i);
521  size_t sz = cbuf.size();
522  enc.memcpy(cbuf.buf(), sz);
523  } // for i
524  }
525  buf.resize(enc.size());
526 }
527 
528 // -------------------------------------------------------------------------
529 template<class CBC>
531  CBC& buffer_coll,
532  const unsigned char* buf,
533  bm::word_t* temp_block)
534 {
535  // TODO: implement correct processing of byte-order corect deserialization
536  // ByteOrder bo_current = globals<true>::byte_order();
537 
538  bm::decoder dec(buf);
539  unsigned char h1 = dec.get_8();
540  unsigned char h2 = dec.get_8();
541 
542  BM_ASSERT(h1 == 'B' && h2 == 'C');
543  if (h1 != 'B' && h2 != 'C') // no magic header? issue...
544  {
545  return -1;
546  }
547  //unsigned char bv_bo =
548  dec.get_8();
549 
550  // -----------------------------------------
551  // restore address resolver
552  //
553  bm::id64_t addr_bv_size = dec.get_64();
554 
555  const unsigned char* bv_buf_ptr = dec.get_pos();
556 
557  address_resolver_type& addr_res = buffer_coll.resolver();
558  bvector_type& bv = addr_res.get_bvector();
559  bv.clear();
560 
561  bm::deserialize(bv, bv_buf_ptr, temp_block);
562  addr_res.sync();
563 
564  typename bvector_type::size_type addr_cnt = bv.count();
565  dec.seek((int)addr_bv_size);
566 
567  // -----------------------------------------
568  // read buffer sizes
569  //
570  bm::id64_t coll_size = dec.get_64();
571  if (coll_size != addr_cnt)
572  {
573  return -2; // buffer size collection does not match address vector
574  }
575 
576  typedef size_t vect_size_type;
577  bm::heap_vector<bm::id64_t, allocator_type, true> buf_size_vec;
578 
579  buf_size_vec.resize(vect_size_type(coll_size));
580  {
581  for (unsigned i = 0; i < coll_size; ++i)
582  {
583  bm::id64_t sz = dec.get_64();
584  buf_size_vec[i] = sz;
585  } // for i
586  }
587 
588  {
589  container_type& buf_vect = buffer_coll.container();
590  buf_vect.resize(vect_size_type(coll_size));
591  for (unsigned i = 0; i < coll_size; ++i)
592  {
593  bm::id64_t sz = buf_size_vec[i];
594  buffer_type& b = buf_vect.at(i);
595  b.resize(sz);
596  dec.memcpy(b.data(), sz);
597  } // for i
598  }
599  buffer_coll.sync();
600  return 0;
601 }
602 
603 // -------------------------------------------------------------------------
604 //
605 // -------------------------------------------------------------------------
606 
607 template<typename SV>
609 {
610  bvs_.gap_length_serialization(false);
611  #ifdef BMXORCOMP
612  is_xor_ref_ = true;
613  #else
614  is_xor_ref_ = false;
615  #endif
616 }
617 
618 // -------------------------------------------------------------------------
619 
620 template<typename SV>
622 {
623  bv_ref_.reset();
624  bv_ref_.build(sv.get_bmatrix());
625 }
626 
627 // -------------------------------------------------------------------------
628 
629 template<typename SV>
632 {
633  typename SV::statistics sv_stat;
634  sv.calc_stat(&sv_stat);
635  unsigned char* buf = sv_layout.reserve(sv_stat.max_serialize_mem);
636 
637  bm::encoder enc(buf, (unsigned)sv_layout.capacity());
638  unsigned plains = sv.stored_plains();
639 
640  // header size in bytes
641  unsigned h_size = 1 + 1 + // "BM" or "BC" (magic header)
642  1 + // byte-order
643  1 + // number of bit-plains (for vector)
644  8 + // size (internal 64-bit)
645  (8 * plains) + // offsets of all plains
646  4; // reserve
647  // for large plain matrixes
648  {
649  h_size += 1 + // version number
650  8; // number of plains (64-bit)
651  }
652 
653  // ---------------------------------
654  // Setup XOR reference compression
655  //
656  if (is_xor_ref())
657  {
658  build_xor_ref_vector(sv);
659  bvs_.set_ref_vectors(&bv_ref_);
660  }
661 
662  // -----------------------------------------------------
663  // Serialize all bvector plains
664  //
665 
666  unsigned char* buf_ptr = buf + h_size; // ptr where plains start (start+hdr)
667 
668  unsigned i;
669  for (i = 0; i < plains; ++i)
670  {
671  typename SV::bvector_type_const_ptr bv = sv.get_plain(i);
672  if (!bv) // empty plain
673  {
674  sv_layout.set_plain(i, 0, 0);
675  continue;
676  }
677  if (is_xor_ref())
678  {
679  unsigned idx = (unsigned)bv_ref_.find(i);
680  BM_ASSERT(idx != bv_ref_.not_found());
681  bvs_.set_curr_ref_idx(idx);
682  }
683 
684  size_t buf_size =
685  bvs_.serialize(*bv, buf_ptr, sv_stat.max_serialize_mem);
686 
687  sv_layout.set_plain(i, buf_ptr, buf_size);
688  buf_ptr += buf_size;
689  if (sv_stat.max_serialize_mem > buf_size)
690  {
691  sv_stat.max_serialize_mem -= buf_size;
692  continue;
693  }
694  BM_ASSERT(0); // TODO: throw an exception here
695  } // for i
696 
697  bvs_.set_ref_vectors(0); // disangage XOR ref vector
698 
699  // -----------------------------------------------------
700  // serialize the re-map matrix
701  //
702  if (bm::conditional<SV::is_remap_support::value>::test()) // test remapping trait
703  {
704  bm::encoder enc_m(buf_ptr, sv_stat.max_serialize_mem);
705  if (sv.is_remap())
706  {
707  bm::id64_t remap_size = sv.remap_size();
708  const unsigned char* matrix_buf = sv.get_remap_buffer();
709  BM_ASSERT(matrix_buf);
710  BM_ASSERT(remap_size);
711 
712  enc_m.put_8('R');
713  enc_m.put_64(remap_size);
714  enc_m.memcpy(matrix_buf, remap_size);
715  enc_m.put_8('E'); // end of matrix (integrity check token)
716  }
717  else
718  {
719  enc_m.put_8('N');
720  }
721  buf_ptr += enc_m.size(); // add mattrix encoded data size
722  }
723 
724  sv_layout.resize(size_t(buf_ptr - buf)); // set the true occupied size
725 
726  // -----------------------------------------------------
727  // save the header
728  //
730 
731  enc.put_8('B'); // magic header 'BM' - bit matrix 'BC' - bit compressed
732  if (sv.is_compressed())
733  enc.put_8('C');
734  else
735  enc.put_8('M');
736 
737  enc.put_8((unsigned char)bo); // byte order
738 
739  unsigned char matr_s_ser = 1;
740 #ifdef BM64ADDR
741  matr_s_ser = 2;
742 #endif
743 
744  enc.put_8(0); // number of plains == 0 (legacy magic number)
745  enc.put_8(matr_s_ser); // matrix serialization version
746  enc.put_64(plains); // number of rows in the bit-matrix
747  enc.put_64(sv.size_internal());
748 
749  // save the offset table (part of the header)
750  //
751  for (i = 0; i < plains; ++i)
752  {
753  const unsigned char* p = sv_layout.get_plain(i);
754  if (!p)
755  {
756  enc.put_64(0);
757  continue;
758  }
759  size_t offset = size_t(p - buf);
760  enc.put_64(offset);
761  } // for
762 }
763 
764 // -------------------------------------------------------------------------
765 //
766 // -------------------------------------------------------------------------
767 
768 template<typename SV>
770  : remap_buf_ptr_(0)
771 {
772  temp_block_ = alloc_.alloc_bit_block();
773  not_null_mask_bv_.set_allocator_pool(&pool_);
774  rsc_mask_bv_.set_allocator_pool(&pool_);
775  idx_range_set_ = false;
776 }
777 
778 // -------------------------------------------------------------------------
779 
780 template<typename SV>
782 {
783  if (temp_block_)
784  alloc_.free_bit_block(temp_block_);
785 }
786 
787 // -------------------------------------------------------------------------
788 
789 template<typename SV>
791  const unsigned char* buf,
792  size_type from, size_type to)
793 {
794  idx_range_set_ = true; idx_range_from_ = from; idx_range_to_ = to;
795 
796  remap_buf_ptr_ = 0;
797  bm::decoder dec(buf); // TODO: implement correct processing of byte-order
798 
799  unsigned char matr_s_ser = 0;
800  unsigned plains = load_header(dec, sv, matr_s_ser);
801 
802  sv.clear();
803 
804  bm::id64_t sv_size = dec.get_64();
805  if (sv_size == 0)
806  return; // empty vector
807 
808  sv.resize_internal(size_type(sv_size));
809  bv_ref_.reset();
810 
811  load_plains_off_table(dec, plains); // read the offset vector of bit-plains
812 
813  // TODO: add range for not NULL plane
814  plains = (unsigned)load_null_plain(sv, int(plains), buf, 0);
815 
816  // check if mask needs to be relaculated using the NULL (index) vector
818  {
819  // recalculate plains range
820  size_type sv_left, sv_right;
821  bool range_valid = sv.resolve_range(from, to, &sv_left, &sv_right);
822  if (!range_valid)
823  {
824  sv.clear();
825  idx_range_set_ = false;
826  return;
827  }
828  else
829  {
830  idx_range_set_ = true; idx_range_from_ = sv_left; idx_range_to_ = sv_right;
831  }
832  }
833 
834  deserialize_plains(sv, plains, buf, 0);
835 
836  op_deserial_.set_ref_vectors(0);
837  deserial_.set_ref_vectors(0);
838  bv_ref_.reset();
839 
840 
841  // load the remap matrix
842  //
843  if (bm::conditional<SV::is_remap_support::value>::test()) // test remap trait
844  {
845  if (matr_s_ser)
846  load_remap(sv, remap_buf_ptr_);
847  } // if remap traits
848 
849  sv.sync(true); // force sync, recalculate RS index, remap tables, etc
850  remap_buf_ptr_ = 0;
851 
852  idx_range_set_ = false;
853 }
854 
855 // -------------------------------------------------------------------------
856 
857 template<typename SV>
859  const unsigned char* buf,
860  const bvector_type* mask_bv)
861 {
862  remap_buf_ptr_ = 0;
863  bm::decoder dec(buf); // TODO: implement correct processing of byte-order
864 
865  unsigned char matr_s_ser = 0;
866  unsigned plains = load_header(dec, sv, matr_s_ser);
867 
868  sv.clear();
869 
870  bm::id64_t sv_size = dec.get_64();
871  if (sv_size == 0)
872  return; // empty vector
873 
874  sv.resize_internal(size_type(sv_size));
875  bv_ref_.reset();
876 
877  load_plains_off_table(dec, plains); // read the offset vector of bit-plains
878 
879  plains = (unsigned)load_null_plain(sv, int(plains), buf, mask_bv);
880 
881  // check if mask needs to be relaculated using the NULL (index) vector
883  {
884  if (mask_bv)
885  {
886  const bvector_type* bv_null = sv.get_null_bvector();
887  BM_ASSERT(bv_null);
888  rsc_mask_bv_.clear(true);
889  not_null_mask_bv_.bit_and(*bv_null, *mask_bv, bvector_type::opt_compress);
890  rsc_compressor_.compress(rsc_mask_bv_, *bv_null, not_null_mask_bv_);
891  mask_bv = &rsc_mask_bv_;
892 
893  // if it needs range recalculation
894  if (idx_range_set_) // range setting is in effect
895  {
896  //bool rf =
897  rsc_mask_bv_.find_range(idx_range_from_, idx_range_to_);
898  }
899  }
900  }
901 
902  deserialize_plains(sv, plains, buf, mask_bv);
903 
904  op_deserial_.set_ref_vectors(0);
905  deserial_.set_ref_vectors(0);
906  bv_ref_.reset();
907 
908 
909  // load the remap matrix
910  //
911  if (bm::conditional<SV::is_remap_support::value>::test()) // test remap trait
912  {
913  if (matr_s_ser)
914  load_remap(sv, remap_buf_ptr_);
915  } // if remap traits
916 
917  sv.sync(true); // force sync, recalculate RS index, remap tables, etc
918  remap_buf_ptr_ = 0;
919 }
920 
921 // -------------------------------------------------------------------------
922 
923 template<typename SV>
925  bm::decoder& dec, SV& sv, unsigned char& matr_s_ser)
926 {
927  unsigned char h1 = dec.get_8();
928  unsigned char h2 = dec.get_8();
929 
930  BM_ASSERT(h1 == 'B' && (h2 == 'M' || h2 == 'C'));
931 
932  bool sig2_ok = (h2 == 'M' || h2 == 'C');
933  if (h1 != 'B' || !sig2_ok) //&& (h2 != 'M' || h2 != 'C')) // no magic header?
934  raise_invalid_header();
935 
936  unsigned char bv_bo = dec.get_8(); (void) bv_bo;
937  unsigned plains = dec.get_8();
938  if (plains == 0) // bit-matrix
939  {
940  matr_s_ser = dec.get_8(); // matrix serialization version
941  plains = (unsigned) dec.get_64(); // number of rows in the bit-matrix
942  }
943  #ifdef BM64ADDR
944  #else
945  if (matr_s_ser == 2) // 64-bit matrix
946  {
947  raise_invalid_64bit();
948  }
949  #endif
950 
951  unsigned sv_plains = sv.stored_plains();
952  if (!plains || plains > sv_plains)
953  raise_invalid_bitdepth();
954  return plains;
955 }
956 
957 // -------------------------------------------------------------------------
958 
959 template<typename SV>
961  SV& sv,
962  unsigned plains,
963  const unsigned char* buf,
964  const bvector_type* mask_bv)
965 {
966  if (mask_bv && !idx_range_set_)
967  idx_range_set_ = mask_bv->find_range(idx_range_from_, idx_range_to_);
968 
969  op_deserial_.set_ref_vectors(&bv_ref_);
970  deserial_.set_ref_vectors(&bv_ref_);
971 
972  // read-deserialize the plains based on offsets
973  // backward order to bring the NULL vector first
974  //
975  for (int i = int(plains-1); i >= 0; --i)
976  {
977  size_t offset = off_vect_[unsigned(i)];
978  if (!offset) // empty vector
979  continue;
980  const unsigned char* bv_buf_ptr = buf + offset; // seek to position
981  bvector_type* bv = sv.get_plain(unsigned(i));
982  BM_ASSERT(bv);
983 
984  bv_ref_.add(bv, unsigned(i));
985 
986  if (mask_bv) // gather mask set, use AND operation deserializer
987  {
988  typename bvector_type::mem_pool_guard mp_g_z(pool_, *bv);
989 
991  && !remap_buf_ptr_) // last plain vector (special case)
992  {
993  size_t read_bytes =
994  deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
995  remap_buf_ptr_ = bv_buf_ptr + read_bytes;
996  bv->bit_and(*mask_bv, bvector_type::opt_compress);
997  continue;
998  }
999  if (idx_range_set_)
1000  deserial_.set_range(idx_range_from_, idx_range_to_);
1001  deserial_.deserialize(*bv, bv_buf_ptr);
1002  bv->bit_and(*mask_bv, bvector_type::opt_compress);
1003  }
1004  else
1005  {
1007  !remap_buf_ptr_)
1008  {
1009  size_t read_bytes =
1010  deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1011  remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1012  if (idx_range_set_)
1013  bv->keep_range(idx_range_from_, idx_range_to_);
1014  continue;
1015  }
1016  if (idx_range_set_)
1017  {
1018  deserial_.set_range(idx_range_from_, idx_range_to_);
1019  deserial_.deserialize(*bv, bv_buf_ptr);
1020  bv->keep_range(idx_range_from_, idx_range_to_);
1021  }
1022  else
1023  {
1024  //size_t read_bytes =
1025  deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1026  }
1027  }
1028 
1029  } // for i
1030  deserial_.unset_range();
1031 
1032 }
1033 
1034 // -------------------------------------------------------------------------
1035 
1036 template<typename SV>
1038  int plains,
1039  const unsigned char* buf,
1040  const bvector_type* mask_bv)
1041 {
1042  BM_ASSERT(plains > 0);
1043  if (!sv.is_nullable())
1044  return plains;
1045  int i = plains - 1;
1046  size_t offset = off_vect_[unsigned(i)];
1047  if (offset)
1048  {
1049  // TODO: improve serialization format to avoid non-range decode of
1050  // the NULL vector just to get to the offset of remap table
1051 
1052  const unsigned char* bv_buf_ptr = buf + offset; // seek to position
1053  bvector_type* bv = sv.get_plain(unsigned(i));
1054  bv_ref_.add(bv, unsigned(i));
1056  {
1057  // load the whole not-NULL vector regardless of range
1058  // TODO: load [0, idx_range_to_]
1059  size_t read_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1060  remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1061  }
1062  else // non-compressed SV
1063  {
1064  // NULL plain in string vector with substitute re-map
1065  //
1067  {
1068  BM_ASSERT(!remap_buf_ptr_);
1069  size_t read_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1070  remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1071  if (idx_range_set_)
1072  bv->keep_range(idx_range_from_, idx_range_to_);
1073  }
1074  else
1075  if (idx_range_set_)
1076  {
1077  deserial_.set_range(idx_range_from_, idx_range_to_);
1078  deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1079  bv->keep_range(idx_range_from_, idx_range_to_);
1080  deserial_.unset_range();
1081  }
1082  else
1083  {
1084  deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1085  }
1086  if (mask_bv)
1087  bv->bit_and(*mask_bv, bvector_type::opt_compress);
1088  }
1089  }
1090  return plains-1;
1091 }
1092 
1093 // -------------------------------------------------------------------------
1094 
1095 template<typename SV>
1097  bm::decoder& dec, unsigned plains)
1098 {
1099  off_vect_.resize(plains);
1100  for (unsigned i = 0; i < plains; ++i)
1101  {
1102  size_t offset = (size_t) dec.get_64();
1103  off_vect_[i] = offset;
1104  } // for i
1105 }
1106 
1107 // -------------------------------------------------------------------------
1108 
1109 template<typename SV>
1111  const unsigned char* remap_buf_ptr)
1112 {
1113  if (!remap_buf_ptr)
1114  return;
1115 
1116  bm::decoder dec_m(remap_buf_ptr);
1117  unsigned char rh = dec_m.get_8();
1118  switch (rh)
1119  {
1120  case 'N':
1121  break;
1122  case 'R':
1123  {
1124  size_t remap_size = (size_t) dec_m.get_64();
1125  unsigned char* remap_buf = sv.init_remap_buffer();
1126  BM_ASSERT(remap_buf);
1127  size_t target_remap_size = sv.remap_size();
1128  if (!remap_size || !remap_buf || remap_size != target_remap_size)
1129  {
1130  #ifndef BM_NO_STL
1131  throw std::logic_error("Invalid serialization format (remap size)");
1132  #else
1133  BM_THROW(BM_ERR_SERIALFORMAT);
1134  #endif
1135  }
1136  dec_m.memcpy(remap_buf, remap_size);
1137  unsigned char end_tok = dec_m.get_8();
1138  if (end_tok != 'E')
1139  {
1140  #ifndef BM_NO_STL
1141  throw std::logic_error("Invalid serialization format");
1142  #else
1143  BM_THROW(BM_ERR_SERIALFORMAT);
1144  #endif
1145  }
1146  sv.set_remap();
1147  }
1148  break;
1149  default:
1150  #ifndef BM_NO_STL
1151  throw std::logic_error("Invalid serialization format (remap error)");
1152  #else
1153  BM_THROW(BM_ERR_SERIALFORMAT);
1154  #endif
1155  } // switch
1156 }
1157 
1158 // -------------------------------------------------------------------------
1159 
1160 template<typename SV>
1162 {
1163 #ifndef BM_NO_STL
1164  throw std::logic_error("Invalid serialization signature header");
1165 #else
1166  BM_THROW(BM_ERR_SERIALFORMAT);
1167 #endif
1168 }
1169 
1170 // -------------------------------------------------------------------------
1171 
1172 template<typename SV>
1174 {
1175 #ifndef BM_NO_STL
1176  throw std::logic_error("Invalid serialization target (64-bit BLOB)");
1177 #else
1178  BM_THROW(BM_ERR_SERIALFORMAT);
1179 #endif
1180 }
1181 
1182 // -------------------------------------------------------------------------
1183 
1184 template<typename SV>
1186 {
1187 #ifndef BM_NO_STL
1188  throw std::logic_error("Invalid serialization target (bit depth)");
1189 #else
1190  BM_THROW(BM_ERR_SERIALFORMAT);
1191 #endif
1192 }
1193 
1194 // -------------------------------------------------------------------------
1195 
1196 } // namespace bm
1197 
1198 #include "bmundef.h"
1199 
1200 #endif
bm::compressed_collection_serializer::buffer_type
CBC::buffer_type buffer_type
Definition: bmsparsevec_serial.h:401
bm::compressed_collection_serializer::address_resolver_type
CBC::address_resolver_type address_resolver_type
Definition: bmsparsevec_serial.h:403
bm::deserialize
size_t deserialize(BV &bv, const unsigned char *buf, bm::word_t *temp_block=0, const bm::bv_ref_vector< BV > *ref_vect=0)
Bitvector deserialization from a memory BLOB.
Definition: bmserial.h:2902
bm::sparse_vector_deserializer::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:229
bm::bvector::mem_pool_guard
Definition: bm.h:785
bm::sparse_vector_serial_layout::plane_size_
size_t plane_size_[SV::sv_plains]
serialized plain size
Definition: bmsparsevec_serial.h:116
bm::sparse_vector_deserializer::raise_invalid_bitdepth
static void raise_invalid_bitdepth()
throw error on incorrect deserialization
Definition: bmsparsevec_serial.h:1185
bm::sparse_vector_deserializer::load_header
unsigned load_header(bm::decoder &dec, SV &sv, unsigned char &matr_s_ser)
Deserialize header/version and other common info.
Definition: bmsparsevec_serial.h:924
bm::sparse_vector_serializer::bv_ref_
bv_ref_vector_type bv_ref_
Definition: bmsparsevec_serial.h:215
bm::sparse_vector_deserializer::load_plains_off_table
void load_plains_off_table(bm::decoder &dec, unsigned plains)
load offset table
Definition: bmsparsevec_serial.h:1096
bm::sparse_vector_deserializer
sparse vector de-serializer
Definition: bmsparsevec_serial.h:223
bm::decoder_base::get_pos
const unsigned char * get_pos() const BMNOEXCEPT
Return current buffer pointer.
Definition: encoding.h:101
bm::sparse_vector_deserializer::load_remap
static void load_remap(SV &sv, const unsigned char *remap_buf_ptr)
load string remap dict
Definition: bmsparsevec_serial.h:1110
bm::sparse_vector_deserializer::alloc_
alloc_type alloc_
Definition: bmsparsevec_serial.h:323
bm::sparse_vector_serial_layout::buf_
buffer_type buf_
serialization buffer
Definition: bmsparsevec_serial.h:114
bm::sparse_vector_serializer::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:163
bm::encoder::put_64
void put_64(bm::id64_t w) BMNOEXCEPT
Puts 64 bits word into encoding buffer.
Definition: encoding.h:568
bm::sparse_vector_deserializer::not_null_mask_bv_
bvector_type not_null_mask_bv_
Definition: bmsparsevec_serial.h:329
bm::sparse_vector_serial_layout::buf
const unsigned char * buf() const
Return serialization buffer pointer.
Definition: bmsparsevec_serial.h:106
bm::sparse_vector_serializer::is_xor_ref
bool is_xor_ref() const
Get XOR reference compression status (enabled/disabled)
Definition: bmsparsevec_serial.h:190
bm::sparse_vector_serializer::allocator_pool_type
bvector_type::allocator_type::allocator_pool_type allocator_pool_type
Definition: bmsparsevec_serial.h:169
bm::id64_t
unsigned long long int id64_t
Definition: bmconst.h:34
bmsparsevec.h
Sparse constainer sparse_vector<> for integer types using bit-transposition transform.
bm::serializer::serialize
size_type serialize(const BV &bv, unsigned char *buf, size_t buf_size)
Bitvector serialization into memory block.
Definition: bmserial.h:2440
bm::encoder::size
size_t size() const BMNOEXCEPT
Returns size of the current encoding stream.
Definition: encoding.h:491
bm::sparse_vector_serial_layout::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:60
bm::sparse_vector_deserializer::bv_ref_
bv_ref_vector_type bv_ref_
Definition: bmsparsevec_serial.h:333
bm::sparse_vector_serializer::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:166
bm::compressed_collection_deserializer::container_type
CBC::container_type container_type
Definition: bmsparsevec_serial.h:424
bm::sparse_vector_deserializer::alloc_type
bvector_type::allocator_type alloc_type
Definition: bmsparsevec_serial.h:280
bm::compressed_collection_deserializer::buffer_type
CBC::buffer_type buffer_type
Definition: bmsparsevec_serial.h:421
bm::decoder_base::memcpy
void memcpy(unsigned char *dst, size_t count) BMNOEXCEPT
read bytes from the decode buffer
Definition: encoding.h:624
bm::bvector::allocator_type
Alloc allocator_type
Definition: bm.h:110
bm::sparse_vector_serializer::is_xor_ref_
bool is_xor_ref_
Definition: bmsparsevec_serial.h:214
bm::sparse_vector_deserializer::sparse_vector_deserializer
sparse_vector_deserializer()
Definition: bmsparsevec_serial.h:769
bm::compressed_collection_deserializer::deserialize
int deserialize(CBC &buffer_coll, const unsigned char *buf, bm::word_t *temp_block=0)
Definition: bmsparsevec_serial.h:530
bm::sparse_vector_deserialize
int sparse_vector_deserialize(SV &sv, const unsigned char *buf, bm::word_t *temp_block=0)
Deserialize sparse vector.
Definition: bmsparsevec_serial.h:380
bm::sparse_vector_deserializer::bvector_type_ptr
bvector_type * bvector_type_ptr
Definition: bmsparsevec_serial.h:228
bm::sparse_vector_serialize
void sparse_vector_serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout, bm::word_t *temp_block=0)
Serialize sparse vector into a memory buffer(s) structure.
Definition: bmsparsevec_serial.h:356
bm::decoder_base::get_8
unsigned char get_8() BMNOEXCEPT
Reads character from the decoding buffer.
Definition: encoding.h:89
bm::compressed_collection_deserializer::bvector_type
CBC::bvector_type bvector_type
Definition: bmsparsevec_serial.h:419
bm::sparse_vector_deserializer::deserialize
void deserialize(SV &sv, const unsigned char *buf, const bvector_type &mask_bv)
Definition: bmsparsevec_serial.h:273
bm::sparse_vector_serial_layout
layout class for serialization buffer structure
Definition: bmsparsevec_serial.h:57
bmundef.h
pre-processor un-defines to avoid global space pollution (internal)
bm::globals::byte_order
static ByteOrder byte_order()
Definition: bmconst.h:464
bm::sparse_vector_serial_layout::sparse_vector_serial_layout
sparse_vector_serial_layout()
Definition: bmsparsevec_serial.h:63
bm::compressed_collection_serializer
Seriaizer for compressed collections.
Definition: bmsparsevec_serial.h:396
bm::encoder::put_8
void put_8(unsigned char c) BMNOEXCEPT
Puts one character into the encoding buffer.
Definition: encoding.h:426
bm::sparse_vector_serial_layout::~sparse_vector_serial_layout
~sparse_vector_serial_layout()
Definition: bmsparsevec_serial.h:65
bm::sparse_vector_serial_layout::capacity
size_t capacity() const
return serialization buffer capacity
Definition: bmsparsevec_serial.h:90
bm::decoder::get_64
bm::id64_t get_64() BMNOEXCEPT
Reads 64-bit word from the decoding buffer.
Definition: encoding.h:709
bm::encoder::set_pos
void set_pos(unsigned char *buf_pos) BMNOEXCEPT
Set current memory stream position.
Definition: encoding.h:507
bm::compressed_collection_serializer::compressed_collection_type
CBC compressed_collection_type
Definition: bmsparsevec_serial.h:399
bm::sparse_vector_deserializer::deserial_
bm::deserializer< bvector_type, bm::decoder > deserial_
Definition: bmsparsevec_serial.h:326
bm::sparse_vector_deserializer::idx_range_from_
size_type idx_range_from_
Definition: bmsparsevec_serial.h:336
bm::sparse_vector_serial_layout::reserve
unsigned char * reserve(size_t capacity)
resize capacity
Definition: bmsparsevec_serial.h:72
bm::bvector::allocator_pool_type
allocator_type::allocator_pool_type allocator_pool_type
Definition: bm.h:111
bm::rank_compressor< bvector_type >
bm::sparse_vector_deserializer::bv_ref_vector_type
bm::serializer< bvector_type >::bv_ref_vector_type bv_ref_vector_type
Definition: bmsparsevec_serial.h:281
bm::sparse_vector_deserializer::pool_
allocator_pool_type pool_
Definition: bmsparsevec_serial.h:325
bm::serializer::gap_length_serialization
void gap_length_serialization(bool value) BMNOEXCEPT
Set GAP length serialization (serializes GAP levels of the original vector)
Definition: bmserial.h:1205
bm::sparse_vector_deserializer::off_vect_
bm::heap_vector< size_t, alloc_type, true > off_vect_
Definition: bmsparsevec_serial.h:331
bm::sparse_vector_deserializer::idx_range_to_
size_type idx_range_to_
Definition: bmsparsevec_serial.h:337
bm::bvector::opt_compress
@ opt_compress
compress blocks when possible (GAP/prefix sum)
Definition: bm.h:134
bm::compressed_collection_serializer::bvector_type
CBC::bvector_type bvector_type
Definition: bmsparsevec_serial.h:400
bm::sparse_vector_deserializer::size_type
SV::size_type size_type
Definition: bmsparsevec_serial.h:230
bm::sparse_vector_serializer::serialize
void serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout)
Serialize sparse vector into a memory buffer(s) structure.
Definition: bmsparsevec_serial.h:630
bm::sparse_vector_deserializer::allocator_pool_type
bvector_type::allocator_type::allocator_pool_type allocator_pool_type
Definition: bmsparsevec_serial.h:231
bm::sparse_vector_deserializer::~sparse_vector_deserializer
~sparse_vector_deserializer()
Definition: bmsparsevec_serial.h:781
bm::sparse_vector_serializer::bv_ref_vector_type
bm::serializer< bvector_type >::bv_ref_vector_type bv_ref_vector_type
Definition: bmsparsevec_serial.h:204
bm::sparse_vector_serial_layout::buffer_type
serializer< bvector_type >::buffer buffer_type
Definition: bmsparsevec_serial.h:61
bm::conditional
ad-hoc conditional expressions
Definition: bmutil.h:110
bm::sparse_vector_deserializer::idx_range_set_
bool idx_range_set_
Definition: bmsparsevec_serial.h:335
bvector_type
bm::bvector bvector_type
Definition: xsample07a.cpp:93
BM_ASSERT
#define BM_ASSERT
Definition: bmdef.h:130
bm::sparse_vector_serial_layout::resize
void resize(size_t ssize)
Set new serialized size.
Definition: bmsparsevec_serial.h:87
bm::sparse_vector_deserializer::deserialize_sv
void deserialize_sv(SV &sv, const unsigned char *buf, const bvector_type *mask_bv)
Definition: bmsparsevec_serial.h:858
bm::deserializer< bvector_type, bm::decoder >
bm::operation_deserializer< bvector_type >
bm::sparse_vector_deserializer::deserialize
void deserialize(SV &sv, const unsigned char *buf)
Definition: bmsparsevec_serial.h:243
bmdef.h
Definitions(internal)
bm::compressed_collection_serializer::statistics_type
CBC::statistics statistics_type
Definition: bmsparsevec_serial.h:402
bm::sparse_vector_serializer::bvector_type_const_ptr
const typedef bvector_type * bvector_type_const_ptr
Definition: bmsparsevec_serial.h:164
bm::serializer
Bit-vector serialization class.
Definition: bmserial.h:75
bm::sparse_vector_serializer::set_bookmarks
void set_bookmarks(bool enable, unsigned bm_interval=256)
Add skip-markers for faster range deserialization.
Definition: bmsparsevec_serial.h:183
bm::compressed_collection_deserializer
Deseriaizer for compressed collections.
Definition: bmsparsevec_serial.h:415
bm::compressed_collection_serializer::serialize
void serialize(const CBC &buffer_coll, buffer_type &buf, bm::word_t *temp_block=0)
Definition: bmsparsevec_serial.h:467
bm::sparse_vector_deserializer::temp_block_
bm::word_t * temp_block_
Definition: bmsparsevec_serial.h:324
bm::sparse_vector_deserializer::bvector_type_const_ptr
const typedef bvector_type * bvector_type_const_ptr
Definition: bmsparsevec_serial.h:227
bm::sparse_vector_serial_layout::set_plain
void set_plain(unsigned i, unsigned char *ptr, size_t buf_size)
Set plain output pointer and size.
Definition: bmsparsevec_serial.h:96
bm::compressed_collection_deserializer::address_resolver_type
CBC::address_resolver_type address_resolver_type
Definition: bmsparsevec_serial.h:423
bm::sparse_vector_serializer::bvs_
bm::serializer< bvector_type > bvs_
Definition: bmsparsevec_serial.h:212
bm::sparse_vector_deserializer::deserialize_plains
void deserialize_plains(SV &sv, unsigned plains, const unsigned char *buf, const bvector_type *mask_bv=0)
deserialize bit-vector plains
Definition: bmsparsevec_serial.h:960
bm::ByteOrder
ByteOrder
Byte orders recognized by the library.
Definition: bmconst.h:428
bm::sparse_vector_serial_layout::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:59
bmserial.h
Serialization / compression of bvector<>. Set theoretical operations on compressed BLOBs.
bm::compressed_collection_deserializer::compressed_collection_type
CBC compressed_collection_type
Definition: bmsparsevec_serial.h:418
bm::sparse_vector_serial_layout::size
size_t size() const
return current serialized size
Definition: bmsparsevec_serial.h:84
bm::sparse_vector_deserializer::rsc_mask_bv_
bvector_type rsc_mask_bv_
Definition: bmsparsevec_serial.h:330
bm::sparse_vector_deserializer::raise_invalid_64bit
static void raise_invalid_64bit()
throw error on incorrect deserialization
Definition: bmsparsevec_serial.h:1173
bm::sparse_vector_serializer::bvector_type_ptr
bvector_type * bvector_type_ptr
Definition: bmsparsevec_serial.h:165
bm
Definition: bm.h:76
bm::encoder::get_pos
unsigned char * get_pos() const BMNOEXCEPT
Get current memory stream position.
Definition: encoding.h:499
bm::sparse_vector_serial_layout::get_plain
const unsigned char * get_plain(unsigned i) const
Get plain pointer.
Definition: bmsparsevec_serial.h:103
bm::sparse_vector_deserializer::load_null_plain
int load_null_plain(SV &sv, int plains, const unsigned char *buf, const bvector_type *mask_bv)
load NULL bit-plain (returns new plains count)
Definition: bmsparsevec_serial.h:1037
bm::sparse_vector_serial_layout::plain_ptrs_
unsigned char * plain_ptrs_[SV::sv_plains]
pointers on serialized bit-plains
Definition: bmsparsevec_serial.h:115
bm::sparse_vector_deserializer::deserialize
void deserialize(SV &sv, const unsigned char *buf, size_type from, size_type to)
Definition: bmsparsevec_serial.h:257
bm::sparse_vector_serial_layout::data
const unsigned char * data() const
Return serialization buffer pointer.
Definition: bmsparsevec_serial.h:108
bm::sparse_vector_deserializer::deserialize_range
void deserialize_range(SV &sv, const unsigned char *buf, size_type from, size_type to)
Definition: bmsparsevec_serial.h:790
bm::sparse_vector_deserializer::remap_buf_ptr_
const unsigned char * remap_buf_ptr_
Definition: bmsparsevec_serial.h:322
bm::word_t
unsigned int word_t
Definition: bmconst.h:38
bm::sparse_vector_serializer::build_xor_ref_vector
void build_xor_ref_vector(const SV &sv)
Definition: bmsparsevec_serial.h:621
bm::bvector::size_type
bm::id_t size_type
Definition: bm.h:117
bm::serializer::set_bookmarks
void set_bookmarks(bool enable, unsigned bm_interval=256) BMNOEXCEPT
Add skip-markers to serialization BLOB for faster range decode at the expense of some BLOB size incre...
Definition: bmserial.h:1217
bm::sparse_vector_serializer::set_xor_ref
void set_xor_ref(bool is_enabled)
Turn ON and OFF XOR compression of sparse vectors.
Definition: bmsparsevec_serial.h:187
bm::encoder
Memory encoding.
Definition: encoding.h:49
bm::sparse_vector_deserializer::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:226
bm::sparse_vector_serializer::sparse_vector_serializer
sparse_vector_serializer()
Definition: bmsparsevec_serial.h:608
bm::decoder
Class for decoding data from memory buffer.
Definition: encoding.h:117
bm::sparse_vector_serial_layout::freemem
void freemem()
free memory
Definition: bmsparsevec_serial.h:93
bm::sparse_vector_serializer
Definition: bmsparsevec_serial.h:160
bm::decoder_base::seek
void seek(int delta) BMNOEXCEPT
change current position
Definition: encoding.h:95
bm::sparse_vector_deserializer::raise_invalid_header
static void raise_invalid_header()
throw error on incorrect deserialization
Definition: bmsparsevec_serial.h:1161
bm::compressed_collection_deserializer::statistics_type
CBC::statistics statistics_type
Definition: bmsparsevec_serial.h:422
bm::encoder::memcpy
void memcpy(const unsigned char *src, size_t count) BMNOEXCEPT
copy bytes into target buffer or just rewind if src is NULL
Definition: encoding.h:478
bm::sparse_vector_deserializer::op_deserial_
bm::operation_deserializer< bvector_type > op_deserial_
Definition: bmsparsevec_serial.h:327
bm::bv_ref_vector
List of reference bit-vectors with their true index associations.
Definition: bmxor.h:240
bm::compressed_collection_deserializer::allocator_type
bvector_type::allocator_type allocator_type
Definition: bmsparsevec_serial.h:420
bm::sparse_vector_deserializer::rsc_compressor_
bm::rank_compressor< bvector_type > rsc_compressor_
Definition: bmsparsevec_serial.h:328
bm::sparse_vector_serializer::size_type
SV::size_type size_type
Definition: bmsparsevec_serial.h:167