Bit-vector serialization class. More...

#include <bmserial.h>

Inheritance diagram for bm::serializer< BV >:

Data Structures
struct	bookmark_state
	Bookmark state structure. More...

Public Types
typedef BV	bvector_type

typedef bvector_type::allocator_type	allocator_type

typedef bvector_type::blocks_manager_type	blocks_manager_type

typedef bvector_type::statistics	statistics_type

typedef bvector_type::block_idx_type	block_idx_type

typedef bvector_type::size_type	size_type

typedef byte_buffer< allocator_type >	buffer

typedef bm::bv_ref_vector< BV >	bv_ref_vector_type

typedef bm::xor_sim_model< BV >	xor_sim_model_type

typedef xor_sim_model_type::block_match_chain_type	block_match_chain_type

Public Member Functions
	serializer (const allocator_type &alloc=allocator_type(), bm::word_t *temp_block=0)
	Constructor. More...

	serializer (bm::word_t *temp_block)
	More...

	~serializer ()

Compression level settings <br>
void	set_compression_level (unsigned clevel) BMNOEXCEPT
	Set compression level. More...

unsigned	get_compression_level () const BMNOEXCEPT
	Get current compression level. More...

Serialization Methods <br>
size_type	serialize (const BV &bv, unsigned char *buf, size_t buf_size)
	Bitvector serialization into memory block. More...

void	serialize (const BV &bv, typename serializer< BV >::buffer &buf, const statistics_type *bv_stat=0)
	Bitvector serialization into buffer object (resized automatically) More...

void	optimize_serialize_destroy (BV &bv, typename serializer< BV >::buffer &buf)
	Bitvector serialization into buffer object (resized automatically) Input bit-vector gets optimized and then destroyed, content is NOT guaranteed after this operation. More...

const size_type *	get_compression_stat () const BMNOEXCEPT
	Return serialization counter vector. More...

void	allow_stat_reset (bool allow) BMNOEXCEPT
	Enable/disable statistics reset on each serilaization. More...

void	reset_compression_stats () BMNOEXCEPT
	Reset all accumulated compression statistics. More...

void	gap_length_serialization (bool value) BMNOEXCEPT
	Set GAP length serialization (serializes GAP levels of the original vector) More...

void	byte_order_serialization (bool value) BMNOEXCEPT
	Set byte-order serialization (for cross platform compatibility) More...

void	set_bookmarks (bool enable, unsigned bm_interval=256) BMNOEXCEPT
	Add skip-markers to serialization BLOB for faster range decode at the expense of some BLOB size increase. More...

void	set_sparse_cutoff (unsigned cutoff) BMNOEXCEPT
	Fine tuning for Binary Interpolative Compression (levels 5+) The parameter sets average population count per block (64Kbits) below which block is considered very sparse. More...

void	set_ref_vectors (const bv_ref_vector_type *ref_vect)
	Attach collection of reference vectors for XOR serialization (no transfer of ownership for the pointers) More...

bool	compute_sim_model (xor_sim_model_type &sim_model, const bv_ref_vector_type &ref_vect, const bm::xor_sim_params &params)
	Calculate XOR similarity model for ref_vector refernece vector must be associated before. More...

void	set_sim_model (const xor_sim_model_type *sim_model) BMNOEXCEPT
	Atach XOR similarity model (must be computed by the same ref vector) More...

void	set_curr_ref_idx (size_type ref_idx) BMNOEXCEPT
	Set current index in rer.vector collection (not a row idx or plain idx) More...

void	encode_header (const BV &bv, bm::encoder &enc) BMNOEXCEPT
	Encode serialization header information. More...

void	encode_gap_block (const bm::gap_word_t *gap_block, bm::encoder &enc)
	More...

void	gamma_gap_block (const bm::gap_word_t *gap_block, bm::encoder &enc) BMNOEXCEPT
	More...

void	gamma_gap_array (const bm::gap_word_t *gap_block, unsigned arr_len, bm::encoder &enc, bool inverted=false) BMNOEXCEPT
	Encode GAP block as delta-array with Elias Gamma coder. More...

void	encode_bit_array (const bm::word_t *block, bm::encoder &enc, bool inverted) BMNOEXCEPT
	Encode bit-block as an array of bits. More...

void	gamma_gap_bit_block (const bm::word_t *block, bm::encoder &enc) BMNOEXCEPT
	More...

void	gamma_arr_bit_block (const bm::word_t *block, bm::encoder &enc, bool inverted) BMNOEXCEPT
	More...

void	bienc_arr_bit_block (const bm::word_t *block, bm::encoder &enc, bool inverted) BMNOEXCEPT
	More...

void	bienc_arr_sblock (const BV &bv, unsigned sb, bm::encoder &enc) BMNOEXCEPT
	More...

void	bienc_gap_bit_block (const bm::word_t *block, bm::encoder &enc) BMNOEXCEPT
	encode bit-block as interpolated bit block of gaps More...

void	interpolated_arr_bit_block (const bm::word_t *block, bm::encoder &enc, bool inverted) BMNOEXCEPT
	More...

void	interpolated_gap_bit_block (const bm::word_t *block, bm::encoder &enc) BMNOEXCEPT
	encode bit-block as interpolated gap block More...

void	interpolated_gap_array (const bm::gap_word_t *gap_block, unsigned arr_len, bm::encoder &enc, bool inverted) BMNOEXCEPT
	Encode GAP block as an array with binary interpolated coder. More...

void	interpolated_gap_array_v0 (const bm::gap_word_t *gap_block, unsigned arr_len, bm::encoder &enc, bool inverted) BMNOEXCEPT
	More...

void	interpolated_encode_gap_block (const bm::gap_word_t *gap_block, bm::encoder &enc) BMNOEXCEPT
	More...

void	encode_bit_interval (const bm::word_t *blk, bm::encoder &enc, unsigned size_control) BMNOEXCEPT
	Encode BIT block with repeatable runs of zeroes. More...

void	encode_bit_digest (const bm::word_t *blk, bm::encoder &enc, bm::id64_t d0) BMNOEXCEPT
	Encode bit-block using digest (hierarchical compression) More...

void	encode_xor_match_chain (bm::encoder &enc, const block_match_chain_type &mchain) BMNOEXCEPT
	Encode XOR match chain. More...

unsigned char	find_gap_best_encoding (const bm::gap_word_t *gap_block) BMNOEXCEPT
	Determine best representation for GAP block based on current set compression level. More...

unsigned char	find_bit_best_encoding (const bm::word_t *block) BMNOEXCEPT
	Determine best representation for a bit-block. More...

unsigned char	find_bit_best_encoding_l5 (const bm::word_t *block) BMNOEXCEPT
	Determine best representation for a bit-block (level 5) More...

void	reset_models () BMNOEXCEPT

void	add_model (unsigned char mod, unsigned score) BMNOEXCEPT
	More...

void	xor_tmp_product (const bm::word_t *s_block, const block_match_chain_type &mchain, unsigned i, unsigned j) BMNOEXCEPT
	Compute digest based XOR product, place into tmp XOR block. More...

static void	process_bookmark (block_idx_type nb, bookmark_state &bookm, bm::encoder &enc) BMNOEXCEPT
	Check if bookmark needs to be placed and if so, encode it into serialization BLOB. More...

Detailed Description

template<class BV>
class bm::serializer< BV >

Bit-vector serialization class.

Class designed to convert sparse bit-vectors into a single block of memory ready for file or database storage or network transfer.

Reuse of this class for multiple serializations (but not across threads). Class resue offers some performance advantage (helps with temp memory reallocations).

Examples: bvsetalgebra.cpp, inv_list.cpp, sample14.cpp, sample22.cpp, sample4.cpp, xsample01.cpp, and xsample07a.cpp.

Definition at line 75 of file bmserial.h.

Constructor & Destructor Documentation

◆ serializer() [1/2]

template<class BV >

bm::serializer< BV >::serializer	(	const allocator_type &	alloc = `allocator_type()`,
		bm::word_t *	temp_block = `0`
	)

Constructor.

Parameters

alloc	- memory allocator
temp_block	- temporary block for various operations (if NULL it will be allocated and managed by serializer class) Temp block is used as a scratch memory during serialization, use of external temp block allows to avoid unnecessary re-allocations.

Temp block attached is not owned by the class and NOT deallocated on destruction.

Definition at line 1170 of file bmserial.h.

References bm::gap_max_bits.

◆ serializer() [2/2]

template<class BV >

bm::serializer< BV >::serializer ( bm::word_t * temp_block )

Definition at line 1203 of file bmserial.h.

References bm::gap_max_bits.

Member Function Documentation

◆ add_model()

template<class BV >

void bm::serializer< BV >::add_model	(	unsigned char	mod,
		unsigned	score
	)

protected

Definition at line 1655 of file bmserial.h.

References BM_ASSERT.

◆ allow_stat_reset()

template<class BV >

void bm::serializer< BV >::allow_stat_reset ( bool allow )

inline

Enable/disable statistics reset on each serilaization.

Definition at line 202 of file bmserial.h.

◆ bienc_arr_bit_block()

template<class BV >

void bm::serializer< BV >::bienc_arr_bit_block	(	const bm::word_t *	block,
		bm::encoder &	enc,
		bool	inverted
	)

protected

Definition at line 2330 of file bmserial.h.

References bm::bit_block_convert_to_arr().

◆ bienc_arr_sblock()

template<class BV >

void bm::serializer< BV >::bienc_arr_sblock	(	const BV &	bv,
		unsigned	sb,
		bm::encoder &	enc
	)

protected

Definition at line 2414 of file bmserial.h.

References bm::bit_out< TEncoder >::bic_encode_u32_cm(), BM_ASSERT, bm::convert_sub_to_arr(), bm::sblock_flag_len16, bm::sblock_flag_max16, bm::sblock_flag_max24, bm::sblock_flag_max32, bm::sblock_flag_min16, bm::sblock_flag_min24, bm::sblock_flag_min32, bm::sblock_flag_sb16, bm::sblock_flag_sb32, bm::set_sblock_bienc, and bm::set_sub_total_bits.

◆ bienc_gap_bit_block()

template<class BV >

void bm::serializer< BV >::bienc_gap_bit_block	(	const bm::word_t *	block,
		bm::encoder &	enc
	)

protected

encode bit-block as interpolated bit block of gaps

Definition at line 2355 of file bmserial.h.

References bm::bit_out< TEncoder >::bic_encode_u16(), bm::bit_to_gap(), BM_ASSERT, bm::bit_out< TEncoder >::flush(), bm::gap_max_bits, bm::set_block_bitgap_bienc, and bm::set_block_size.

◆ byte_order_serialization()

template<class BV >

void bm::serializer< BV >::byte_order_serialization ( bool value )

Set byte-order serialization (for cross platform compatibility)

Parameters

value - TRUE serialization format includes byte-order marker

Examples: inv_list.cpp, sample4.cpp, and xsample01.cpp.

Definition at line 1281 of file bmserial.h.

Referenced by compress_inv_dump_file(), convert_bv2bvs(), main(), and bm::serialize().

◆ compute_sim_model()

template<class BV >

bool bm::serializer< BV >::compute_sim_model	(	xor_sim_model_type &	sim_model,
		const bv_ref_vector_type &	ref_vect,
		const bm::xor_sim_params &	params
	)

Calculate XOR similarity model for ref_vector refernece vector must be associated before.

Parameters

sim_model	- [out] similarity model to compute
ref_vect	- [in] reference vectors
params	- parameters to regulate search depth

Returns: true - if similarity model created successfully

See also: set_ref_vectors

Definition at line 1316 of file bmserial.h.

◆ encode_bit_array()

template<class BV >

void bm::serializer< BV >::encode_bit_array	(	const bm::word_t *	block,
		bm::encoder &	enc,
		bool	inverted
	)

protected

Encode bit-block as an array of bits.

Definition at line 2286 of file bmserial.h.

References bm::bit_block_convert_to_arr(), bm::set_block_arrbit, and bm::set_block_arrbit_inv.

◆ encode_bit_digest()

template<class BV >

void bm::serializer< BV >::encode_bit_digest	(	const bm::word_t *	blk,
		bm::encoder &	enc,
		bm::id64_t	d0
	)

protected

Encode bit-block using digest (hierarchical compression)

Definition at line 2111 of file bmserial.h.

References bm::bmi_blsi_u64(), bm::bmi_bslr_u64(), bm::set_block_bit, bm::set_block_bit_digest0, bm::set_block_digest_wave_size, bm::set_block_size, and bm::word_bitcount64().

◆ encode_bit_interval()

template<class BV >

void bm::serializer< BV >::encode_bit_interval	(	const bm::word_t *	blk,
		bm::encoder &	enc,
		unsigned	size_control
	)

protected

Encode BIT block with repeatable runs of zeroes.

Definition at line 2059 of file bmserial.h.

References BM_ASSERT, bm::set_block_bit_0runs, and bm::set_block_size.

◆ encode_gap_block()

template<class BV >

void bm::serializer< BV >::encode_gap_block	(	const bm::gap_word_t *	gap_block,
		bm::encoder &	enc
	)

protected

Encode GAP block

Definition at line 1999 of file bmserial.h.

References BM_ASSERT, BM_FALLTHROUGH, bm::gap_convert_to_arr(), bm::gap_equiv_len, bm::encoder::put_16(), bm::encoder::put_8(), bm::set_block_arrgap, bm::set_block_arrgap_bienc, bm::set_block_arrgap_bienc_inv, bm::set_block_arrgap_egamma, bm::set_block_arrgap_egamma_inv, bm::set_block_arrgap_inv, bm::set_block_bit_1bit, bm::set_block_gap, and bm::set_block_gap_bienc.

◆ encode_header()

template<class BV >

void bm::serializer< BV >::encode_header	(	const BV &	bv,
		bm::encoder &	enc
	)

protected

Encode serialization header information.

Definition at line 1336 of file bmserial.h.

References bm::BM_HM_64_BIT, bm::BM_HM_DEFAULT, bm::BM_HM_HXOR, bm::BM_HM_NO_BO, bm::BM_HM_NO_GAPL, bm::BM_HM_RESIZE, bm::globals< T >::byte_order(), bm::gap_levels, and bm::id_max.

◆ encode_xor_match_chain()

template<class BV >

void bm::serializer< BV >::encode_xor_match_chain	(	bm::encoder &	enc,
		const block_match_chain_type &	mchain
	)

protected

Encode XOR match chain.

Definition at line 2164 of file bmserial.h.

References BM_ASSERT, bm::check_pair_vect_vbr(), and bm::set_block_xor_chain.

◆ find_bit_best_encoding()

template<class BV >

unsigned char bm::serializer< BV >::find_bit_best_encoding ( const bm::word_t * block )

protected

Determine best representation for a bit-block.

Definition at line 1780 of file bmserial.h.

References bm::best_representation(), bm::bit_block_change_bc(), bm::bit_block_count(), bm::bit_count_nonzero_size(), bm::block_waves, BM_ASSERT, bm::calc_block_digest0(), bm::compute_s_block_descr(), bm::e_bit_0, bm::e_bit_1, bm::e_bit_bit, bm::e_bit_end, bm::e_bit_GAP, bm::e_bit_IINT, bm::e_bit_INT, bm::gap_equiv_len, bm::gap_max_bits, bm::gap_max_buff_len, bm::block_waves_xor_descr::sb_bc, bm::block_waves_xor_descr::sb_gc, bm::set_block_aone, bm::set_block_arrbit, bm::set_block_arrbit_inv, bm::set_block_arrgap_egamma, bm::set_block_arrgap_egamma_inv, bm::set_block_azero, bm::set_block_bit, bm::set_block_bit_0runs, bm::set_block_bit_1bit, bm::set_block_bit_digest0, bm::set_block_digest_wave_size, bm::set_block_gap_egamma, bm::set_block_size, and bm::word_bitcount64().

◆ find_bit_best_encoding_l5()

template<class BV >

unsigned char bm::serializer< BV >::find_bit_best_encoding_l5 ( const bm::word_t * block )

protected

Determine best representation for a bit-block (level 5)

Definition at line 1664 of file bmserial.h.

References bm::bit_block_change_bc(), bm::bit_count_nonzero_size(), bm::calc_block_digest0(), bm::gap_max_bits, bm::gap_max_buff_len, bm::set_block_aone, bm::set_block_arr_bienc, bm::set_block_arr_bienc_inv, bm::set_block_arrbit, bm::set_block_arrbit_inv, bm::set_block_arrgap, bm::set_block_arrgap_bienc, bm::set_block_arrgap_bienc_inv, bm::set_block_arrgap_egamma, bm::set_block_arrgap_egamma_inv, bm::set_block_arrgap_inv, bm::set_block_azero, bm::set_block_bit, bm::set_block_bit_0runs, bm::set_block_bit_1bit, bm::set_block_bit_digest0, bm::set_block_bitgap_bienc, bm::set_block_gap_bienc, bm::set_block_gap_egamma, bm::set_block_size, and bm::word_bitcount64().

◆ find_gap_best_encoding()

template<class BV >

unsigned char bm::serializer< BV >::find_gap_best_encoding ( const bm::gap_word_t * gap_block )

protected

Determine best representation for GAP block based on current set compression level.

Returns: set_block_bit, set_block_bit_1bit, set_block_arrgap set_block_arrgap_egamma, set_block_arrgap_bienc set_block_arrgap_inv, set_block_arrgap_egamma_inv set_block_arrgap_bienc_inv, set_block_gap_egamma set_block_gap_bienc

Definition at line 1948 of file bmserial.h.

References bm::gap_bit_count_unr(), bm::gap_length(), bm::gap_max_bits, bm::set_block_arrgap, bm::set_block_arrgap_bienc, bm::set_block_arrgap_bienc_inv, bm::set_block_arrgap_egamma, bm::set_block_arrgap_egamma_inv, bm::set_block_arrgap_inv, bm::set_block_bit_1bit, bm::set_block_gap, bm::set_block_gap_bienc, and bm::set_block_gap_egamma.

◆ gamma_arr_bit_block()

template<class BV >

void bm::serializer< BV >::gamma_arr_bit_block	(	const bm::word_t *	block,
		bm::encoder &	enc,
		bool	inverted
	)

protected

Definition at line 2314 of file bmserial.h.

References bm::bit_block_convert_to_arr(), bm::set_block_bit, and bm::set_block_size.

◆ gamma_gap_array()

template<class BV >

void bm::serializer< BV >::gamma_gap_array	(	const bm::gap_word_t *	gap_block,
		unsigned	arr_len,
		bm::encoder &	enc,
		bool	inverted = `false`
	)

protected

Encode GAP block as delta-array with Elias Gamma coder.

Definition at line 1487 of file bmserial.h.

References bm::bit_out< TEncoder >::gamma(), bm::set_block_arrgap, bm::set_block_arrgap_egamma, bm::set_block_arrgap_egamma_inv, and bm::set_block_arrgap_inv.

◆ gamma_gap_bit_block()

template<class BV >

void bm::serializer< BV >::gamma_gap_bit_block	(	const bm::word_t *	block,
		bm::encoder &	enc
	)

protected

Definition at line 2305 of file bmserial.h.

References bm::bit_to_gap(), BM_ASSERT, and bm::gap_equiv_len.

◆ gamma_gap_block()

template<class BV >

void bm::serializer< BV >::gamma_gap_block	(	const bm::gap_word_t *	gap_block,
		bm::encoder &	enc
	)

protected

Encode GAP block with Elias Gamma coder

Definition at line 1448 of file bmserial.h.

References bm::for_each_dgap(), bm::gap_length(), bm::set_block_gap, and bm::set_block_gap_egamma.

◆ gap_length_serialization()

template<class BV >

void bm::serializer< BV >::gap_length_serialization ( bool value )

Set GAP length serialization (serializes GAP levels of the original vector)

Parameters

value - when TRUE serialized vector includes GAP levels parameters

Examples: inv_list.cpp, sample4.cpp, and xsample01.cpp.

Definition at line 1275 of file bmserial.h.

Referenced by compress_inv_dump_file(), convert_bv2bvs(), main(), bm::serialize(), bm::compressed_collection_serializer< CBC >::serialize(), and bm::sparse_vector_serializer< SV >::sparse_vector_serializer().

◆ get_compression_level()

template<class BV >

unsigned bm::serializer< BV >::get_compression_level ( ) const

inline

Get current compression level.

Definition at line 133 of file bmserial.h.

◆ get_compression_stat()

template<class BV >

const size_type* bm::serializer< BV >::get_compression_stat ( ) const

inline

Return serialization counter vector.

Definition at line 196 of file bmserial.h.

◆ interpolated_arr_bit_block()

template<class BV >

void bm::serializer< BV >::interpolated_arr_bit_block	(	const bm::word_t *	block,
		bm::encoder &	enc,
		bool	inverted
	)

protected

Definition at line 2505 of file bmserial.h.

References bm::bit_out< TEncoder >::bic_encode_u16(), bm::bit_block_convert_to_arr(), BM_ASSERT, bm::bit_out< TEncoder >::flush(), bm::set_block_arr_bienc, bm::set_block_arr_bienc_8bh, bm::set_block_arr_bienc_inv, and bm::set_block_size.

◆ interpolated_encode_gap_block()

template<class BV >

void bm::serializer< BV >::interpolated_encode_gap_block	(	const bm::gap_word_t *	gap_block,
		bm::encoder &	enc
	)

protected

Encode GAP block with using binary interpolated encoder

Definition at line 1388 of file bmserial.h.

References bm::bit_out< TEncoder >::bic_encode_u16(), BM_ASSERT, bm::bit_out< TEncoder >::flush(), bm::gap_length(), bm::set_block_gap, bm::set_block_gap_bienc, and bm::set_block_gap_bienc_v2.

◆ interpolated_gap_array()

template<class BV >

void bm::serializer< BV >::interpolated_gap_array	(	const bm::gap_word_t *	gap_block,
		unsigned	arr_len,
		bm::encoder &	enc,
		bool	inverted
	)

protected

Encode GAP block as an array with binary interpolated coder.

Definition at line 1580 of file bmserial.h.

References bm::bit_out< TEncoder >::bic_encode_u16(), BM_ASSERT, bm::bit_out< TEncoder >::flush(), bm::set_block_arrgap, bm::set_block_arrgap_bienc_inv_v2, bm::set_block_arrgap_bienc_v2, and bm::set_block_arrgap_inv.

◆ interpolated_gap_array_v0()

template<class BV >

void bm::serializer< BV >::interpolated_gap_array_v0	(	const bm::gap_word_t *	gap_block,
		unsigned	arr_len,
		bm::encoder &	enc,
		bool	inverted
	)

protected

Definition at line 1532 of file bmserial.h.

References bm::bit_out< TEncoder >::bic_encode_u16(), BM_ASSERT, bm::bit_out< TEncoder >::flush(), bm::bit_out< TEncoder >::gamma(), bm::set_block_arrgap, bm::set_block_arrgap_bienc, bm::set_block_arrgap_bienc_inv, and bm::set_block_arrgap_inv.

◆ interpolated_gap_bit_block()

template<class BV >

void bm::serializer< BV >::interpolated_gap_bit_block	(	const bm::word_t *	block,
		bm::encoder &	enc
	)

protected

encode bit-block as interpolated gap block

Definition at line 2345 of file bmserial.h.

References bm::bit_to_gap(), BM_ASSERT, and bm::gap_max_bits.

◆ optimize_serialize_destroy()

template<class BV >

void bm::serializer< BV >::optimize_serialize_destroy	(	BV &	bv,
		typename serializer< BV >::buffer &	buf
	)

Bitvector serialization into buffer object (resized automatically) Input bit-vector gets optimized and then destroyed, content is NOT guaranteed after this operation.

Effectively it moves data into the buffer.

The reason this operation exsists is because it is faster to do all three operations in one single pass. This is a destructive serialization!

Parameters

bv	- input/output bitvector
buf	- output buffer object

Examples: sample4.cpp.

Definition at line 2268 of file bmserial.h.

References bm::serialize().

Referenced by main().

◆ process_bookmark()

template<class BV >

void bm::serializer< BV >::process_bookmark	(	block_idx_type	nb,
		bookmark_state &	bookm,
		bm::encoder &	enc
	)

staticprotected

Check if bookmark needs to be placed and if so, encode it into serialization BLOB.

Parameters

nb	- block idx
bookm	- bookmark state structure
enc	- BLOB encoder

Definition at line 2596 of file bmserial.h.

References BM_ASSERT, bm::set_nb_bookmark16, bm::set_nb_bookmark24, bm::set_nb_bookmark32, bm::set_nb_sync_mark16, bm::set_nb_sync_mark24, bm::set_nb_sync_mark32, bm::set_nb_sync_mark48, bm::set_nb_sync_mark64, and bm::set_nb_sync_mark8.

◆ reset_compression_stats()

template<class BV >

void bm::serializer< BV >::reset_compression_stats

Reset all accumulated compression statistics.

Definition at line 1247 of file bmserial.h.

◆ serialize() [1/2]

template<class BV >

void bm::serializer< BV >::serialize	(	const BV &	bv,
		typename serializer< BV >::buffer &	buf,
		const statistics_type *	bv_stat = `0`
	)

Bitvector serialization into buffer object (resized automatically)

Parameters

bv	- input bitvector
buf	- output buffer object
bv_stat	- input (optional) bit-vector statistics object if NULL, serialize will compute the statistics

Definition at line 2244 of file bmserial.h.

References BM_ASSERT, bm::bv_statistics::max_serialize_mem, and bm::serialize().

◆ serialize() [2/2]

template<class BV >

serializer< BV >::size_type bm::serializer< BV >::serialize	(	const BV &	bv,
		unsigned char *	buf,
		size_t	buf_size
	)

Bitvector serialization into memory block.

Parameters

bv	- input bitvector
buf	- out buffer (pre-allocated) No range checking is done in this method. It is responsibility of caller to allocate sufficient amount of memory using information from calc_stat() function.
buf_size	- size of the output buffer

Returns: Size of serialization block.

See also: calc_stat

Examples: bvsetalgebra.cpp, sample14.cpp, sample22.cpp, sample4.cpp, xsample01.cpp, and xsample07a.cpp.

Definition at line 2706 of file bmserial.h.

References bm::bit_block_find(), BM_ASSERT, bm::BM_HM_SPARSE, BM_IS_GAP, BM_SER_NEXT_GRP, BMGAP_PTR, bm::block_match_chain< BLOCK_IDX >::chain_size, bm::check_block_one(), bm::check_block_zero(), bm::e_no_xor_match, bm::e_xor_match_BC, bm::e_xor_match_EQ, bm::e_xor_match_GC, bm::e_xor_match_iBC, FULL_BLOCK_FAKE_ADDR, bm::get_block_coord(), bm::block_match_chain< BLOCK_IDX >::match, bm::block_match_chain< BLOCK_IDX >::nb, bm::encoder::put_16(), bm::encoder::put_32(), bm::encoder::put_64(), bm::encoder::put_8(), bm::encoder::put_8_16_32(), bm::encoder::put_prefixed_array_32(), bm::block_match_chain< BLOCK_IDX >::ref_idx, bm::set_block_16one, bm::set_block_16zero, bm::set_block_1one, bm::set_block_1zero, bm::set_block_32one, bm::set_block_32zero, bm::set_block_64one, bm::set_block_64zero, bm::set_block_8one, bm::set_block_8zero, bm::set_block_aone, bm::set_block_arr_bienc, bm::set_block_arr_bienc_inv, bm::set_block_arrbit, bm::set_block_arrbit_inv, bm::set_block_arrgap_bienc, bm::set_block_arrgap_bienc_inv, bm::set_block_arrgap_egamma, bm::set_block_arrgap_egamma_inv, bm::set_block_azero, bm::set_block_bit, bm::set_block_bit_0runs, bm::set_block_bit_1bit, bm::set_block_bit_digest0, bm::set_block_bitgap_bienc, bm::set_block_end, bm::set_block_gap_bienc, bm::set_block_gap_egamma, bm::set_block_ref_eq, bm::set_block_size, bm::set_block_xor_ref16, bm::set_block_xor_ref16_um, bm::set_block_xor_ref32, bm::set_block_xor_ref32_um, bm::set_block_xor_ref8, bm::set_block_xor_ref8_um, bm::encoder::set_pos(), bm::set_sub_array_size, bm::set_total_blocks, bm::encoder::size(), and bm::block_match_chain< BLOCK_IDX >::xor_d64.

Referenced by convert_bv2bvs(), generate_k_mers(), main(), make_BLOB(), bm::serialize(), and bm::compressed_collection_serializer< CBC >::serialize().

◆ set_bookmarks()

template<class BV >

void bm::serializer< BV >::set_bookmarks	(	bool	enable,
		unsigned	bm_interval = `256`
	)

Add skip-markers to serialization BLOB for faster range decode at the expense of some BLOB size increase.

Parameters

enable	- TRUE searilization will add bookmark codes
bm_interval	- bookmark interval in (number of blocks) suggested values between 4 and 512 (block size is 64K bits) smaller interval means more bookmarks added to the skip list allows faster range deserialization at the expense of somewhat increased BLOB size.

Examples: sample22.cpp, and xsample07a.cpp.

Definition at line 1287 of file bmserial.h.

Referenced by generate_k_mers(), main(), and bm::sparse_vector_serializer< SV >::set_bookmarks().

◆ set_compression_level()

template<class BV >

void bm::serializer< BV >::set_compression_level ( unsigned clevel )

Set compression level.

Higher compression takes more time to process.

Parameters

clevel - compression level (0-6) 0 - take as is 1, 2 - apply light weight RLE/GAP encodings, limited depth hierarchical compression, intervals encoding 3 - variant of 2 with different cut-offs 4 - delta transforms plus Elias Gamma encoding where possible legacy) 5 - Binary Interpolative Coding (BIC) - light settings 6 - Binary Interpolative Coding (BIC) - harder settings

See also: get_compression_level

Examples: inv_list.cpp, and xsample01.cpp.

Definition at line 1254 of file bmserial.h.

References bm::set_compression_max, bm::sparse_max_l5, and bm::sparse_max_l6.

Referenced by compress_inv_dump_file(), convert_bv2bvs(), main(), and make_BLOB().

◆ set_curr_ref_idx()

template<class BV >

void bm::serializer< BV >::set_curr_ref_idx ( size_type ref_idx )

Set current index in rer.vector collection (not a row idx or plain idx)

Definition at line 1330 of file bmserial.h.

◆ set_ref_vectors()

template<class BV >

void bm::serializer< BV >::set_ref_vectors ( const bv_ref_vector_type * ref_vect )

Attach collection of reference vectors for XOR serialization (no transfer of ownership for the pointers)

Definition at line 1302 of file bmserial.h.

References bm::set_block_size.

◆ set_sim_model()

template<class BV >

void bm::serializer< BV >::set_sim_model ( const xor_sim_model_type * sim_model )

Atach XOR similarity model (must be computed by the same ref vector)

Definition at line 1324 of file bmserial.h.

◆ set_sparse_cutoff()

template<class BV >

void bm::serializer< BV >::set_sparse_cutoff ( unsigned cutoff )

Fine tuning for Binary Interpolative Compression (levels 5+) The parameter sets average population count per block (64Kbits) below which block is considered very sparse.

If super block (group of 256 blocks) is very sparse it applies block size expansion (for the compression purposes) to improve compression rates.

Definition at line 1265 of file bmserial.h.

References BM_ASSERT, and bm::sparse_max_l6.

◆ xor_tmp_product()

template<class BV >

void bm::serializer< BV >::xor_tmp_product	(	const bm::word_t *	s_block,
		const block_match_chain_type &	mchain,
		unsigned	i,
		unsigned	j
	)

protected

Compute digest based XOR product, place into tmp XOR block.

Definition at line 2209 of file bmserial.h.

References bm::bit_block_xor(), BM_IS_GAP, BMGAP_PTR, and bm::gap_convert_to_bitset().

The documentation for this class was generated from the following file:

bmserial.h

Data Structures

Public Types

Public Member Functions

Serialization Methods <br>

Detailed Description

Constructor & Destructor Documentation

◆ serializer() [1/2]

◆ serializer() [2/2]

Member Function Documentation

◆ add_model()

◆ allow_stat_reset()

◆ bienc_arr_bit_block()

◆ bienc_arr_sblock()

◆ bienc_gap_bit_block()

◆ byte_order_serialization()

◆ compute_sim_model()

◆ encode_bit_array()

◆ encode_bit_digest()

◆ encode_bit_interval()

◆ encode_gap_block()

◆ encode_header()

◆ encode_xor_match_chain()

◆ find_bit_best_encoding()

◆ find_bit_best_encoding_l5()

◆ find_gap_best_encoding()

◆ gamma_arr_bit_block()

◆ gamma_gap_array()

◆ gamma_gap_bit_block()

◆ gamma_gap_block()

◆ gap_length_serialization()

◆ get_compression_level()

◆ get_compression_stat()

◆ interpolated_arr_bit_block()

◆ interpolated_encode_gap_block()

◆ interpolated_gap_array()

◆ interpolated_gap_array_v0()

◆ interpolated_gap_bit_block()

◆ optimize_serialize_destroy()

◆ process_bookmark()

◆ reset_compression_stats()

◆ serialize() [1/2]

◆ serialize() [2/2]

◆ set_bookmarks()

◆ set_compression_level()

◆ set_curr_ref_idx()

◆ set_ref_vectors()

◆ set_sim_model()

◆ set_sparse_cutoff()

◆ xor_tmp_product()