Processor specific optimizations for SSE4.2 instructions (internals) More...

Functions
bm::id_t	bm::sse4_bit_count (const __m128i block, const __m128i block_end) BMNOEXCEPT
	More...

bool	bm::sse4_is_all_zero (const __m128i *BMRESTRICT block) BMNOEXCEPT
	check if block is all zero bits More...

BMFORCEINLINE bool	bm::sse4_is_digest_zero (const __m128i *BMRESTRICT block) BMNOEXCEPT
	check if digest stride is all zero bits More...

BMFORCEINLINE void	bm::sse4_block_set_digest (__m128i *dst, unsigned value) BMNOEXCEPT
	set digest stride to 0xFF.. or 0x0 value More...

unsigned	bm::sse4_and_block (__m128i BMRESTRICT dst, const __m128i BMRESTRICT src) BMNOEXCEPT
	AND blocks2 dst &= *src. More...

BMFORCEINLINE bool	bm::sse4_and_digest (__m128i BMRESTRICT dst, const __m128i BMRESTRICT src) BMNOEXCEPT
	AND block digest stride dst &= *src. More...

BMFORCEINLINE bool	bm::sse4_and_digest_2way (__m128i BMRESTRICT dst, const __m128i BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
	AND block digest stride dst = *src1 & src2. More...

bool	bm::sse4_and_or_digest_2way (__m128i BMRESTRICT dst, const __m128i BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
	AND-OR block digest stride dst \|= *src1 & src2. More...

bool	bm::sse4_and_digest_5way (__m128i BMRESTRICT dst, const __m128i BMRESTRICT src1, const __m128i BMRESTRICT src2, const __m128i BMRESTRICT src3, const __m128i *BMRESTRICT src4) BMNOEXCEPT
	AND block digest stride. More...

BMFORCEINLINE bool	bm::sse4_sub_digest (__m128i BMRESTRICT dst, const __m128i BMRESTRICT src) BMNOEXCEPT
	SUB (AND NOT) block digest stride dst &= ~*src. More...

BMFORCEINLINE bool	bm::sse4_sub_digest_2way (__m128i BMRESTRICT dst, const __m128i BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
	2-operand SUB (AND NOT) block digest stride dst = src1 & ~*src2 More...

bool	bm::sse4_is_all_one (const __m128i *BMRESTRICT block) BMNOEXCEPT
	check if block is all ONE bits More...

BMFORCEINLINE bool	bm::sse42_test_all_one_wave (const void *ptr) BMNOEXCEPT
	check if SSE wave is all oxFFFF...FFF More...

BMFORCEINLINE bool	bm::sse42_test_all_zero_wave (const void *ptr) BMNOEXCEPT
	check if wave of pointers is all NULL More...

BMFORCEINLINE bool	bm::sse42_test_all_zero_wave2 (const void ptr0, const void ptr1) BMNOEXCEPT
	check if 2 waves of pointers are all NULL More...

BMFORCEINLINE bool	bm::sse42_test_all_eq_wave2 (const void ptr0, const void ptr1) BMNOEXCEPT
	check if wave of 2 pointers are the same (null or FULL) More...

unsigned	bm::sse42_bit_block_calc_change (const __m128i *BMRESTRICT block, unsigned size) BMNOEXCEPT
	More...

void	bm::sse42_bit_block_calc_xor_change (const __m128i BMRESTRICT block, const __m128i BMRESTRICT xor_block, unsigned size, unsigned BMRESTRICT gc, unsigned BMRESTRICT bc) BMNOEXCEPT
	More...

void	bm::sse42_bit_block_calc_change_bc (const __m128i BMRESTRICT block, unsigned gc, unsigned *bc) BMNOEXCEPT
	More...

bool	bm::sse42_bit_find_first_diff (const __m128i BMRESTRICT block1, const __m128i BMRESTRICT block2, unsigned *pos) BMNOEXCEPT
	Find first bit which is different between two bit-blocks. More...

bool	bm::sse42_bit_find_first (const __m128i BMRESTRICT block, unsigned pos) BMNOEXCEPT
	Find first non-zero bit. More...

unsigned	bm::sse4_gap_find (const bm::gap_word_t *BMRESTRICT pbuf, const bm::gap_word_t pos, const unsigned size) BMNOEXCEPT
	More...

unsigned	bm::sse42_gap_bfind (const unsigned short BMRESTRICT buf, unsigned pos, unsigned BMRESTRICT is_set) BMNOEXCEPT
	Hybrid binary search, starts as binary, then switches to linear scan. More...

unsigned	bm::sse42_gap_test (const unsigned short *BMRESTRICT buf, unsigned pos) BMNOEXCEPT
	Hybrid binary search, starts as binary, then switches to scan. More...

int	bm::sse42_cmpge_u32 (__m128i vect4, unsigned value) BMNOEXCEPT
	Experimental (test) function to do SIMD vector search (lower bound) in sorted, growing array. More...

bool	bm::sse42_shift_l1 (__m128i block, unsigned empty_acc, unsigned co1) BMNOEXCEPT
	block shift left by 1 More...

bool	bm::sse42_shift_r1 (__m128i block, unsigned empty_acc, unsigned co1) BMNOEXCEPT
	block shift right by 1 More...

bool	bm::sse42_shift_r1_and (__m128i block, bm::word_t co1, const __m128i BMRESTRICT mask_block, bm::id64_t *digest) BMNOEXCEPT
	block shift right by 1 plus AND More...

void	bm::sse42_bit_block_xor (bm::word_t target_block, const bm::word_t block, const bm::word_t *xor_block, bm::id64_t digest) BMNOEXCEPT
	Build partial XOR product of 2 bit-blocks using digest mask. More...

void	bm::sse42_bit_block_xor_2way (bm::word_t target_block, const bm::word_t xor_block, bm::id64_t digest) BMNOEXCEPT
	Build partial XOR product of 2 bit-blocks using digest mask. More...

Detailed Description

Processor specific optimizations for SSE4.2 instructions (internals)

Function Documentation

◆ sse42_bit_block_calc_change()

unsigned bm::sse42_bit_block_calc_change	(	const __m128i *BMRESTRICT	block,
		unsigned	size
	)

inline

SSE4.2 calculate number of bit changes from 0 to 1

Definition at line 709 of file bmsse4.h.

References BM_ALIGN32, and BM_ALIGN32ATTR.

◆ sse42_bit_block_calc_change_bc()

void bm::sse42_bit_block_calc_change_bc	(	const __m128i *BMRESTRICT	block,
		unsigned *	gc,
		unsigned *	bc
	)

inline

SSE4.2 calculate number of bit changes from 0 to 1

Definition at line 904 of file bmsse4.h.

References bm::set_block_size.

◆ sse42_bit_block_calc_xor_change()

void bm::sse42_bit_block_calc_xor_change	(	const __m128i *BMRESTRICT	block,
		const __m128i *BMRESTRICT	xor_block,
		unsigned	size,
		unsigned *BMRESTRICT	gc,
		unsigned *BMRESTRICT	bc
	)

inline

SSE4.2 calculate number of bit changes from 0 to 1 of a XOR product

Definition at line 786 of file bmsse4.h.

References BM_ALIGN32, and BM_ALIGN32ATTR.

◆ sse42_bit_block_xor()

void bm::sse42_bit_block_xor	(	bm::word_t *	target_block,
		const bm::word_t *	block,
		const bm::word_t *	xor_block,
		bm::id64_t	digest
	)

inline

Build partial XOR product of 2 bit-blocks using digest mask.

Parameters

target_block	- target := block ^ xor_block
block	- arg1
xor_block	- arg2
digest	- mask for each block wave to XOR (1) or just copy (0)

Definition at line 1686 of file bmsse4.h.

References bm::block_waves, and bm::set_block_digest_wave_size.

◆ sse42_bit_block_xor_2way()

void bm::sse42_bit_block_xor_2way	(	bm::word_t *	target_block,
		const bm::word_t *	xor_block,
		bm::id64_t	digest
	)

inline

Build partial XOR product of 2 bit-blocks using digest mask.

Parameters

target_block	- target ^= xor_block
xor_block	- arg1
digest	- mask for each block wave to XOR (if 1)

Definition at line 1756 of file bmsse4.h.

References bm::bmi_blsi_u64(), bm::bmi_bslr_u64(), and bm::set_block_digest_wave_size.

◆ sse42_bit_find_first()

bool bm::sse42_bit_find_first	(	const __m128i *BMRESTRICT	block,
		unsigned *	pos
	)

inline

Find first non-zero bit.

Definition at line 1035 of file bmsse4.h.

References BM_ALIGN32, BM_ALIGN32ATTR, BM_ASSERT, BM_BSF32, and bm::set_block_size.

◆ sse42_bit_find_first_diff()

bool bm::sse42_bit_find_first_diff	(	const __m128i *BMRESTRICT	block1,
		const __m128i *BMRESTRICT	block2,
		unsigned *	pos
	)

inline

Find first bit which is different between two bit-blocks.

Definition at line 979 of file bmsse4.h.

References BM_ALIGN32, BM_ALIGN32ATTR, BM_ASSERT, BM_BSF32, and bm::set_block_size.

◆ sse42_cmpge_u32()

int bm::sse42_cmpge_u32	(	__m128i	vect4,
		unsigned	value
	)

inline

Experimental (test) function to do SIMD vector search (lower bound) in sorted, growing array.

Definition at line 1225 of file bmsse4.h.

References BM_BSF32.

◆ sse42_gap_bfind()

unsigned bm::sse42_gap_bfind	(	const unsigned short *BMRESTRICT	buf,
		unsigned	pos,
		unsigned *BMRESTRICT	is_set
	)

inline

Hybrid binary search, starts as binary, then switches to linear scan.

Parameters

buf	- GAP buffer pointer.
pos	- index of the element.
is_set	- output. GAP value (0 or 1).

Returns: GAP index.

Definition at line 1160 of file bmsse4.h.

References BM_ASSERT, and bm::sse4_gap_find().

Referenced by bm::sse42_gap_test().

◆ sse42_gap_test()

unsigned bm::sse42_gap_test	(	const unsigned short *BMRESTRICT	buf,
		unsigned	pos
	)

inline

Hybrid binary search, starts as binary, then switches to scan.

Definition at line 1208 of file bmsse4.h.

References bm::sse42_gap_bfind().

Referenced by bm::gap_test_unr().

◆ sse42_shift_l1()

bool bm::sse42_shift_l1	(	__m128i *	block,
		unsigned *	empty_acc,
		unsigned	co1
	)

inline

block shift left by 1

Definition at line 1471 of file bmsse4.h.

References bm::set_block_size.

◆ sse42_shift_r1()

bool bm::sse42_shift_r1	(	__m128i *	block,
		unsigned *	empty_acc,
		unsigned	co1
	)

inline

block shift right by 1

Definition at line 1521 of file bmsse4.h.

References bm::set_block_size.

◆ sse42_shift_r1_and()

bool bm::sse42_shift_r1_and	(	__m128i *	block,
		bm::word_t	co1,
		const __m128i *BMRESTRICT	mask_block,
		bm::id64_t *	digest
	)

inline

block shift right by 1 plus AND

Returns: carry over flag

Definition at line 1571 of file bmsse4.h.

References BM_ASSERT, bm::id_max, and bm::set_block_digest_wave_size.

◆ sse42_test_all_eq_wave2()

BMFORCEINLINE bool bm::sse42_test_all_eq_wave2	(	const void *	ptr0,
		const void *	ptr1
	)

check if wave of 2 pointers are the same (null or FULL)

Definition at line 695 of file bmsse4.h.

Referenced by bm::bvector< Alloc >::combine_operation_or().

◆ sse42_test_all_one_wave()

BMFORCEINLINE bool bm::sse42_test_all_one_wave ( const void * ptr )

check if SSE wave is all oxFFFF...FFF

Definition at line 660 of file bmsse4.h.

◆ sse42_test_all_zero_wave()

BMFORCEINLINE bool bm::sse42_test_all_zero_wave ( const void * ptr )

check if wave of pointers is all NULL

Definition at line 671 of file bmsse4.h.

Referenced by bm::bvector< Alloc >::combine_operation_and(), bm::bvector< Alloc >::combine_operation_sub(), bm::for_each_bit(), and bm::for_each_nzblock().

◆ sse42_test_all_zero_wave2()

BMFORCEINLINE bool bm::sse42_test_all_zero_wave2	(	const void *	ptr0,
		const void *	ptr1
	)

check if 2 waves of pointers are all NULL

Definition at line 682 of file bmsse4.h.

Referenced by bm::bvector< Alloc >::combine_operation_xor().

◆ sse4_and_block()

unsigned bm::sse4_and_block	(	__m128i *BMRESTRICT	dst,
		const __m128i *BMRESTRICT	src
	)

inline

AND blocks2 dst &= *src.

Returns: 0 if no bits were set

Definition at line 257 of file bmsse4.h.

References BMRESTRICT, and bm::set_block_size.

◆ sse4_and_digest()

BMFORCEINLINE bool bm::sse4_and_digest	(	__m128i *BMRESTRICT	dst,
		const __m128i *BMRESTRICT	src
	)

AND block digest stride dst &= *src.

Returns: true if stide is all zero

Definition at line 304 of file bmsse4.h.

◆ sse4_and_digest_2way()

BMFORCEINLINE bool bm::sse4_and_digest_2way	(	__m128i *BMRESTRICT	dst,
		const __m128i *BMRESTRICT	src1,
		const __m128i *BMRESTRICT	src2
	)

AND block digest stride dst = *src1 & src2.

Returns: true if stide is all zero

Definition at line 352 of file bmsse4.h.

◆ sse4_and_digest_5way()

bool bm::sse4_and_digest_5way	(	__m128i *BMRESTRICT	dst,
		const __m128i *BMRESTRICT	src1,
		const __m128i *BMRESTRICT	src2,
		const __m128i *BMRESTRICT	src3,
		const __m128i *BMRESTRICT	src4
	)

inline

AND block digest stride.

Returns: true if stide is all zero

Definition at line 455 of file bmsse4.h.

◆ sse4_and_or_digest_2way()

bool bm::sse4_and_or_digest_2way	(	__m128i *BMRESTRICT	dst,
		const __m128i *BMRESTRICT	src1,
		const __m128i *BMRESTRICT	src2
	)

inline

AND-OR block digest stride dst |= *src1 & src2.

Returns: true if stide is all zero

Definition at line 401 of file bmsse4.h.

◆ sse4_bit_count()

bm::id_t bm::sse4_bit_count	(	const __m128i *	block,
		const __m128i *	block_end
	)

inline

SSE4.2 optimized bitcounting .

Definition at line 93 of file bmsse4.h.

◆ sse4_block_set_digest()

BMFORCEINLINE void bm::sse4_block_set_digest	(	__m128i *	dst,
		unsigned	value
	)

set digest stride to 0xFF.. or 0x0 value

Definition at line 239 of file bmsse4.h.

◆ sse4_gap_find()

unsigned bm::sse4_gap_find	(	const bm::gap_word_t *BMRESTRICT	pbuf,
		const bm::gap_word_t	pos,
		const unsigned	size
	)

inline

SSE4.2 check for one to two (variable len) 128 bit SSE lines for gap search results (8 elements)

Definition at line 1099 of file bmsse4.h.

References BM_ASSERT, and BMRESTRICT.

Referenced by bm::sse42_gap_bfind().

◆ sse4_is_all_one()

bool bm::sse4_is_all_one ( const __m128i *BMRESTRICT block )

inline

check if block is all ONE bits

Definition at line 635 of file bmsse4.h.

References BMRESTRICT, and bm::set_block_size.

◆ sse4_is_all_zero()

bool bm::sse4_is_all_zero ( const __m128i *BMRESTRICT block )

inline

check if block is all zero bits

Definition at line 195 of file bmsse4.h.

References BMRESTRICT, and bm::set_block_size.

◆ sse4_is_digest_zero()

BMFORCEINLINE bool bm::sse4_is_digest_zero ( const __m128i *BMRESTRICT block )

check if digest stride is all zero bits

Definition at line 220 of file bmsse4.h.

◆ sse4_sub_digest()

BMFORCEINLINE bool bm::sse4_sub_digest	(	__m128i *BMRESTRICT	dst,
		const __m128i *BMRESTRICT	src
	)

SUB (AND NOT) block digest stride dst &= ~*src.

Returns: true if stide is all zero

Definition at line 538 of file bmsse4.h.

◆ sse4_sub_digest_2way()

BMFORCEINLINE bool bm::sse4_sub_digest_2way	(	__m128i *BMRESTRICT	dst,
		const __m128i *BMRESTRICT	src1,
		const __m128i *BMRESTRICT	src2
	)

2-operand SUB (AND NOT) block digest stride dst = src1 & ~*src2

Returns: true if stide is all zero

Definition at line 587 of file bmsse4.h.

Functions

Detailed Description

Function Documentation

◆ sse42_bit_block_calc_change()

◆ sse42_bit_block_calc_change_bc()

◆ sse42_bit_block_calc_xor_change()

◆ sse42_bit_block_xor()

◆ sse42_bit_block_xor_2way()

◆ sse42_bit_find_first()

◆ sse42_bit_find_first_diff()

◆ sse42_cmpge_u32()

◆ sse42_gap_bfind()

◆ sse42_gap_test()

◆ sse42_shift_l1()

◆ sse42_shift_r1()

◆ sse42_shift_r1_and()

◆ sse42_test_all_eq_wave2()

◆ sse42_test_all_one_wave()

◆ sse42_test_all_zero_wave()

◆ sse42_test_all_zero_wave2()

◆ sse4_and_block()

◆ sse4_and_digest()

◆ sse4_and_digest_2way()

◆ sse4_and_digest_5way()

◆ sse4_and_or_digest_2way()

◆ sse4_bit_count()

◆ sse4_block_set_digest()

◆ sse4_gap_find()

◆ sse4_is_all_one()

◆ sse4_is_all_zero()

◆ sse4_is_digest_zero()

◆ sse4_sub_digest()

◆ sse4_sub_digest_2way()