BitMagic-C++
Namespaces | Macros | Functions
bmavx2.h File Reference
#include <emmintrin.h>
#include <immintrin.h>
#include "bmdef.h"
#include "bmbmi2.h"
#include "bmutil.h"
Include dependency graph for bmavx2.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Namespaces

 bm
 

Macros

#define BM_CSA256(h, l, a, b, c)
 
#define BM_AVX2_BIT_COUNT(ret, v)
 
#define BM_AVX2_DECL_LOOKUP1
 
#define BM_AVX2_DECL_LOOKUP2
 
#define BM_AVX2_POPCNT_PROLOG
 
#define VECT_XOR_ARR_2_MASK(dst, src, src_end, mask)   avx2_xor_arr_2_mask((__m256i*)(dst), (__m256i*)(src), (__m256i*)(src_end), (bm::word_t)mask)
 
#define VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, mask)   avx2_andnot_arr_2_mask((__m256i*)(dst), (__m256i*)(src), (__m256i*)(src_end), (bm::word_t)mask)
 
#define VECT_BITCOUNT(first, last)   avx2_bit_count((__m256i*) (first), (__m256i*) (last))
 
#define VECT_BITCOUNT_AND(first, last, mask)   avx2_bit_count_and((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))
 
#define VECT_BITCOUNT_OR(first, last, mask)   avx2_bit_count_or((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))
 
#define VECT_BITCOUNT_XOR(first, last, mask)   avx2_bit_count_xor((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))
 
#define VECT_BITCOUNT_SUB(first, last, mask)   avx2_bit_count_sub((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))
 
#define VECT_INVERT_BLOCK(first)   avx2_invert_block((__m256i*)first);
 
#define VECT_AND_BLOCK(dst, src)   avx2_and_block((__m256i*) dst, (const __m256i*) (src))
 
#define VECT_AND_DIGEST(dst, src)   avx2_and_digest((__m256i*) dst, (const __m256i*) (src))
 
#define VECT_AND_DIGEST_2WAY(dst, src1, src2)   avx2_and_digest_2way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))
 
#define VECT_AND_DIGEST_5WAY(dst, src1, src2, src3, src4)   avx2_and_digest_5way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2), (const __m256i*) (src3), (const __m256i*) (src4))
 
#define VECT_OR_BLOCK(dst, src)   avx2_or_block((__m256i*) dst, (__m256i*) (src))
 
#define VECT_OR_BLOCK_3WAY(dst, src1, src2)   avx2_or_block_3way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))
 
#define VECT_OR_BLOCK_2WAY(dst, src1, src2)   avx2_or_block_2way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))
 
#define VECT_OR_BLOCK_3WAY(dst, src1, src2)   avx2_or_block_3way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))
 
#define VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4)   avx2_or_block_5way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2), (__m256i*) (src3), (__m256i*) (src4))
 
#define VECT_SUB_BLOCK(dst, src)   avx2_sub_block((__m256i*) dst, (__m256i*) (src))
 
#define VECT_SUB_DIGEST(dst, src)   avx2_sub_digest((__m256i*) dst, (const __m256i*) (src))
 
#define VECT_SUB_DIGEST_2WAY(dst, src1, src2)   avx2_sub_digest_2way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))
 
#define VECT_XOR_BLOCK(dst, src)   avx2_xor_block((__m256i*) dst, (__m256i*) (src))
 
#define VECT_XOR_BLOCK_2WAY(dst, src1, src2)   avx2_xor_block_2way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))
 
#define VECT_COPY_BLOCK(dst, src)   avx2_copy_block((__m256i*) dst, (__m256i*) (src))
 
#define VECT_STREAM_BLOCK(dst, src)   avx2_stream_block((__m256i*) dst, (__m256i*) (src))
 
#define VECT_SET_BLOCK(dst, value)   avx2_set_block((__m256i*) dst, (value))
 
#define VECT_IS_ZERO_BLOCK(dst)   avx2_is_all_zero((__m256i*) dst)
 
#define VECT_IS_ONE_BLOCK(dst)   avx2_is_all_one((__m256i*) dst)
 
#define VECT_IS_DIGEST_ZERO(start)   avx2_is_digest_zero((__m256i*)start)
 
#define VECT_BLOCK_SET_DIGEST(dst, val)   avx2_block_set_digest((__m256i*)dst, val)
 
#define VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to)   avx2_lower_bound_scan_u32(arr, target, from, to)
 
#define VECT_SHIFT_L1(b, acc, co)   avx2_shift_l1((__m256i*)b, acc, co)
 
#define VECT_SHIFT_R1(b, acc, co)   avx2_shift_r1((__m256i*)b, acc, co)
 
#define VECT_SHIFT_R1_AND(b, co, m, digest)   avx2_shift_r1_and((__m256i*)b, co, (__m256i*)m, digest)
 
#define VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start)   avx2_idx_arr_block_lookup(idx, size, nb, start)
 
#define VECT_SET_BLOCK_BITS(block, idx, start, stop)   avx2_set_block_bits3(block, idx, start, stop)
 
#define VECT_BLOCK_CHANGE(block, size)   avx2_bit_block_calc_change((__m256i*)block, size)
 
#define VECT_BLOCK_XOR_CHANGE(block, xor_block, size, gc, bc)   avx2_bit_block_calc_xor_change((__m256i*)block, (__m256i*)xor_block, size, gc, bc)
 
#define VECT_BLOCK_CHANGE_BC(block, gc, bc)   avx2_bit_block_calc_change_bc((__m256i*)block, gc, bc)
 
#define VECT_BIT_TO_GAP(dest, src, dest_len)   avx2_bit_to_gap(dest, src, dest_len)
 
#define VECT_BIT_FIND_FIRST(src1, pos)   avx2_bit_find_first((__m256i*) src1, pos)
 
#define VECT_BIT_FIND_DIFF(src1, src2, pos)   avx2_bit_find_first_diff((__m256i*) src1, (__m256i*) (src2), pos)
 
#define VECT_BIT_BLOCK_XOR(t, src, src_xor, d)   avx2_bit_block_xor(t, src, src_xor, d)
 
#define VECT_GAP_BFIND(buf, pos, is_set)   avx2_gap_bfind(buf, pos, is_set)
 
#define VECT_BIT_COUNT_DIGEST(blk, d)   avx2_bit_block_count(blk, d)
 

Functions

bm::id_t bm::avx2_bit_count (const __m256i *BMRESTRICT block, const __m256i *BMRESTRICT block_end)
 AVX2 Harley-Seal popcount The algorithm is based on the paper "Faster Population Counts using AVX2 Instructions" by Daniel Lemire, Nathan Kurz and Wojciech Mula (23 Nov 2016). More...
 
bm::id_t bm::avx2_bit_block_count (const bm::word_t *const block, bm::id64_t digest)
 Calculate population count based on digest. More...
 
bm::id_t bm::avx2_bit_count_and (const __m256i *BMRESTRICT block, const __m256i *BMRESTRICT block_end, const __m256i *BMRESTRICT mask_block)
 AND bit count for two aligned bit-blocks. More...
 
bm::id_t bm::avx2_bit_count_or (const __m256i *BMRESTRICT block, const __m256i *BMRESTRICT block_end, const __m256i *BMRESTRICT mask_block)
 
bm::id_t bm::avx2_bit_count_xor (const __m256i *BMRESTRICT block, const __m256i *BMRESTRICT block_end, const __m256i *BMRESTRICT mask_block)
 XOR bit count for two aligned bit-blocks. More...
 
bm::id_t bm::avx2_bit_count_sub (const __m256i *BMRESTRICT block, const __m256i *BMRESTRICT block_end, const __m256i *BMRESTRICT mask_block)
 AND NOT bit count for two aligned bit-blocks. More...
 
void bm::avx2_xor_arr_2_mask (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src, const __m256i *BMRESTRICT src_end, bm::word_t mask)
 XOR array elements to specified mask dst = *src ^ mask. More...
 
void bm::avx2_andnot_arr_2_mask (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src, const __m256i *BMRESTRICT src_end, bm::word_t mask)
 Inverts array elements and NOT them to specified mask dst = ~*src & mask. More...
 
unsigned bm::avx2_and_block (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src)
 AND array elements against another array dst &= *src. More...
 
bool bm::avx2_and_digest (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src)
 AND block digest stride dst &= *src. More...
 
bool bm::avx2_and_digest_2way (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src1, const __m256i *BMRESTRICT src2)
 AND block digest stride 2 way dst = *src1 & *src2. More...
 
bool bm::avx2_and_digest_5way (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src1, const __m256i *BMRESTRICT src2, const __m256i *BMRESTRICT src3, const __m256i *BMRESTRICT src4)
 AND block digest stride. More...
 
unsigned bm::avx2_and_arr_unal (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src, const __m256i *BMRESTRICT src_end)
 AND array elements against another array (unaligned) dst &= *src. More...
 
bool bm::avx2_or_block (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src)
 OR array elements against another array dst |= *src. More...
 
bool bm::avx2_or_arr_unal (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src, const __m256i *BMRESTRICT src_end)
 OR array elements against another unaligned array dst |= *src. More...
 
bool bm::avx2_or_block_2way (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src1, const __m256i *BMRESTRICT src2)
 OR 2 arrays and copy to the destination dst = *src1 | src2. More...
 
bool bm::avx2_or_block_3way (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src1, const __m256i *BMRESTRICT src2)
 OR array elements against another 2 arrays dst |= *src1 | src2. More...
 
bool bm::avx2_or_block_5way (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src1, const __m256i *BMRESTRICT src2, const __m256i *BMRESTRICT src3, const __m256i *BMRESTRICT src4)
 OR array elements against another 4 arrays dst |= *src1 | src2. More...
 
unsigned bm::avx2_xor_block (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src)
 XOR block against another dst ^= *src. More...
 
unsigned bm::avx2_xor_block_2way (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src1, const __m256i *BMRESTRICT src2)
 3 operand XOR dst = *src1 ^ src2 More...
 
unsigned bm::avx2_sub_block (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src)
 AND-NOT (SUB) array elements against another array dst &= ~*src. More...
 
bool bm::avx2_sub_digest (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src)
 SUB (AND NOT) block digest stride dst &= ~*src. More...
 
bool bm::avx2_sub_digest_2way (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src1, const __m256i *BMRESTRICT src2)
 2-operand SUB (AND NOT) block digest stride dst = *src1 & ~*src2 More...
 
BMFORCEINLINE void bm::avx2_set_block (__m256i *BMRESTRICT dst, bm::word_t value)
 AVX2 block memset dst = value. More...
 
void bm::avx2_copy_block (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src)
 AVX2 block copy dst = *src. More...
 
void bm::avx2_stream_block (__m256i *BMRESTRICT dst, const __m256i *BMRESTRICT src)
 AVX2 block copy dst = *src. More...
 
void bm::avx2_invert_block (__m256i *BMRESTRICT dst)
 Invert bit-block dst = ~*dst or dst ^= *dst. More...
 
bool bm::avx2_is_all_zero (const __m256i *BMRESTRICT block)
 check if block is all zero bits More...
 
bool bm::avx2_is_digest_zero (const __m256i *BMRESTRICT block)
 check if digest stride is all zero bits More...
 
void bm::avx2_block_set_digest (__m256i *dst, unsigned value)
 set digest stride to 0xFF.. or 0x0 value More...
 
bool bm::avx2_is_all_one (const __m256i *BMRESTRICT block)
 check if block is all one bits More...
 
BMFORCEINLINE bool bm::avx2_test_all_one_wave (const void *ptr)
 check if wave of pointers is all 0xFFF More...
 
BMFORCEINLINE bool bm::avx2_test_all_zero_wave (const void *ptr)
 check if wave of pointers is all NULL More...
 
BMFORCEINLINE bool bm::avx2_test_all_zero_wave2 (const void *ptr0, const void *ptr1)
 check if 2 wave of pointers are all NULL More...
 
BMFORCEINLINE bool bm::avx2_test_all_eq_wave2 (const void *ptr0, const void *ptr1)
 check if 2 wave of pointers are all the same (NULL or FULL) More...
 
bool bm::avx2_shift_l1 (__m256i *block, bm::word_t *empty_acc, unsigned co1)
 block shift left by 1 More...
 
bool bm::avx2_shift_r1 (__m256i *block, bm::word_t *empty_acc, unsigned co1)
 block shift right by 1 More...
 
bool bm::avx2_shift_r1_and (__m256i *BMRESTRICT block, bm::word_t co1, const __m256i *BMRESTRICT mask_block, bm::id64_t *BMRESTRICT digest)
 fused block shift right by 1 plus AND More...
 
unsigned bm::avx2_bit_block_calc_change (const __m256i *BMRESTRICT block, unsigned size)
 
void bm::avx2_bit_block_calc_xor_change (const __m256i *BMRESTRICT block, const __m256i *BMRESTRICT xor_block, unsigned size, unsigned *BMRESTRICT gcount, unsigned *BMRESTRICT bcount)
 
void bm::avx2_bit_block_calc_change_bc (const __m256i *BMRESTRICT block, unsigned *gcount, unsigned *bcount)
 
bool bm::avx2_bit_find_first_diff (const __m256i *BMRESTRICT block1, const __m256i *BMRESTRICT block2, unsigned *pos)
 Find first bit which is different between two bit-blocks. More...
 
bool bm::avx2_bit_find_first (const __m256i *BMRESTRICT block, unsigned *pos)
 Find first bit set. More...
 
const bm::gap_word_tbm::avx2_gap_sum_arr (const bm::gap_word_t *pbuf, unsigned avx_vect_waves, unsigned *sum)
 
unsigned bm::avx2_idx_arr_block_lookup (const unsigned *idx, unsigned size, unsigned nb, unsigned start)
 
void bm::avx2_set_block_bits (bm::word_t *BMRESTRICT block, const unsigned *BMRESTRICT idx, unsigned start, unsigned stop)
 
BMFORCEINLINE __m256i bm::avx2_setbit_256 (__m256i target, __m256i source)
 Set a bits in an AVX target, by indexes (int4) from the source. More...
 
void bm::avx2_set_block_bits2 (bm::word_t *BMRESTRICT block, const unsigned *BMRESTRICT idx, unsigned start, unsigned stop)
 Experimental code to set bits via AVX strides. More...
 
void bm::avx2_set_block_bits3 (bm::word_t *BMRESTRICT block, const unsigned *BMRESTRICT idx, unsigned start, unsigned stop)
 Experimental code to set bits via AVX strides. More...
 
__m256i bm::avx2_setbit_to256 (unsigned i)
 Experiemntal. More...
 
int bm::avx2_cmpge_u32 (__m256i vect8, unsigned value)
 Experimental (test) function to do SIMD vector search (lower bound) in sorted, growing array. More...
 
int bm::avx2_cmpge_u16 (__m256i vect16, unsigned short value)
 Experimental (test) function to do SIMD vector search in sorted, growing array. More...
 
unsigned bm::avx2_gap_bfind (const unsigned short *BMRESTRICT buf, unsigned pos, unsigned *BMRESTRICT is_set)
 Hybrid binary search, starts as binary, then switches to scan. More...
 
unsigned bm::avx2_gap_test (const unsigned short *BMRESTRICT buf, unsigned pos)
 Hybrid binary search, starts as binary, then switches to scan. More...
 
unsigned bm::avx2_lower_bound_scan_u32 (const unsigned *BMRESTRICT arr, unsigned target, unsigned from, unsigned to)
 lower bound (great or equal) linear scan in ascending order sorted array More...
 
void bm::avx2_bit_block_gather_scatter (unsigned *BMRESTRICT arr, const unsigned *BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx)
 
unsigned bm::avx2_bit_to_gap (gap_word_t *BMRESTRICT dest, const unsigned *BMRESTRICT block, unsigned dest_len)
 Convert bit block to GAP block. More...
 
void bm::avx2_bit_block_xor (bm::word_t *target_block, const bm::word_t *block, const bm::word_t *xor_block, bm::id64_t digest)
 Build partial XOR product of 2 bit-blocks using digest mask. More...
 

Macro Definition Documentation

◆ BM_AVX2_BIT_COUNT

#define BM_AVX2_BIT_COUNT (   ret,
 
)
Value:
{ \
__m256i lo = _mm256_and_si256(v, low_mask); \
__m256i hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), low_mask); \
__m256i cnt1 = _mm256_shuffle_epi8(lookup1, lo); \
__m256i cnt2 = _mm256_shuffle_epi8(lookup2, hi); \
ret = _mm256_sad_epu8(cnt1, cnt2); \
}

Definition at line 124 of file bmavx2.h.

Referenced by bm::avx2_bit_block_calc_change(), bm::avx2_bit_block_calc_change_bc(), bm::avx2_bit_block_calc_xor_change(), bm::avx2_bit_block_count(), bm::avx2_bit_count(), bm::avx2_bit_count_and(), bm::avx2_bit_count_or(), bm::avx2_bit_count_sub(), bm::avx2_bit_count_xor(), and bm::bit_block_calc_count_to().

◆ BM_AVX2_DECL_LOOKUP1

#define BM_AVX2_DECL_LOOKUP1
Value:
__m256i lookup1 = _mm256_setr_epi8(4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, \
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8);

Definition at line 133 of file bmavx2.h.

◆ BM_AVX2_DECL_LOOKUP2

#define BM_AVX2_DECL_LOOKUP2
Value:
__m256i lookup2 = _mm256_setr_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, \
4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0);

Definition at line 136 of file bmavx2.h.

◆ BM_AVX2_POPCNT_PROLOG

#define BM_AVX2_POPCNT_PROLOG

◆ BM_CSA256

#define BM_CSA256 (   h,
  l,
  a,
  b,
 
)
Value:
{ \
__m256i u = _mm256_xor_si256(a, b); \
h = _mm256_or_si256(_mm256_and_si256(a, b), _mm256_and_si256(u, c)); \
l = _mm256_xor_si256(u, c); \
}

Definition at line 117 of file bmavx2.h.

Referenced by bm::avx2_bit_count().

◆ VECT_AND_BLOCK

#define VECT_AND_BLOCK (   dst,
  src 
)    avx2_and_block((__m256i*) dst, (const __m256i*) (src))

Definition at line 3035 of file bmavx2.h.

Referenced by bm::bit_block_and().

◆ VECT_AND_DIGEST

#define VECT_AND_DIGEST (   dst,
  src 
)    avx2_and_digest((__m256i*) dst, (const __m256i*) (src))

Definition at line 3038 of file bmavx2.h.

Referenced by bm::bit_block_and().

◆ VECT_AND_DIGEST_2WAY

#define VECT_AND_DIGEST_2WAY (   dst,
  src1,
  src2 
)    avx2_and_digest_2way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))

Definition at line 3041 of file bmavx2.h.

Referenced by bm::bit_block_and_2way().

◆ VECT_AND_DIGEST_5WAY

#define VECT_AND_DIGEST_5WAY (   dst,
  src1,
  src2,
  src3,
  src4 
)    avx2_and_digest_5way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2), (const __m256i*) (src3), (const __m256i*) (src4))

Definition at line 3044 of file bmavx2.h.

Referenced by bm::bit_block_and_5way().

◆ VECT_ANDNOT_ARR_2_MASK

#define VECT_ANDNOT_ARR_2_MASK (   dst,
  src,
  src_end,
  mask 
)    avx2_andnot_arr_2_mask((__m256i*)(dst), (__m256i*)(src), (__m256i*)(src_end), (bm::word_t)mask)

Definition at line 3014 of file bmavx2.h.

Referenced by bm::bit_andnot_arr_ffmask().

◆ VECT_ARR_BLOCK_LOOKUP

#define VECT_ARR_BLOCK_LOOKUP (   idx,
  size,
  nb,
  start 
)    avx2_idx_arr_block_lookup(idx, size, nb, start)

Definition at line 3110 of file bmavx2.h.

Referenced by bm::idx_arr_block_lookup_u32().

◆ VECT_BIT_BLOCK_XOR

#define VECT_BIT_BLOCK_XOR (   t,
  src,
  src_xor,
 
)    avx2_bit_block_xor(t, src, src_xor, d)

Definition at line 3134 of file bmavx2.h.

Referenced by bm::bit_block_xor().

◆ VECT_BIT_COUNT_DIGEST

#define VECT_BIT_COUNT_DIGEST (   blk,
 
)    avx2_bit_block_count(blk, d)

Definition at line 3140 of file bmavx2.h.

Referenced by bm::bit_block_count().

◆ VECT_BIT_FIND_DIFF

#define VECT_BIT_FIND_DIFF (   src1,
  src2,
  pos 
)    avx2_bit_find_first_diff((__m256i*) src1, (__m256i*) (src2), pos)

Definition at line 3131 of file bmavx2.h.

Referenced by bm::bit_find_first_diff().

◆ VECT_BIT_FIND_FIRST

#define VECT_BIT_FIND_FIRST (   src1,
  pos 
)    avx2_bit_find_first((__m256i*) src1, pos)

Definition at line 3128 of file bmavx2.h.

Referenced by bm::bit_find_first().

◆ VECT_BIT_TO_GAP

#define VECT_BIT_TO_GAP (   dest,
  src,
  dest_len 
)    avx2_bit_to_gap(dest, src, dest_len)

Definition at line 3125 of file bmavx2.h.

Referenced by bm::bit_to_gap().

◆ VECT_BITCOUNT

#define VECT_BITCOUNT (   first,
  last 
)    avx2_bit_count((__m256i*) (first), (__m256i*) (last))

Definition at line 3017 of file bmavx2.h.

Referenced by bm::bit_block_count().

◆ VECT_BITCOUNT_AND

#define VECT_BITCOUNT_AND (   first,
  last,
  mask 
)    avx2_bit_count_and((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))

Definition at line 3020 of file bmavx2.h.

Referenced by bm::bit_block_and_count().

◆ VECT_BITCOUNT_OR

#define VECT_BITCOUNT_OR (   first,
  last,
  mask 
)    avx2_bit_count_or((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))

Definition at line 3023 of file bmavx2.h.

Referenced by bm::bit_block_or_count().

◆ VECT_BITCOUNT_SUB

#define VECT_BITCOUNT_SUB (   first,
  last,
  mask 
)    avx2_bit_count_sub((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))

Definition at line 3029 of file bmavx2.h.

Referenced by bm::bit_block_sub_count().

◆ VECT_BITCOUNT_XOR

#define VECT_BITCOUNT_XOR (   first,
  last,
  mask 
)    avx2_bit_count_xor((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))

Definition at line 3026 of file bmavx2.h.

Referenced by bm::bit_block_xor_count().

◆ VECT_BLOCK_CHANGE

#define VECT_BLOCK_CHANGE (   block,
  size 
)    avx2_bit_block_calc_change((__m256i*)block, size)

Definition at line 3116 of file bmavx2.h.

Referenced by bm::bit_block_calc_change(), and bm::compute_complexity_descr().

◆ VECT_BLOCK_CHANGE_BC

#define VECT_BLOCK_CHANGE_BC (   block,
  gc,
  bc 
)    avx2_bit_block_calc_change_bc((__m256i*)block, gc, bc)

Definition at line 3122 of file bmavx2.h.

Referenced by bm::bit_block_change_bc().

◆ VECT_BLOCK_SET_DIGEST

#define VECT_BLOCK_SET_DIGEST (   dst,
  val 
)    avx2_block_set_digest((__m256i*)dst, val)

Definition at line 3095 of file bmavx2.h.

Referenced by bm::block_init_digest0().

◆ VECT_BLOCK_XOR_CHANGE

#define VECT_BLOCK_XOR_CHANGE (   block,
  xor_block,
  size,
  gc,
  bc 
)    avx2_bit_block_calc_xor_change((__m256i*)block, (__m256i*)xor_block, size, gc, bc)

Definition at line 3119 of file bmavx2.h.

Referenced by bm::bit_block_xor_change().

◆ VECT_COPY_BLOCK

#define VECT_COPY_BLOCK (   dst,
  src 
)    avx2_copy_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3077 of file bmavx2.h.

Referenced by bm::bit_block_copy().

◆ VECT_GAP_BFIND

#define VECT_GAP_BFIND (   buf,
  pos,
  is_set 
)    avx2_gap_bfind(buf, pos, is_set)

Definition at line 3137 of file bmavx2.h.

Referenced by bm::gap_bfind().

◆ VECT_INVERT_BLOCK

#define VECT_INVERT_BLOCK (   first)    avx2_invert_block((__m256i*)first);

Definition at line 3032 of file bmavx2.h.

Referenced by bm::bit_invert().

◆ VECT_IS_DIGEST_ZERO

#define VECT_IS_DIGEST_ZERO (   start)    avx2_is_digest_zero((__m256i*)start)

Definition at line 3092 of file bmavx2.h.

Referenced by bm::calc_block_digest0(), and bm::update_block_digest0().

◆ VECT_IS_ONE_BLOCK

#define VECT_IS_ONE_BLOCK (   dst)    avx2_is_all_one((__m256i*) dst)

Definition at line 3089 of file bmavx2.h.

Referenced by bm::is_bits_one().

◆ VECT_IS_ZERO_BLOCK

#define VECT_IS_ZERO_BLOCK (   dst)    avx2_is_all_zero((__m256i*) dst)

Definition at line 3086 of file bmavx2.h.

Referenced by bm::bit_is_all_zero().

◆ VECT_LOWER_BOUND_SCAN_U32

#define VECT_LOWER_BOUND_SCAN_U32 (   arr,
  target,
  from,
  to 
)    avx2_lower_bound_scan_u32(arr, target, from, to)

Definition at line 3098 of file bmavx2.h.

Referenced by bm::lower_bound_linear_u32().

◆ VECT_OR_BLOCK

#define VECT_OR_BLOCK (   dst,
  src 
)    avx2_or_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3047 of file bmavx2.h.

Referenced by bm::bit_block_or().

◆ VECT_OR_BLOCK_2WAY

#define VECT_OR_BLOCK_2WAY (   dst,
  src1,
  src2 
)    avx2_or_block_2way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))

Definition at line 3053 of file bmavx2.h.

Referenced by bm::bit_block_or_2way().

◆ VECT_OR_BLOCK_3WAY [1/2]

#define VECT_OR_BLOCK_3WAY (   dst,
  src1,
  src2 
)    avx2_or_block_3way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))

Definition at line 3056 of file bmavx2.h.

Referenced by bm::bit_block_or_3way().

◆ VECT_OR_BLOCK_3WAY [2/2]

#define VECT_OR_BLOCK_3WAY (   dst,
  src1,
  src2 
)    avx2_or_block_3way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))

Definition at line 3056 of file bmavx2.h.

◆ VECT_OR_BLOCK_5WAY

#define VECT_OR_BLOCK_5WAY (   dst,
  src1,
  src2,
  src3,
  src4 
)    avx2_or_block_5way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2), (__m256i*) (src3), (__m256i*) (src4))

Definition at line 3059 of file bmavx2.h.

Referenced by bm::bit_block_or_5way().

◆ VECT_SET_BLOCK

#define VECT_SET_BLOCK (   dst,
  value 
)    avx2_set_block((__m256i*) dst, (value))

Definition at line 3083 of file bmavx2.h.

Referenced by bm::bit_block_set().

◆ VECT_SET_BLOCK_BITS

#define VECT_SET_BLOCK_BITS (   block,
  idx,
  start,
  stop 
)    avx2_set_block_bits3(block, idx, start, stop)

Definition at line 3113 of file bmavx2.h.

Referenced by bm::set_block_bits_u32().

◆ VECT_SHIFT_L1

#define VECT_SHIFT_L1 (   b,
  acc,
  co 
)    avx2_shift_l1((__m256i*)b, acc, co)

Definition at line 3101 of file bmavx2.h.

Referenced by bm::bit_block_shift_l1_unr().

◆ VECT_SHIFT_R1

#define VECT_SHIFT_R1 (   b,
  acc,
  co 
)    avx2_shift_r1((__m256i*)b, acc, co)

Definition at line 3104 of file bmavx2.h.

Referenced by bm::bit_block_shift_r1_unr().

◆ VECT_SHIFT_R1_AND

#define VECT_SHIFT_R1_AND (   b,
  co,
  m,
  digest 
)    avx2_shift_r1_and((__m256i*)b, co, (__m256i*)m, digest)

Definition at line 3107 of file bmavx2.h.

Referenced by bm::bit_block_shift_r1_and_unr().

◆ VECT_STREAM_BLOCK

#define VECT_STREAM_BLOCK (   dst,
  src 
)    avx2_stream_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3080 of file bmavx2.h.

Referenced by bm::bit_block_stream().

◆ VECT_SUB_BLOCK

#define VECT_SUB_BLOCK (   dst,
  src 
)    avx2_sub_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3062 of file bmavx2.h.

Referenced by bm::bit_block_sub().

◆ VECT_SUB_DIGEST

#define VECT_SUB_DIGEST (   dst,
  src 
)    avx2_sub_digest((__m256i*) dst, (const __m256i*) (src))

Definition at line 3065 of file bmavx2.h.

Referenced by bm::bit_block_sub().

◆ VECT_SUB_DIGEST_2WAY

#define VECT_SUB_DIGEST_2WAY (   dst,
  src1,
  src2 
)    avx2_sub_digest_2way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))

Definition at line 3068 of file bmavx2.h.

Referenced by bm::bit_block_sub_2way().

◆ VECT_XOR_ARR_2_MASK

#define VECT_XOR_ARR_2_MASK (   dst,
  src,
  src_end,
  mask 
)    avx2_xor_arr_2_mask((__m256i*)(dst), (__m256i*)(src), (__m256i*)(src_end), (bm::word_t)mask)

Definition at line 3011 of file bmavx2.h.

Referenced by bm::bvector<>::combine_operation().

◆ VECT_XOR_BLOCK

#define VECT_XOR_BLOCK (   dst,
  src 
)    avx2_xor_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3071 of file bmavx2.h.

Referenced by bm::bit_block_xor().

◆ VECT_XOR_BLOCK_2WAY

#define VECT_XOR_BLOCK_2WAY (   dst,
  src1,
  src2 
)    avx2_xor_block_2way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))

Definition at line 3074 of file bmavx2.h.

Referenced by bm::bit_block_xor_2way().