507 lines
25 KiB
Rust
507 lines
25 KiB
Rust
// diff_fast corpus program: the AVX / AVX2 / FMA double-precision intrinsics of core::arch::x86_64
|
|
// (_mm256_*_pd and the 128-bit AVX/FMA _pd/_sd forms: arithmetic, logic, compares with all 32 predicates,
|
|
// shuffles/permutes/blends with their immediates, broadcasts, masked loads/stores, gathers, casts, conversions,
|
|
// tests, fused multiply-adds) on the edge lanes of arch_sse2_pd and random bits, bit-exact, one hash per
|
|
// intrinsic (and immediate), `D <name> <hash>`. Runs where the CPU has AVX2 + FMA (`D no avx2/fma` otherwise);
|
|
// x86-64 only. ARCH_DETAIL=<name> prints every case.
|
|
#![allow(unused_unsafe, deprecated, non_snake_case)]
|
|
macro_rules! d { ($($a:tt)*) => { println!("D {}", format!($($a)*)) } }
|
|
fn rounds(f: fn()) {
|
|
let ms: u64 = std::env::var("DIFF_SLEEP_MS").ok().and_then(|v| v.parse().ok()).unwrap_or(0);
|
|
for r in 0..4 { f(); if r < 3 && ms > 0 { std::thread::sleep(std::time::Duration::from_millis(ms)); } }
|
|
}
|
|
#[test]
|
|
fn run() { rounds(work); }
|
|
|
|
#[cfg(not(target_arch = "x86_64"))]
|
|
fn work() {
|
|
d!("sse2 pd: x86-64 only");
|
|
}
|
|
|
|
#[cfg(target_arch = "x86_64")]
|
|
fn work() {
|
|
if !(std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("fma")) {
|
|
d!("no avx2/fma");
|
|
return;
|
|
}
|
|
unsafe { x::all() };
|
|
}
|
|
|
|
#[cfg(target_arch = "x86_64")]
|
|
mod x {
|
|
use core::arch::x86_64::*;
|
|
use std::hint::black_box;
|
|
|
|
struct Rng(u64);
|
|
impl Rng {
|
|
fn next(&mut self) -> u64 {
|
|
self.0 ^= self.0 << 13;
|
|
self.0 ^= self.0 >> 7;
|
|
self.0 ^= self.0 << 17;
|
|
self.0
|
|
}
|
|
}
|
|
|
|
const EDGE64: [u64; 40] = [
|
|
0x0000_0000_0000_0000, // 0.0
|
|
0x8000_0000_0000_0000, // -0.0
|
|
0x3ff0_0000_0000_0000, // 1.0
|
|
0xbff0_0000_0000_0000, // -1.0
|
|
0x3fe0_0000_0000_0000, // 0.5
|
|
0xbfe0_0000_0000_0000, // -0.5
|
|
0x3ff8_0000_0000_0000, // 1.5
|
|
0x4004_0000_0000_0000, // 2.5
|
|
0xc004_0000_0000_0000, // -2.5
|
|
0x3fdf_ffff_ffff_ffff, // 0.49999999999999994
|
|
0x7ff0_0000_0000_0000, // inf
|
|
0xfff0_0000_0000_0000, // -inf
|
|
0x7fef_ffff_ffff_ffff, // f64::MAX
|
|
0xffef_ffff_ffff_ffff, // f64::MIN
|
|
0x0010_0000_0000_0000, // MIN_POSITIVE
|
|
0x0000_0000_0000_0001, // smallest subnormal
|
|
0x800f_ffff_ffff_ffff, // largest negative subnormal
|
|
0x41df_ffff_ffe0_0000, // 2147483647.5
|
|
0x41e0_0000_0000_0000, // 2147483648.0
|
|
0xc1e0_0000_0010_0000, // -2147483648.5
|
|
0xc1e0_0000_0020_0000, // -2147483649.0
|
|
0x41df_ffff_ffc0_0000, // 2147483647.0
|
|
0xc1e0_0000_0000_0000, // -2147483648.0
|
|
0x43e0_0000_0000_0000, // 9223372036854775808.0
|
|
0xc3e0_0000_0000_0000, // -9223372036854775808.0
|
|
0x43df_ffff_ffff_ffff, // 9223372036854774784.0
|
|
0x47ef_ffff_e000_0000, // f32::MAX
|
|
0x47ef_ffff_f000_0000, // f32::MAX + half ulp (rounds to inf in f32)
|
|
0x3690_0000_0000_0000, // ~1.0e-45 range: f32 subnormal territory
|
|
0x3ff0_0000_1000_0000, // 1 + 2^-24: an f32 rounding tie
|
|
0x4330_0000_0000_0001, // 2^52 + 1
|
|
0x7ff8_0000_0000_0000, // qNaN
|
|
0xfff8_0000_0000_0000, // -qNaN (Intel default NaN)
|
|
0x7ff0_0000_0000_0001, // sNaN
|
|
0xfff4_0000_1234_5678, // -sNaN with payload
|
|
0x7ff8_0000_1234_5678, // qNaN with payload
|
|
0x7fff_ffff_ffff_ffff, // qNaN all payload bits
|
|
0x4024_0000_0000_0000, // 10.0
|
|
0xc059_0000_0000_0000, // -100.0
|
|
0x3fb9_9999_9999_999a, // 0.1
|
|
];
|
|
#[allow(dead_code)]
|
|
const EDGE32: [u32; 20] = [
|
|
0x0000_0000, 0x8000_0000, 0x3f80_0000, 0xbf80_0000, 0x3fc0_0000, 0x4020_0000, 0xc020_0000, 0x7f80_0000,
|
|
0xff80_0000, 0x7f7f_ffff, 0x0000_0001, 0x4f00_0000, 0xcf00_0000, 0x4eff_ffff, 0xcf00_0001, 0x7fc0_0000,
|
|
0xffc0_0000, 0x7f80_0001, 0xffa0_1234, 0x7fc0_5678,
|
|
];
|
|
#[allow(dead_code)]
|
|
const EDGEI32: [i32; 8] = [0, 1, -1, i32::MAX, i32::MIN, 16777217, -16777217, 123456789];
|
|
#[allow(dead_code)]
|
|
const EDGEI64: [i64; 8] = [0, 1, -1, i64::MAX, i64::MIN, 9007199254740993, -9007199254740993, 1234567890123456789];
|
|
|
|
fn mix(h: &mut u64, x: u64) {
|
|
*h = (*h ^ x).wrapping_mul(0x100000001b3).rotate_left(7);
|
|
}
|
|
fn pd(lo: u64, hi: u64) -> __m128d {
|
|
black_box(unsafe { core::mem::transmute::<[u64; 2], __m128d>([lo, hi]) })
|
|
}
|
|
fn bits_pd(v: __m128d) -> [u64; 2] {
|
|
unsafe { core::mem::transmute(v) }
|
|
}
|
|
fn pd4(l: [u64; 4]) -> __m256d {
|
|
black_box(unsafe { core::mem::transmute::<[u64; 4], __m256d>(l) })
|
|
}
|
|
fn bits4(v: __m256d) -> [[u64; 2]; 2] {
|
|
unsafe { core::mem::transmute(v) }
|
|
}
|
|
fn si4(l: [u64; 4]) -> __m256i {
|
|
black_box(unsafe { core::mem::transmute::<[u64; 4], __m256i>(l) })
|
|
}
|
|
|
|
/// One intrinsic's results: a hash, or every case when ARCH_DETAIL names it.
|
|
struct Out {
|
|
name: &'static str,
|
|
h: u64,
|
|
n: u64,
|
|
detail: bool,
|
|
}
|
|
impl Out {
|
|
fn new(name: &'static str) -> Out {
|
|
let detail = std::env::var("ARCH_DETAIL").map(|v| v == name).unwrap_or(false);
|
|
Out { name, h: 0xcbf29ce484222325, n: 0, detail }
|
|
}
|
|
fn put(&mut self, input: &[u64], r: [u64; 2]) {
|
|
if self.detail {
|
|
println!("D {} {:016x?} -> {:016x} {:016x}", self.name, input, r[1], r[0]);
|
|
}
|
|
mix(&mut self.h, r[0]);
|
|
mix(&mut self.h, r[1]);
|
|
self.n += 1;
|
|
}
|
|
fn put4(&mut self, input: &[u64], r: __m256d) {
|
|
let [a, b] = bits4(r);
|
|
self.put(input, a);
|
|
self.put(input, b);
|
|
}
|
|
fn end(self) {
|
|
d!("{} n={} {:016x}", self.name, self.n, self.h);
|
|
}
|
|
}
|
|
|
|
/// f64 lane values: every edge, then random bits and random finite values.
|
|
fn lanes64(rng: &mut Rng) -> Vec<u64> {
|
|
let mut v = EDGE64.to_vec();
|
|
for _ in 0..24 {
|
|
v.push(rng.next());
|
|
let f = (rng.next() as i64 as f64) / ((rng.next() >> 40) as f64 + 1.0);
|
|
v.push(f.to_bits());
|
|
}
|
|
v
|
|
}
|
|
|
|
/// at most one NaN operand per lane for the fused ops: with several, the NaN rustc returns depends on the
|
|
/// FMA instruction form LLVM picks (operand order), which Rust leaves unspecified
|
|
fn one_nan(a: &mut [u64], b: &mut [u64], c: &mut [u64]) {
|
|
let nan = |x: u64| f64::from_bits(x).is_nan();
|
|
for i in 0..a.len() {
|
|
if nan(a[i]) {
|
|
if nan(b[i]) { b[i] = 0x3ff8_0000_0000_0000; }
|
|
if nan(c[i]) { c[i] = 0xc004_0000_0000_0000; }
|
|
} else if nan(b[i]) && nan(c[i]) {
|
|
c[i] = 0x4024_0000_0000_0000;
|
|
}
|
|
}
|
|
}
|
|
// lane values of a 4-lane case k: edge pairs walked with different strides
|
|
fn four(v: &[u64], i: usize, j: usize, s: usize) -> [u64; 4] {
|
|
let n = v.len();
|
|
[v[i % n], v[(i + s) % n], v[j % n], v[(j * 3 + s) % n]]
|
|
}
|
|
macro_rules! bin4 {
|
|
($rng:ident, $($name:expr => |$a:ident, $b:ident| $e:expr;)*) => {$({
|
|
#[inline(never)]
|
|
#[target_feature(enable = "avx,avx2,fma")]
|
|
unsafe fn op($a: __m256d, $b: __m256d) -> __m256d { $e }
|
|
let mut o = Out::new($name);
|
|
let v = lanes64($rng);
|
|
for i in 0..v.len() {
|
|
for j in 0..v.len() {
|
|
let (x, y) = (four(&v, i, j, 5), four(&v, j, i, 11));
|
|
o.put4(&[x[0], y[0]], op(pd4(x), pd4(y)));
|
|
}
|
|
}
|
|
o.end();
|
|
})*};
|
|
}
|
|
macro_rules! un4 {
|
|
($rng:ident, $($name:expr => |$a:ident| $e:expr;)*) => {$({
|
|
#[inline(never)]
|
|
#[target_feature(enable = "avx,avx2,fma")]
|
|
unsafe fn op($a: __m256d) -> __m256d { $e }
|
|
let mut o = Out::new($name);
|
|
let v = lanes64($rng);
|
|
for i in 0..v.len() {
|
|
for j in 0..8 {
|
|
let x = four(&v, i, j * 5 + i, 3);
|
|
o.put4(&[x[0]], op(pd4(x)));
|
|
}
|
|
}
|
|
o.end();
|
|
})*};
|
|
}
|
|
// any result type, from two 256-bit inputs, hashed through its bits
|
|
macro_rules! any4 {
|
|
($rng:ident, $($name:expr => |$a:ident, $b:ident| -> $t:ty { $e:expr };)*) => {$({
|
|
#[inline(never)]
|
|
#[target_feature(enable = "avx,avx2,fma")]
|
|
unsafe fn op($a: __m256d, $b: __m256d) -> $t { $e }
|
|
let mut o = Out::new($name);
|
|
let v = lanes64($rng);
|
|
for i in 0..v.len() {
|
|
for j in 0..v.len() {
|
|
let (x, y) = (four(&v, i, j, 5), four(&v, j, i, 11));
|
|
let r = op(pd4(x), pd4(y));
|
|
let mut w = [0u64; 4];
|
|
unsafe { core::ptr::copy_nonoverlapping(&r as *const $t as *const u8, w.as_mut_ptr() as *mut u8, core::mem::size_of::<$t>()) };
|
|
o.put(&[x[0], y[0]], [w[0] ^ w[2].rotate_left(13), w[1] ^ w[3].rotate_left(29)]);
|
|
}
|
|
}
|
|
o.end();
|
|
})*};
|
|
}
|
|
macro_rules! fma3 {
|
|
($rng:ident, $($name:expr => |$a:ident, $b:ident, $c:ident| $e:expr;)*) => {$({
|
|
#[inline(never)]
|
|
#[target_feature(enable = "avx,avx2,fma")]
|
|
unsafe fn op($a: __m256d, $b: __m256d, $c: __m256d) -> __m256d { $e }
|
|
let mut o = Out::new($name);
|
|
let v = lanes64($rng);
|
|
for i in 0..v.len() {
|
|
for j in 0..v.len() {
|
|
let (mut x, mut y, mut z) = (four(&v, i, j, 5), four(&v, j, i, 11), four(&v, i + j, i * 7, 17));
|
|
one_nan(&mut x, &mut y, &mut z);
|
|
o.put4(&[x[0], y[0], z[0]], op(pd4(x), pd4(y), pd4(z)));
|
|
}
|
|
}
|
|
o.end();
|
|
})*};
|
|
}
|
|
macro_rules! fma2 {
|
|
($rng:ident, $($name:expr => |$a:ident, $b:ident, $c:ident| $e:expr;)*) => {$({
|
|
#[inline(never)]
|
|
#[target_feature(enable = "avx,avx2,fma")]
|
|
unsafe fn op($a: __m128d, $b: __m128d, $c: __m128d) -> __m128d { $e }
|
|
let mut o = Out::new($name);
|
|
let v = lanes64($rng);
|
|
for i in 0..v.len() {
|
|
for j in 0..v.len() {
|
|
let (mut x, mut y, mut z) = (four(&v, i, j, 5), four(&v, j, i, 11), four(&v, i + j, i * 7, 17));
|
|
one_nan(&mut x[..2], &mut y[..2], &mut z[..2]);
|
|
o.put(&[x[0], y[0], z[0]], bits_pd(op(pd(x[0], x[1]), pd(y[0], y[1]), pd(z[0], z[1]))));
|
|
}
|
|
}
|
|
o.end();
|
|
})*};
|
|
}
|
|
// a 128-bit pair -> 128-bit
|
|
macro_rules! bin2 {
|
|
($rng:ident, $($name:expr => |$a:ident, $b:ident| $e:expr;)*) => {$({
|
|
#[inline(never)]
|
|
#[target_feature(enable = "avx,avx2,fma")]
|
|
unsafe fn op($a: __m128d, $b: __m128d) -> __m128d { $e }
|
|
let mut o = Out::new($name);
|
|
let v = lanes64($rng);
|
|
for i in 0..v.len() {
|
|
for j in 0..v.len() {
|
|
let (x, y) = (four(&v, i, j, 5), four(&v, j, i, 11));
|
|
o.put(&[x[0], y[0]], bits_pd(op(pd(x[0], x[1]), pd(y[0], y[1]))));
|
|
}
|
|
}
|
|
o.end();
|
|
})*};
|
|
}
|
|
|
|
#[target_feature(enable = "avx,avx2,fma")]
|
|
pub unsafe fn all() {
|
|
let rng = &mut Rng(0x0123_4567_89ab_cdef);
|
|
bin4!(rng,
|
|
"_mm256_add_pd" => |a, b| _mm256_add_pd(a, b);
|
|
"_mm256_sub_pd" => |a, b| _mm256_sub_pd(a, b);
|
|
"_mm256_mul_pd" => |a, b| _mm256_mul_pd(a, b);
|
|
"_mm256_div_pd" => |a, b| _mm256_div_pd(a, b);
|
|
"_mm256_min_pd" => |a, b| _mm256_min_pd(a, b);
|
|
"_mm256_max_pd" => |a, b| _mm256_max_pd(a, b);
|
|
"_mm256_and_pd" => |a, b| _mm256_and_pd(a, b);
|
|
"_mm256_andnot_pd" => |a, b| _mm256_andnot_pd(a, b);
|
|
"_mm256_or_pd" => |a, b| _mm256_or_pd(a, b);
|
|
"_mm256_xor_pd" => |a, b| _mm256_xor_pd(a, b);
|
|
"_mm256_addsub_pd" => |a, b| _mm256_addsub_pd(a, b);
|
|
"_mm256_hadd_pd" => |a, b| _mm256_hadd_pd(a, b);
|
|
"_mm256_hsub_pd" => |a, b| _mm256_hsub_pd(a, b);
|
|
"_mm256_unpackhi_pd" => |a, b| _mm256_unpackhi_pd(a, b);
|
|
"_mm256_unpacklo_pd" => |a, b| _mm256_unpacklo_pd(a, b);
|
|
"_mm256_shuffle_pd::<0>" => |a, b| _mm256_shuffle_pd::<0>(a, b);
|
|
"_mm256_shuffle_pd::<5>" => |a, b| _mm256_shuffle_pd::<5>(a, b);
|
|
"_mm256_shuffle_pd::<10>" => |a, b| _mm256_shuffle_pd::<10>(a, b);
|
|
"_mm256_shuffle_pd::<15>" => |a, b| _mm256_shuffle_pd::<15>(a, b);
|
|
"_mm256_blend_pd::<0>" => |a, b| _mm256_blend_pd::<0>(a, b);
|
|
"_mm256_blend_pd::<6>" => |a, b| _mm256_blend_pd::<6>(a, b);
|
|
"_mm256_blend_pd::<9>" => |a, b| _mm256_blend_pd::<9>(a, b);
|
|
"_mm256_blend_pd::<15>" => |a, b| _mm256_blend_pd::<15>(a, b);
|
|
"_mm256_blendv_pd" => |a, b| _mm256_blendv_pd(a, b, _mm256_mul_pd(a, b));
|
|
"_mm256_permute2f128_pd::<0x20>" => |a, b| _mm256_permute2f128_pd::<0x20>(a, b);
|
|
"_mm256_permute2f128_pd::<0x31>" => |a, b| _mm256_permute2f128_pd::<0x31>(a, b);
|
|
"_mm256_permute2f128_pd::<0x83>" => |a, b| _mm256_permute2f128_pd::<0x83>(a, b);
|
|
"_mm256_permute2f128_pd::<0x12>" => |a, b| _mm256_permute2f128_pd::<0x12>(a, b);
|
|
"_mm256_permutevar_pd" => |a, b| _mm256_permutevar_pd(a, _mm256_castpd_si256(b));
|
|
"_mm256_cmp_pd::<0>" => |a, b| _mm256_cmp_pd::<0>(a, b);
|
|
"_mm256_cmp_pd::<1>" => |a, b| _mm256_cmp_pd::<1>(a, b);
|
|
"_mm256_cmp_pd::<2>" => |a, b| _mm256_cmp_pd::<2>(a, b);
|
|
"_mm256_cmp_pd::<3>" => |a, b| _mm256_cmp_pd::<3>(a, b);
|
|
"_mm256_cmp_pd::<4>" => |a, b| _mm256_cmp_pd::<4>(a, b);
|
|
"_mm256_cmp_pd::<5>" => |a, b| _mm256_cmp_pd::<5>(a, b);
|
|
"_mm256_cmp_pd::<6>" => |a, b| _mm256_cmp_pd::<6>(a, b);
|
|
"_mm256_cmp_pd::<7>" => |a, b| _mm256_cmp_pd::<7>(a, b);
|
|
"_mm256_cmp_pd::<8>" => |a, b| _mm256_cmp_pd::<8>(a, b);
|
|
"_mm256_cmp_pd::<9>" => |a, b| _mm256_cmp_pd::<9>(a, b);
|
|
"_mm256_cmp_pd::<10>" => |a, b| _mm256_cmp_pd::<10>(a, b);
|
|
"_mm256_cmp_pd::<11>" => |a, b| _mm256_cmp_pd::<11>(a, b);
|
|
"_mm256_cmp_pd::<12>" => |a, b| _mm256_cmp_pd::<12>(a, b);
|
|
"_mm256_cmp_pd::<13>" => |a, b| _mm256_cmp_pd::<13>(a, b);
|
|
"_mm256_cmp_pd::<14>" => |a, b| _mm256_cmp_pd::<14>(a, b);
|
|
"_mm256_cmp_pd::<15>" => |a, b| _mm256_cmp_pd::<15>(a, b);
|
|
"_mm256_cmp_pd::<16>" => |a, b| _mm256_cmp_pd::<16>(a, b);
|
|
"_mm256_cmp_pd::<24>" => |a, b| _mm256_cmp_pd::<24>(a, b);
|
|
"_mm256_cmp_pd::<28>" => |a, b| _mm256_cmp_pd::<28>(a, b);
|
|
"_mm256_cmp_pd::<31>" => |a, b| _mm256_cmp_pd::<31>(a, b);
|
|
"_mm256_insertf128_pd::<0>" => |a, b| _mm256_insertf128_pd::<0>(a, _mm256_castpd256_pd128(b));
|
|
"_mm256_insertf128_pd::<1>" => |a, b| _mm256_insertf128_pd::<1>(a, _mm256_extractf128_pd::<1>(b));
|
|
);
|
|
un4!(rng,
|
|
"_mm256_sqrt_pd" => |a| _mm256_sqrt_pd(a);
|
|
"_mm256_movedup_pd" => |a| _mm256_movedup_pd(a);
|
|
"_mm256_floor_pd" => |a| _mm256_floor_pd(a);
|
|
"_mm256_ceil_pd" => |a| _mm256_ceil_pd(a);
|
|
"_mm256_round_pd::<0>" => |a| _mm256_round_pd::<0>(a);
|
|
"_mm256_round_pd::<1>" => |a| _mm256_round_pd::<1>(a);
|
|
"_mm256_round_pd::<2>" => |a| _mm256_round_pd::<2>(a);
|
|
"_mm256_round_pd::<3>" => |a| _mm256_round_pd::<3>(a);
|
|
"_mm256_round_pd::<4>" => |a| _mm256_round_pd::<4>(a);
|
|
"_mm256_permute_pd::<0>" => |a| _mm256_permute_pd::<0>(a);
|
|
"_mm256_permute_pd::<5>" => |a| _mm256_permute_pd::<5>(a);
|
|
"_mm256_permute_pd::<9>" => |a| _mm256_permute_pd::<9>(a);
|
|
"_mm256_permute_pd::<15>" => |a| _mm256_permute_pd::<15>(a);
|
|
"_mm256_permute4x64_pd::<0x1b>" => |a| _mm256_permute4x64_pd::<0x1b>(a);
|
|
"_mm256_permute4x64_pd::<0xe4>" => |a| _mm256_permute4x64_pd::<0xe4>(a);
|
|
"_mm256_permute4x64_pd::<0x4e>" => |a| _mm256_permute4x64_pd::<0x4e>(a);
|
|
"_mm256_permute4x64_pd::<0x00>" => |a| _mm256_permute4x64_pd::<0x00>(a);
|
|
"_mm256_permute4x64_pd::<0xb1>" => |a| _mm256_permute4x64_pd::<0xb1>(a);
|
|
"_mm256_cvtps_pd" => |a| _mm256_cvtps_pd(_mm256_castps256_ps128(_mm256_castpd_ps(a)));
|
|
"_mm256_cvtepi32_pd" => |a| _mm256_cvtepi32_pd(_mm256_castsi256_si128(_mm256_castpd_si256(a)));
|
|
"_mm256_castpd128_pd256" => |a| _mm256_castpd128_pd256(_mm256_castpd256_pd128(a));
|
|
"_mm256_zextpd128_pd256" => |a| _mm256_zextpd128_pd256(_mm256_extractf128_pd::<1>(a));
|
|
"_mm256_broadcastsd_pd" => |a| _mm256_broadcastsd_pd(_mm256_extractf128_pd::<1>(a));
|
|
"_mm256_castps_pd" => |a| _mm256_castps_pd(_mm256_castpd_ps(a));
|
|
"_mm256_castsi256_pd" => |a| _mm256_castsi256_pd(_mm256_castpd_si256(a));
|
|
"_mm256_undefined_pd/setzero" => |a| _mm256_or_pd(_mm256_setzero_pd(), _mm256_and_pd(a, _mm256_undefined_pd()));
|
|
);
|
|
any4!(rng,
|
|
"_mm256_movemask_pd" => |a, b| -> i32 { _mm256_movemask_pd(_mm256_xor_pd(a, b)) };
|
|
"_mm256_testz_pd" => |a, b| -> i32 { _mm256_testz_pd(a, b) };
|
|
"_mm256_testc_pd" => |a, b| -> i32 { _mm256_testc_pd(a, b) };
|
|
"_mm256_testnzc_pd" => |a, b| -> i32 { _mm256_testnzc_pd(a, b) };
|
|
"_mm_testz_pd" => |a, b| -> i32 { _mm_testz_pd(_mm256_castpd256_pd128(a), _mm256_castpd256_pd128(b)) };
|
|
"_mm_testc_pd" => |a, b| -> i32 { _mm_testc_pd(_mm256_castpd256_pd128(a), _mm256_castpd256_pd128(b)) };
|
|
"_mm_testnzc_pd" => |a, b| -> i32 { _mm_testnzc_pd(_mm256_castpd256_pd128(a), _mm256_castpd256_pd128(b)) };
|
|
"_mm256_cvtpd_ps" => |a, b| -> __m128 { _mm256_cvtpd_ps(a) };
|
|
"_mm256_cvtpd_epi32" => |a, b| -> __m128i { _mm256_cvtpd_epi32(a) };
|
|
"_mm256_cvttpd_epi32" => |a, b| -> __m128i { _mm256_cvttpd_epi32(b) };
|
|
"_mm256_extractf128_pd::<0>" => |a, b| -> __m128d { _mm256_extractf128_pd::<0>(a) };
|
|
"_mm256_extractf128_pd::<1>" => |a, b| -> __m128d { _mm256_extractf128_pd::<1>(a) };
|
|
"_mm256_castpd_ps/si256" => |a, b| -> __m256i { _mm256_castps_si256(_mm256_castpd_ps(a)) };
|
|
);
|
|
bin2!(rng,
|
|
"_mm_permute_pd::<0>" => |a, b| _mm_permute_pd::<0>(a);
|
|
"_mm_permute_pd::<1>" => |a, b| _mm_permute_pd::<1>(a);
|
|
"_mm_permute_pd::<2>" => |a, b| _mm_permute_pd::<2>(b);
|
|
"_mm_permute_pd::<3>" => |a, b| _mm_permute_pd::<3>(b);
|
|
"_mm_permutevar_pd" => |a, b| _mm_permutevar_pd(a, _mm_castpd_si128(b));
|
|
"_mm_broadcastsd_pd" => |a, b| _mm_broadcastsd_pd(a);
|
|
"_mm_cmp_pd::<0>" => |a, b| _mm_cmp_pd::<0>(a, b);
|
|
"_mm_cmp_pd::<3>" => |a, b| _mm_cmp_pd::<3>(a, b);
|
|
"_mm_cmp_pd::<8>" => |a, b| _mm_cmp_pd::<8>(a, b);
|
|
"_mm_cmp_pd::<12>" => |a, b| _mm_cmp_pd::<12>(a, b);
|
|
"_mm_cmp_pd::<15>" => |a, b| _mm_cmp_pd::<15>(a, b);
|
|
"_mm_cmp_pd::<21>" => |a, b| _mm_cmp_pd::<21>(a, b);
|
|
"_mm_cmp_sd::<1>" => |a, b| _mm_cmp_sd::<1>(a, b);
|
|
"_mm_cmp_sd::<8>" => |a, b| _mm_cmp_sd::<8>(a, b);
|
|
"_mm_cmp_sd::<12>" => |a, b| _mm_cmp_sd::<12>(a, b);
|
|
"_mm_cmp_sd::<30>" => |a, b| _mm_cmp_sd::<30>(a, b);
|
|
);
|
|
fma2!(rng,
|
|
"_mm_fmadd_pd" => |a, b, c| _mm_fmadd_pd(a, b, c);
|
|
"_mm_fmsub_pd" => |a, b, c| _mm_fmsub_pd(a, b, c);
|
|
"_mm_fnmadd_pd" => |a, b, c| _mm_fnmadd_pd(a, b, c);
|
|
"_mm_fnmsub_pd" => |a, b, c| _mm_fnmsub_pd(a, b, c);
|
|
"_mm_fmaddsub_pd" => |a, b, c| _mm_fmaddsub_pd(a, b, c);
|
|
"_mm_fmsubadd_pd" => |a, b, c| _mm_fmsubadd_pd(a, b, c);
|
|
"_mm_fmadd_sd" => |a, b, c| _mm_fmadd_sd(a, b, c);
|
|
"_mm_fmsub_sd" => |a, b, c| _mm_fmsub_sd(a, b, c);
|
|
"_mm_fnmadd_sd" => |a, b, c| _mm_fnmadd_sd(a, b, c);
|
|
"_mm_fnmsub_sd" => |a, b, c| _mm_fnmsub_sd(a, b, c);
|
|
);
|
|
fma3!(rng,
|
|
"_mm256_fmadd_pd" => |a, b, c| _mm256_fmadd_pd(a, b, c);
|
|
"_mm256_fmsub_pd" => |a, b, c| _mm256_fmsub_pd(a, b, c);
|
|
"_mm256_fnmadd_pd" => |a, b, c| _mm256_fnmadd_pd(a, b, c);
|
|
"_mm256_fnmsub_pd" => |a, b, c| _mm256_fnmsub_pd(a, b, c);
|
|
"_mm256_fmaddsub_pd" => |a, b, c| _mm256_fmaddsub_pd(a, b, c);
|
|
"_mm256_fmsubadd_pd" => |a, b, c| _mm256_fmsubadd_pd(a, b, c);
|
|
);
|
|
memory(rng);
|
|
}
|
|
|
|
#[inline(never)]
|
|
#[target_feature(enable = "avx,avx2,fma")]
|
|
unsafe fn memory(rng: &mut Rng) {
|
|
let v = lanes64(rng);
|
|
let mut o = Out::new("256 loads/sets/broadcasts");
|
|
for w in v.windows(5) {
|
|
#[repr(C, align(32))]
|
|
struct A([f64; 5]);
|
|
let m = A(core::array::from_fn(|k| f64::from_bits(w[k])));
|
|
let p = black_box(m.0.as_ptr());
|
|
let h = pd(w[3], w[4]);
|
|
let (x, y, z, t) = (black_box(m.0[0]), black_box(m.0[1]), black_box(m.0[2]), black_box(m.0[3]));
|
|
for r in [
|
|
_mm256_load_pd(p),
|
|
_mm256_loadu_pd(p.add(1)),
|
|
_mm256_broadcast_sd(&*p.add(2)),
|
|
_mm256_broadcast_pd(&h),
|
|
_mm256_set1_pd(x),
|
|
_mm256_set_pd(x, y, z, t),
|
|
_mm256_setr_pd(x, y, z, t),
|
|
_mm256_setzero_pd(),
|
|
] {
|
|
o.put4(w, r);
|
|
}
|
|
o.put(w, bits_pd(_mm_broadcastsd_pd(h)));
|
|
}
|
|
o.end();
|
|
let mut o = Out::new("256 stores");
|
|
for w in v.windows(4) {
|
|
#[repr(C, align(32))]
|
|
struct A([u64; 6]);
|
|
let a = pd4([w[0], w[1], w[2], w[3]]);
|
|
for k in 0..3 {
|
|
let mut m = A([0x1111_1111_1111_1111; 6]);
|
|
let p = black_box(m.0.as_mut_ptr() as *mut f64);
|
|
match k {
|
|
0 => _mm256_store_pd(p, a),
|
|
1 => _mm256_storeu_pd(p.add(1), a),
|
|
_ => _mm256_stream_pd(p, a),
|
|
}
|
|
let m = black_box(m);
|
|
o.put(w, [m.0[0] ^ m.0[1].rotate_left(7) ^ m.0[2].rotate_left(19), m.0[3] ^ m.0[4].rotate_left(11) ^ m.0[5].rotate_left(23)]);
|
|
}
|
|
}
|
|
o.end();
|
|
let mut o = Out::new("masked loads/stores");
|
|
for (i, w) in v.windows(4).enumerate() {
|
|
let src = [f64::from_bits(w[0]), f64::from_bits(w[1]), f64::from_bits(w[2]), f64::from_bits(w[3])];
|
|
let r = rng.next();
|
|
let mk = |k: u32| if r >> k & 1 != 0 { -1i64 - (r as i64 & 0xff) } else { r as i64 & 0x7fff_ffff };
|
|
let m4 = si4([mk(0) as u64, mk(1) as u64, mk(2) as u64, mk(3) as u64]);
|
|
let m2 = _mm256_castsi256_si128(m4);
|
|
o.put4(w, _mm256_maskload_pd(black_box(src.as_ptr()), m4));
|
|
o.put(w, bits_pd(_mm_maskload_pd(black_box(src.as_ptr()), m2)));
|
|
let mut dst = [7.0f64; 4];
|
|
_mm256_maskstore_pd(black_box(dst.as_mut_ptr()), m4, pd4([w[3], w[2], w[1], w[0]]));
|
|
_mm_maskstore_pd(black_box(dst.as_mut_ptr().add(2)), m2, pd(w[(i + 1) % 4], w[0]));
|
|
o.put(w, [dst[0].to_bits() ^ dst[1].to_bits().rotate_left(9), dst[2].to_bits() ^ dst[3].to_bits().rotate_left(21)]);
|
|
}
|
|
o.end();
|
|
let mut o = Out::new("gathers");
|
|
let table: Vec<f64> = v.iter().map(|&x| f64::from_bits(x)).collect();
|
|
let base = black_box(table.as_ptr().add(20));
|
|
for _ in 0..64 {
|
|
let r = rng.next();
|
|
let off = |k: u32| ((r >> (8 * k)) & 31) as i64 - 16;
|
|
let o32 = _mm_setr_epi32(off(0) as i32, off(1) as i32, off(2) as i32, off(3) as i32);
|
|
let o64 = si4([off(0) as u64, off(1) as u64, off(2) as u64, off(3) as u64]);
|
|
let o64h = _mm256_castsi256_si128(o64);
|
|
let msk = pd4([r << 3, r << 2, r << 1, r]);
|
|
let src = pd4([rng.next(), 2, 3, 4]);
|
|
o.put(&[r], bits_pd(_mm_i32gather_pd::<8>(base, o32)));
|
|
o.put(&[r], bits_pd(_mm_i64gather_pd::<8>(base, o64h)));
|
|
o.put(&[r], bits_pd(_mm_mask_i32gather_pd::<8>(_mm256_castpd256_pd128(src), base, o32, _mm256_castpd256_pd128(msk))));
|
|
o.put(&[r], bits_pd(_mm_mask_i64gather_pd::<8>(_mm256_castpd256_pd128(src), base, o64h, _mm256_castpd256_pd128(msk))));
|
|
o.put4(&[r], _mm256_i32gather_pd::<8>(base, o32));
|
|
o.put4(&[r], _mm256_i64gather_pd::<8>(base, o64));
|
|
o.put4(&[r], _mm256_mask_i32gather_pd::<8>(src, base, o32, msk));
|
|
o.put4(&[r], _mm256_mask_i64gather_pd::<8>(src, base, o64, msk));
|
|
// byte offsets with scale 1 (multiples of 8) and scale 4 (even)
|
|
let ob = _mm_setr_epi32(off(0) as i32 * 8, off(1) as i32 * 8, 0, 0);
|
|
let oq = _mm_setr_epi32(off(2) as i32 * 2, off(3) as i32 * 2, 0, 0);
|
|
o.put(&[r], bits_pd(_mm_i32gather_pd::<1>(base, ob)));
|
|
o.put(&[r], bits_pd(_mm_i32gather_pd::<4>(base, oq)));
|
|
}
|
|
o.end();
|
|
}
|
|
}
|