/*
Copyright (c) 2017, Norbert Juffa
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// multiply two unsigned 64-bit integers into an unsigned 128-bit product
__device__ ulonglong2 umul64wide (uint64_t a, uint64_t b)
{
ulonglong2 res;
#if (__CUDA_ARCH__ >= 200) && (__CUDA_ARCH__ < 500)
asm ("{\n\t"
".reg .u32 r0, r1, r2, r3, alo, ahi, blo, bhi;\n\t"
"mov.b64 {alo,ahi}, %2;\n\t"
"mov.b64 {blo,bhi}, %3;\n\t"
"mul.lo.u32 r0, alo, blo;\n\t"
"mul.hi.u32 r1, alo, blo; \n\t"
"mad.lo.cc.u32 r1, alo, bhi, r1;\n\t"
"madc.hi.u32 r2, alo, bhi, 0;\n\t"
"mad.lo.cc.u32 r1, ahi, blo, r1;\n\t"
"madc.hi.cc.u32 r2, ahi, blo, r2;\n\t"
"madc.hi.u32 r3, ahi, bhi, 0;\n\t"
"mad.lo.cc.u32 r2, ahi, bhi, r2;\n\t"
"addc.u32 r3, r3, 0;\n\t"
"mov.b64 %0, {r0,r1};\n\t"
"mov.b64 %1, {r2,r3};\n\t"
"}"
: "=l"(res.x), "=l"(res.y)
: "l"(a), "l"(b));
#elif __CUDA_ARCH__ >= 500
asm ("{\n\t"
".reg .u32 alo, ahi, blo, bhi, r0, r1, r2, r3;\n\t"
".reg .u32 s0, s1, s2, s3, t0, t1, t2, t3;\n\t"
".reg .u16 a0, a1, a2, a3, b0, b1, b2, b3;\n\t"
// split inputs into 16-bit chunks
"mov.b64 {alo,ahi}, %2;\n\t"
"mov.b64 {blo,bhi}, %3;\n\t"
"mov.b32 {a0,a1}, alo;\n\t"
"mov.b32 {a2,a3}, ahi;\n\t"
"mov.b32 {b0,b1}, blo;\n\t"
"mov.b32 {b2,b3}, bhi;\n\t"
// first partial sum:
// a3b3.wide a1b3.wide a0b2.wide a0b0.wide
// 0 a2b2.wide a1b1.wide
// 0 a3b1.wide a2b0.wide
"mul.wide.u16 r0, a0, b0;\n\t"
"mul.wide.u16 r1, a0, b2;\n\t"
"mul.wide.u16 r2, a1, b3;\n\t"
"mul.wide.u16 r3, a3, b3;\n\t"
"mul.wide.u16 t1, a1, b1;\n\t"
"mul.wide.u16 t2, a2, b2;\n\t"
"add.cc.u32 r1, r1, t1;\n\t"
"addc.cc.u32 r2, r2, t2;\n\t"
"addc.u32 r3, r3, 0;\n\t"
"mul.wide.u16 t1, a2, b0;\n\t"
"mul.wide.u16 t2, a3, b1;\n\t"
"add.cc.u32 r1, r1, t1;\n\t"
"addc.cc.u32 r2, r2, t2;\n\t"
"addc.u32 r3, r3, 0;\n\t"
// second partial sum:
// a2b3.wide a0b3.wide a0b1.wide
// a3b2.wide a1b2.wide a1b0.wide
// 0 a2b1.wide
// 0 a3b0.wide
"mul.wide.u16 s0, a0, b1;\n\t"
"mul.wide.u16 s1, a0, b3;\n\t"
"mul.wide.u16 s2, a2, b3;\n\t"
"mul.wide.u16 t1, a2, b1;\n\t"
"add.cc.u32 s1, s1, t1;\n\t"
"addc.u32 s2, s2, 0;\n\t"
"mul.wide.u16 t1, a3, b0;\n\t"
"add.cc.u32 s1, s1, t1;\n\t"
"addc.u32 s2, s2, 0;\n\t"
"mul.wide.u16 t0, a1, b0;\n\t"
"mul.wide.u16 t1, a1, b2;\n\t"
"mul.wide.u16 t2, a3, b2;\n\t"
"add.cc.u32 s0, s0, t0;\n\t"
"addc.cc.u32 s1, s1, t1;\n\t"
"addc.cc.u32 s2, s2, t2;\n\t"
"addc.u32 s3, 0, 0;\n\t"
// offset second partial sum by 16 bits to the left
"shf.l.clamp.b32 t3, s2, s3, 16;\n\t"
"shf.l.clamp.b32 t2, s1, s2, 16;\n\t"
"shf.l.clamp.b32 t1, s0, s1, 16;\n\t"
"shf.l.clamp.b32 t0, 0, s0, 16;\n\t"
// add first sum in r{0,1,2,3} to second sum in t{0,1,2,3}
"add.cc.u32 r0, r0, t0;\n\t"
"addc.cc.u32 r1, r1, t1;\n\t"
"addc.cc.u32 r2, r2, t2;\n\t"
"addc.u32 r3, r3, t3;\n\t"
// pack outputs
"mov.b64 %0, {r0,r1};\n\t"
"mov.b64 %1, {r2,r3};\n\t"
"}"
: "=l"(res.x), "=l"(res.y)
: "l"(a), "l"(b));
#elif __CUDA_ARCH__
#error unsupported __CUDA_ARCH__
#else // avoid warning
res.x = 0;
res.y = 0;
#endif
return res;
}
// multiply two unsigned 128-bit integers into an unsigned 256-bit product
__device__ ulonglong4 umul128wide (ulonglong2 a, ulonglong2 b)
{
ulonglong4 res;
#if (__CUDA_ARCH__ >= 200) && (__CUDA_ARCH__ < 500)
asm ("{\n\t"
".reg .u32 r0, r1, r2, r3, r4, r5, r6, r7;\n\t"
".reg .u32 a0, a1, a2, a3, b0, b1, b2, b3;\n\t"
"mov.b64 {a0,a1}, %4;\n\t"
"mov.b64 {a2,a3}, %5;\n\t"
"mov.b64 {b0,b1}, %6;\n\t"
"mov.b64 {b2,b3}, %7;\n\t"
"mul.lo.u32 r0, a0, b0;\n\t"
"mul.hi.u32 r1, a0, b0;\n\t"
"mad.lo.cc.u32 r1, a0, b1, r1;\n\t"
"madc.hi.u32 r2, a0, b1, 0;\n\t"
"mad.lo.cc.u32 r1, a1, b0, r1;\n\t"
"madc.hi.cc.u32 r2, a1, b0, r2;\n\t"
"madc.hi.u32 r3, a0, b2, 0;\n\t"
"mad.lo.cc.u32 r2, a0, b2, r2;\n\t"
"madc.hi.cc.u32 r3, a1, b1, r3;\n\t"
"madc.hi.u32 r4, a0, b3, 0;\n\t"
"mad.lo.cc.u32 r2, a1, b1, r2;\n\t"
"madc.hi.cc.u32 r3, a2, b0, r3;\n\t"
"madc.hi.cc.u32 r4, a1, b2, r4;\n\t"
"madc.hi.u32 r5, a1, b3, 0;\n\t"
"mad.lo.cc.u32 r2, a2, b0, r2;\n\t"
"madc.lo.cc.u32 r3, a0, b3, r3;\n\t"
"madc.hi.cc.u32 r4, a2, b1, r4;\n\t"
"madc.hi.cc.u32 r5, a2, b2, r5;\n\t"
"madc.hi.u32 r6, a2, b3, 0;\n\t"
"mad.lo.cc.u32 r3, a1, b2, r3;\n\t"
"madc.hi.cc.u32 r4, a3, b0, r4;\n\t"
"madc.hi.cc.u32 r5, a3, b1, r5;\n\t"
"madc.hi.cc.u32 r6, a3, b2, r6;\n\t"
"madc.hi.u32 r7, a3, b3, 0;\n\t"
"mad.lo.cc.u32 r3, a2, b1, r3;\n\t"
"madc.lo.cc.u32 r4, a1, b3, r4;\n\t"
"madc.lo.cc.u32 r5, a2, b3, r5;\n\t"
"madc.lo.cc.u32 r6, a3, b3, r6;\n\t"
"addc.u32 r7, r7, 0;\n\t"
"mad.lo.cc.u32 r3, a3, b0, r3;\n\t"
"madc.lo.cc.u32 r4, a2, b2, r4;\n\t"
"madc.lo.cc.u32 r5, a3, b2, r5;\n\t"
"addc.cc.u32 r6, r6, 0;\n\t"
"addc.u32 r7, r7, 0;\n\t"
"mad.lo.cc.u32 r4, a3, b1, r4;\n\t"
"addc.cc.u32 r5, r5, 0;\n\t"
"addc.cc.u32 r6, r6, 0;\n\t"
"addc.u32 r7, r7, 0;\n\t"
"mov.b64 %0, {r0,r1};\n\t"
"mov.b64 %1, {r2,r3};\n\t"
"mov.b64 %2, {r4,r5};\n\t"
"mov.b64 %3, {r6,r7};\n\t"
"}"
: "=l"(res.x), "=l"(res.y), "=l"(res.z), "=l"(res.w)
: "l"(a.x), "l"(a.y), "l"(b.x), "l"(b.y));
#elif __CUDA_ARCH__ >= 500
asm ("{\n\t"
".reg .u32 aa0, aa1, aa2, aa3, bb0, bb1, bb2, bb3;\n\t"
".reg .u32 r0, r1, r2, r3, r4, r5, r6, r7;\n\t"
".reg .u32 s0, s1, s2, s3, s4, s5, s6, s7;\n\t"
".reg .u32 t0, t1, t2, t3, t4, t5, t6, t7;\n\t"
".reg .u16 a0, a1, a2, a3, a4, a5, a6, a7;\n\t"
".reg .u16 b0, b1, b2, b3, b4, b5, b6, b7;\n\t"
// unpack source operands
"mov.b64 {aa0,aa1}, %4;\n\t"
"mov.b64 {aa2,aa3}, %5;\n\t"
"mov.b64 {bb0,bb1}, %6;\n\t"
"mov.b64 {bb2,bb3}, %7;\n\t"
"mov.b32 {a0,a1}, aa0;\n\t"
"mov.b32 {a2,a3}, aa1;\n\t"
"mov.b32 {a4,a5}, aa2;\n\t"
"mov.b32 {a6,a7}, aa3;\n\t"
"mov.b32 {b0,b1}, bb0;\n\t"
"mov.b32 {b2,b3}, bb1;\n\t"
"mov.b32 {b4,b5}, bb2;\n\t"
"mov.b32 {b6,b7}, bb3;\n\t"
// compute first partial sum
"mul.wide.u16 r0, a0, b0;\n\t"
"mul.wide.u16 r1, a0, b2;\n\t"
"mul.wide.u16 r2, a0, b4;\n\t"
"mul.wide.u16 r3, a0, b6;\n\t"
"mul.wide.u16 r4, a1, b7;\n\t"
"mul.wide.u16 r5, a3, b7;\n\t"
"mul.wide.u16 r6, a5, b7;\n\t"
"mul.wide.u16 r7, a7, b7;\n\t"
"mul.wide.u16 t3, a1, b5;\n\t"
"mul.wide.u16 t4, a2, b6;\n\t"
"add.cc.u32 r3, r3, t3;\n\t"
"addc.cc.u32 r4, r4, t4;\n\t"
"addc.u32 r5, r5, 0;\n\t"
"mul.wide.u16 t3, a2, b4;\n\t"
"mul.wide.u16 t4, a3, b5;\n\t"
"add.cc.u32 r3, r3, t3;\n\t"
"addc.cc.u32 r4, r4, t4;\n\t"
"addc.u32 r5, r5, 0;\n\t"
"mul.wide.u16 t2, a1, b3;\n\t"
"mul.wide.u16 t3, a3, b3;\n\t"
"mul.wide.u16 t4, a4, b4;\n\t"
"mul.wide.u16 t5, a4, b6;\n\t"
"add.cc.u32 r2, r2, t2;\n\t"
"addc.cc.u32 r3, r3, t3;\n\t"
"addc.cc.u32 r4, r4, t4;\n\t"
"addc.cc.u32 r5, r5, t5;\n\t"
"addc.u32 r6, r6, 0;\n\t"
"mul.wide.u16 t2, a2, b2;\n\t"
"mul.wide.u16 t3, a4, b2;\n\t"
"mul.wide.u16 t4, a5, b3;\n\t"
"mul.wide.u16 t5, a5, b5;\n\t"
"add.cc.u32 r2, r2, t2;\n\t"
"addc.cc.u32 r3, r3, t3;\n\t"
"addc.cc.u32 r4, r4, t4;\n\t"
"addc.cc.u32 r5, r5, t5;\n\t"
"addc.u32 r6, r6, 0;\n\t"
"mul.wide.u16 t1, a1, b1;\n\t"
"mul.wide.u16 t2, a3, b1;\n\t"
"mul.wide.u16 t3, a5, b1;\n\t"
"mul.wide.u16 t4, a6, b2;\n\t"
"mul.wide.u16 t5, a6, b4;\n\t"
"mul.wide.u16 t6, a6, b6;\n\t"
"add.cc.u32 r1, r1, t1;\n\t"
"addc.cc.u32 r2, r2, t2;\n\t"
"addc.cc.u32 r3, r3, t3;\n\t"
"addc.cc.u32 r4, r4, t4;\n\t"
"addc.cc.u32 r5, r5, t5;\n\t"
"addc.cc.u32 r6, r6, t6;\n\t"
"addc.u32 r7, r7, 0;\n\t"
"mul.wide.u16 t1, a2, b0;\n\t"
"mul.wide.u16 t2, a4, b0;\n\t"
"mul.wide.u16 t3, a6, b0;\n\t"
"mul.wide.u16 t4, a7, b1;\n\t"
"mul.wide.u16 t5, a7, b3;\n\t"
"mul.wide.u16 t6, a7, b5;\n\t"
"add.cc.u32 r1, r1, t1;\n\t"
"addc.cc.u32 r2, r2, t2;\n\t"
"addc.cc.u32 r3, r3, t3;\n\t"
"addc.cc.u32 r4, r4, t4;\n\t"
"addc.cc.u32 r5, r5, t5;\n\t"
"addc.cc.u32 r6, r6, t6;\n\t"
"addc.u32 r7, r7, 0;\n\t"
// compute second partial sum
"mul.wide.u16 t0, a0, b1;\n\t"
"mul.wide.u16 t1, a0, b3;\n\t"
"mul.wide.u16 t2, a0, b5;\n\t"
"mul.wide.u16 t3, a0, b7;\n\t"
"mul.wide.u16 t4, a2, b7;\n\t"
"mul.wide.u16 t5, a4, b7;\n\t"
"mul.wide.u16 t6, a6, b7;\n\t"
"mul.wide.u16 s3, a1, b6;\n\t"
"add.cc.u32 t3, t3, s3;\n\t"
"addc.u32 t4, t4, 0;\n\t"
"mul.wide.u16 s3, a2, b5;\n\t"
"add.cc.u32 t3, t3, s3;\n\t"
"addc.u32 t4, t4, 0;\n\t"
"mul.wide.u16 s2, a1, b4;\n\t"
"mul.wide.u16 s3, a3, b4;\n\t"
"mul.wide.u16 s4, a3, b6;\n\t"
"add.cc.u32 t2, t2, s2;\n\t"
"addc.cc.u32 t3, t3, s3;\n\t"
"addc.cc.u32 t4, t4, s4;\n\t"
"addc.u32 t5, t5, 0;\n\t"
"mul.wide.u16 s2, a2, b3;\n\t"
"mul.wide.u16 s3, a4, b3;\n\t"
"mul.wide.u16 s4, a4, b5;\n\t"
"add.cc.u32 t2, t2, s2;\n\t"
"addc.cc.u32 t3, t3, s3;\n\t"
"addc.cc.u32 t4, t4, s4;\n\t"
"addc.u32 t5, t5, 0;\n\t"
"mul.wide.u16 s1, a1, b2;\n\t"
"mul.wide.u16 s2, a3, b2;\n\t"
"mul.wide.u16 s3, a5, b2;\n\t"
"mul.wide.u16 s4, a5, b4;\n\t"
"mul.wide.u16 s5, a5, b6;\n\t"
"add.cc.u32 t1, t1, s1;\n\t"
"addc.cc.u32 t2, t2, s2;\n\t"
"addc.cc.u32 t3, t3, s3;\n\t"
"addc.cc.u32 t4, t4, s4;\n\t"
"addc.cc.u32 t5, t5, s5;\n\t"
"addc.u32 t6, t6, 0;\n\t"
"mul.wide.u16 s1, a2, b1;\n\t"
"mul.wide.u16 s2, a4, b1;\n\t"
"mul.wide.u16 s3, a6, b1;\n\t"
"mul.wide.u16 s4, a6, b3;\n\t"
"mul.wide.u16 s5, a6, b5;\n\t"
"add.cc.u32 t1, t1, s1;\n\t"
"addc.cc.u32 t2, t2, s2;\n\t"
"addc.cc.u32 t3, t3, s3;\n\t"
"addc.cc.u32 t4, t4, s4;\n\t"
"addc.cc.u32 t5, t5, s5;\n\t"
"addc.u32 t6, t6, 0;\n\t"
"mul.wide.u16 s0, a1, b0;\n\t"
"mul.wide.u16 s1, a3, b0;\n\t"
"mul.wide.u16 s2, a5, b0;\n\t"
"mul.wide.u16 s3, a7, b0;\n\t"
"mul.wide.u16 s4, a7, b2;\n\t"
"mul.wide.u16 s5, a7, b4;\n\t"
"mul.wide.u16 s6, a7, b6;\n\t"
"add.cc.u32 t0, t0, s0;\n\t"
"addc.cc.u32 t1, t1, s1;\n\t"
"addc.cc.u32 t2, t2, s2;\n\t"
"addc.cc.u32 t3, t3, s3;\n\t"
"addc.cc.u32 t4, t4, s4;\n\t"
"addc.cc.u32 t5, t5, s5;\n\t"
"addc.cc.u32 t6, t6, s6;\n\t"
"addc.u32 t7, 0, 0;\n\t"
// offset second partial sum by 16 bits
"shf.l.clamp.b32 s7, t6, t7, 16;\n\t"
"shf.l.clamp.b32 s6, t5, t6, 16;\n\t"
"shf.l.clamp.b32 s5, t4, t5, 16;\n\t"
"shf.l.clamp.b32 s4, t3, t4, 16;\n\t"
"shf.l.clamp.b32 s3, t2, t3, 16;\n\t"
"shf.l.clamp.b32 s2, t1, t2, 16;\n\t"
"shf.l.clamp.b32 s1, t0, t1, 16;\n\t"
"shf.l.clamp.b32 s0, 0, t0, 16;\n\t"
// add partial sums
"add.cc.u32 r0, r0, s0;\n\t"
"addc.cc.u32 r1, r1, s1;\n\t"
"addc.cc.u32 r2, r2, s2;\n\t"
"addc.cc.u32 r3, r3, s3;\n\t"
"addc.cc.u32 r4, r4, s4;\n\t"
"addc.cc.u32 r5, r5, s5;\n\t"
"addc.cc.u32 r6, r6, s6;\n\t"
"addc.u32 r7, r7, s7;\n\t"
// pack up result
"mov.b64 %0, {r0,r1};\n\t"
"mov.b64 %1, {r2,r3};\n\t"
"mov.b64 %2, {r4,r5};\n\t"
"mov.b64 %3, {r6,r7};\n\t"
"}"
: "=l"(res.x), "=l"(res.y), "=l"(res.z), "=l"(res.w)
: "l"(a.x), "l"(a.y), "l"(b.x), "l"(b.y));
#elif __CUDA_ARCH__
#error unsupported __CUDA_ARCH__
#else // avoid warning
res.x = 0;
res.y = 0;
res.z = 0;
res.w = 0;
#endif
return res;
}
