Merge with Linux 2.3.41.

1 files changed, 914 insertions, 0 deletions

diff --git a/arch/sparc64/lib/VIScsumcopyusr.S b/arch/sparc64/lib/VIScsumcopyusr.S
new file mode 100644
index 000000000..17bbe78b1
--- /dev/null
+++ b/arch/sparc64/lib/VIScsumcopyusr.S
@@ -0,0 +1,914 @@
+/* $Id: VIScsumcopyusr.S,v 1.1 2000/01/19 04:06:04 davem Exp $
+ * VIScsumcopyusr.S: High bandwidth IP checksumming with simultaneous
+ *                   copying utilizing the UltraSparc Visual Instruction Set.
+ *
+ * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ * Copyright (C) 2000 David S. Miller (davem@redhat.com)
+ *
+ * Based on older sparc32/sparc64 checksum.S, which is:
+ *
+ *      Copyright(C) 1995 Linus Torvalds
+ *      Copyright(C) 1995 Miguel de Icaza
+ *      Copyright(C) 1996,1997 David S. Miller
+ *    derived from:
+ *	  Linux/Alpha checksum c-code
+ *        Linux/ix86 inline checksum assembly
+ *        RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code)
+ *	  David Mosberger-Tang for optimized reference c-code
+ *	  BSD4.4 portable checksum routine
+ */
+
+#ifdef __sparc_v9__
+#define STACKOFF	0x7ff+128
+#else
+#define STACKOFF	64
+#endif
+
+#ifdef __KERNEL__
+#include <asm/head.h>
+#include <asm/asi.h>
+#include <asm/page.h>
+#include <asm/visasm.h>
+#define ASI_BLK_XOR	0
+#define ASI_BLK_XOR1	(ASI_BLK_P ^ (ASI_BLK_P >> 3) ^ ASI_P)
+#define ASI_BLK_OR	(ASI_BLK_P & ~ASI_P)
+#else
+#define ASI_P		0x80
+#define ASI_BLK_P	0xf0
+#define FRPS_FEF	0x04
+#define FPRS_DU		0x02
+#define FPRS_DL		0x01
+#define ASI_BLK_XOR	(ASI_BLK_P ^ ASI_P)
+#endif
+
+#define src		o0
+#define dst		o1
+#define	len		o2
+#define sum		o3
+#define x1		g1
+#define x2		g2
+#define x3		o4
+#define x4		g4
+#define x5		g5
+#define x6		g7
+#define x7		g3
+#define x8		o5
+
+/* Dobrou noc, SunSoft engineers. Spete sladce.
+ * This has a couple of tricks in and those
+ * tricks are UltraLinux trade secrets :))
+ * Once AGAIN, the SunSoft engineers are caught
+ * asleep at the keyboard :)).
+ * The main loop does about 20 superscalar cycles
+ * per 64bytes checksummed/copied.
+ */
+
+#define LDBLK(O0)										\
+	ldda		[%src] ASI_BLK_P, %O0 	/*  Load	Group			*/
+
+#define STBLK											\
+	stda		%f48, [%dst] %asi	/*  Store				*/
+
+#ifdef __KERNEL__
+#define STBLK_XORASI(tmpreg1,tmpreg2)								\
+	stda		%f48, [%dst] %asi	/*  Store				*/;	\
+	rd		%asi, %tmpreg1;								\
+	srl		%tmpreg1, 3, %tmpreg2;							\
+	xor		%tmpreg1, ASI_BLK_XOR1, %tmpreg1;					\
+	wr		%tmpreg1, %tmpreg2, %asi;
+#else
+#define STBLK_XORASI(tmpreg1,tmpreg2)								\
+	stda		%f48, [%dst] %asi	/*  Store				*/;	\
+	rd		%asi, %tmpreg1;								\
+	wr		%tmpreg1, ASI_BLK_XOR, %asi;
+#endif
+
+#define ST(fx,off)										\
+	stda		%fx, [%dst + off] %asi	/*  Store				*/
+
+#define SYNC											\
+	membar		#Sync
+
+
+#define DO_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,F0,F2,F4,F6,F8,F10,F12,F14,DUMMY1,A0,A2,A4,A6,A8,A10,A12,A14,B14,DUMMY2,LOAD,STORE1,STORE2,STORE3,STORE4,STORE5,STORE6,STORE7,STORE8,DUMMY3,BRANCH...)	\
+	LOAD					/*  Load	Group			*/;	\
+	faligndata	%A14, %F0, %A14		/*  FPA		Group			*/;	\
+	inc		%x5			/*  IEU0				*/;	\
+	STORE1					/*  Store (optional)			*/;	\
+	faligndata	%F0, %F2, %A0		/*  FPA		Group			*/;	\
+	srl		%x5, 1, %x5		/*  IEU0				*/;	\
+	add		%sum, %x4, %sum		/*  IEU1				*/;	\
+	fpadd32		%F0, %f0, %F0		/*  FPA		Group			*/;	\
+	inc		%x6			/*  IEU0				*/;	\
+	STORE2					/*  Store (optional)			*/;	\
+	faligndata	%F2, %F4, %A2		/*  FPA		Group			*/;	\
+	srl		%x6, 1, %x6		/*  IEU0				*/;	\
+	add		%sum, %x5, %sum		/*  IEU1				*/;	\
+	fpadd32		%F2, %f2, %F2		/*  FPA		Group			*/;	\
+	add		%src, 64, %src		/*  IEU0				*/;	\
+	add		%dst, 64, %dst		/*  IEU1				*/;	\
+	fcmpgt32	%f0, %F0, %x1		/*  FPM		Group			*/;	\
+	inc		%x7			/*  IEU0				*/;	\
+	STORE3					/*  Store (optional)			*/;	\
+	faligndata	%F4, %F6, %A4		/*  FPA					*/;	\
+	srl		%x7, 1, %x7		/*  IEU0	Group			*/;	\
+	add		%sum, %x6, %sum		/*  IEU1				*/;	\
+	fpadd32		%F4, %f4, %F4		/*  FPA					*/;	\
+	fcmpgt32	%f2, %F2, %x2		/*  FPM		Group			*/;	\
+	inc		%x8			/*  IEU0				*/;	\
+	STORE4					/*  Store (optional)			*/;	\
+	faligndata	%F6, %F8, %A6		/*  FPA					*/;	\
+	srl		%x8, 1, %x8		/*  IEU0	Group			*/;	\
+	add		%sum, %x7, %sum		/*  IEU1				*/;	\
+	fpadd32		%F6, %f6, %F6		/*  FPA					*/;	\
+	fcmpgt32	%f4, %F4, %x3		/*  FPM		Group			*/;	\
+	inc		%x1			/*  IEU0				*/;	\
+	STORE5					/*  Store (optional)			*/;	\
+	faligndata	%F8, %F10, %A8		/*  FPA					*/;	\
+	srl		%x1, 1, %x1		/*  IEU0	Group			*/;	\
+	add		%sum, %x8, %sum		/*  IEU1				*/;	\
+	fpadd32		%F8, %f8, %F8		/*  FPA					*/;	\
+	fcmpgt32	%f6, %F6, %x4		/*  FPM		Group			*/;	\
+	inc		%x2			/*  IEU0				*/;	\
+	STORE6					/*  Store (optional)			*/;	\
+	faligndata	%F10, %F12, %A10	/*  FPA					*/;	\
+	srl		%x2, 1, %x2		/*  IEU0	Group			*/;	\
+	add		%sum, %x1, %sum		/*  IEU1				*/;	\
+	fpadd32		%F10, %f10, %F10	/*  FPA					*/;	\
+	fcmpgt32	%f8, %F8, %x5		/*  FPM		Group			*/;	\
+	inc		%x3			/*  IEU0				*/;	\
+	STORE7					/*  Store (optional)			*/;	\
+	faligndata	%F12, %F14, %A12	/*  FPA					*/;	\
+	srl		%x3, 1, %x3		/*  IEU0	Group			*/;	\
+	add		%sum, %x2, %sum		/*  IEU1				*/;	\
+	fpadd32		%F12, %f12, %F12	/*  FPA					*/;	\
+	fcmpgt32	%f10, %F10, %x6		/*  FPM		Group			*/;	\
+	inc		%x4			/*  IEU0				*/;	\
+	STORE8					/*  Store (optional)			*/;	\
+	fmovd		%F14, %B14		/*  FPA					*/;	\
+	srl		%x4, 1, %x4		/*  IEU0	Group			*/;	\
+	add		%sum, %x3, %sum		/*  IEU1				*/;	\
+	fpadd32		%F14, %f14, %F14	/*  FPA					*/;	\
+	fcmpgt32	%f12, %F12, %x7		/*  FPM		Group			*/;	\
+	subcc		%len, 64, %len		/*  IEU1				*/;	\
+	BRANCH					/*  CTI					*/;	\
+	fcmpgt32	%f14, %F14, %x8		/*  FPM		Group			*/;	\
+
+#define END_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB,S0,S1,S2,S3,T0,T1,U0,fz) \
+	inc		%x5			/*  IEU0	Group			*/;	\
+	fpadd32		%f2, %f0, %S0		/*  FPA					*/;	\
+	srl		%x5, 1, %x5		/*  IEU0	Group			*/;	\
+	add		%sum, %x4, %sum		/*  IEU1				*/;	\
+	fpadd32		%f6, %f4, %S1		/*  FPA					*/;	\
+	inc		%x6			/*  IEU0	Group			*/;	\
+	add		%sum, %x5, %sum		/*  IEU1				*/;	\
+	fcmpgt32	%f0, %S0, %x1		/*  FPM		Group			*/;	\
+	srl		%x6, 1, %x6		/*  IEU0				*/;	\
+	inc		%x7			/*  IEU1				*/;	\
+	fpadd32		%f10, %f8, %S2		/*  FPA					*/;	\
+	fcmpgt32	%f4, %S1, %x2		/*  FPM		Group			*/;	\
+	srl		%x7, 1, %x7		/*  IEU0				*/;	\
+	add		%sum, %x6, %sum		/*  IEU1				*/;	\
+	fpadd32		%f14, %f12, %S3		/*  FPA					*/;	\
+	inc		%x8			/*  IEU0	Group			*/;	\
+	add		%sum, %x7, %sum		/*  IEU1				*/;	\
+	fzero		%fz			/*  FPA					*/;	\
+	fcmpgt32	%f8, %S2, %x3		/*  FPM		Group			*/;	\
+	srl		%x8, 1, %x8		/*  IEU0				*/;	\
+	inc		%x1			/*  IEU1				*/;	\
+	fpadd32		%S0, %S1, %T0		/*  FPA					*/;	\
+	fcmpgt32	%f12, %S3, %x4		/*  FPM		Group			*/;	\
+	srl		%x1, 1, %x1		/*  IEU0				*/;	\
+	add		%sum, %x8, %sum		/*  IEU1				*/;	\
+	fpadd32		%S2, %S3, %T1		/*  FPA					*/;	\
+	inc		%x2			/*  IEU0	Group			*/;	\
+	add		%sum, %x1, %sum		/*  IEU1				*/;	\
+	fcmpgt32	%S0, %T0, %x5		/*  FPM		Group			*/;	\
+	srl		%x2, 1, %x2		/*  IEU0				*/;	\
+	inc		%x3			/*  IEU1				*/;	\
+	fcmpgt32	%S2, %T1, %x6		/*  FPM		Group			*/;	\
+	srl		%x3, 1, %x3		/*  IEU0				*/;	\
+	add		%sum, %x2, %sum		/*  IEU1				*/;	\
+	inc		%x4			/*  IEU0	Group			*/;	\
+	add		%sum, %x3, %sum		/*  IEU1				*/;	\
+	fcmpgt32	%fz, %f2, %x7		/*  FPM		Group			*/;	\
+	srl		%x4, 1, %x4		/*  IEU0				*/;	\
+	inc		%x5			/*  IEU1				*/;	\
+	fpadd32		%T0, %T1, %U0		/*  FPA					*/;	\
+	fcmpgt32	%fz, %f6, %x8		/*  FPM		Group			*/;	\
+	srl		%x5, 1, %x5		/*  IEU0				*/;	\
+	add		%sum, %x4, %sum		/*  IEU1				*/;	\
+	inc		%x6			/*  IEU0	Group			*/;	\
+	add		%sum, %x5, %sum		/*  IEU1				*/;	\
+	fcmpgt32	%fz, %f10, %x1		/*  FPM		Group			*/;	\
+	srl		%x6, 1, %x6		/*  IEU0				*/;	\
+	inc		%x7			/*  IEU1				*/;	\
+	fcmpgt32	%fz, %f14, %x2		/*  FPM		Group			*/;	\
+	ba,pt		%xcc, ett		/*  CTI					*/;	\
+	 fmovd		%FA, %FB		/*  FPA					*/;	\
+
+#define END_THE_TRICK1(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB) 					\
+	END_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB,f48,f50,f52,f54,f56,f58,f60,f62)
+
+#define END_THE_TRICK2(S0,S1,S2,S3,T0,T1,U0,U1,V0,fz) 						\
+	fpadd32		%U0, %U1, %V0		/*  FPA		Group			*/;	\
+	srl		%x7, 1, %x7		/*  IEU0				*/;	\
+	add		%sum, %x6, %sum		/*  IEU1				*/;	\
+	std		%V0, [%sp + STACKOFF]	/*  Store	Group			*/;	\
+	inc		%x8			/*  IEU0				*/;	\
+	sub		%sum, %x7, %sum		/*  IEU1				*/;	\
+	fcmpgt32	%fz, %S1, %x3		/*  FPM		Group			*/;	\
+	srl		%x8, 1, %x8		/*  IEU0				*/;	\
+	inc		%x1			/*  IEU1				*/;	\
+	fcmpgt32	%fz, %S3, %x4		/*  FPM		Group			*/;	\
+	srl		%x1, 1, %x1		/*  IEU0				*/;	\
+	sub		%sum, %x8, %sum		/*  IEU1				*/;	\
+	ldx		[%sp + STACKOFF], %x8	/*  Load	Group			*/;	\
+	inc		%x2			/*  IEU0				*/;	\
+	sub		%sum, %x1, %sum		/*  IEU1				*/;	\
+	fcmpgt32	%fz, %T1, %x5		/*  FPM		Group			*/;	\
+	srl		%x2, 1, %x2		/*  IEU0				*/;	\
+	inc		%x3			/*  IEU1				*/;	\
+	fcmpgt32	%T0, %U0, %x6		/*  FPM		Group			*/;	\
+	srl		%x3, 1, %x3		/*  IEU0				*/;	\
+	sub		%sum, %x2, %sum		/*  IEU1				*/;	\
+	inc		%x4			/*  IEU0	Group			*/;	\
+	sub		%sum, %x3, %sum		/*  IEU1				*/;	\
+	fcmpgt32	%fz, %U1, %x7		/*  FPM		Group			*/;	\
+	srl		%x4, 1, %x4		/*  IEU0				*/;	\
+	inc		%x5			/*  IEU1				*/;	\
+	fcmpgt32	%U0, %V0, %x1		/*  FPM		Group			*/;	\
+	srl		%x5, 1, %x5		/*  IEU0				*/;	\
+	sub		%sum, %x4, %sum		/*  IEU1				*/;	\
+	fcmpgt32	%fz, %V0, %x2		/*  FPM		Group			*/;	\
+	inc		%x6			/*  IEU0				*/;	\
+	sub		%sum, %x5, %sum		/*  IEU1				*/;	\
+	srl		%x6, 1, %x6		/*  IEU0	Group			*/;	\
+	inc		%x7			/*  IEU1				*/;	\
+	srl		%x7, 1, %x7		/*  IEU0	Group			*/;	\
+	add		%sum, %x6, %sum		/*  IEU1				*/;	\
+	inc		%x1			/*  IEU0	Group			*/;	\
+	sub		%sum, %x7, %sum		/*  IEU1				*/;	\
+	srl		%x1, 1, %x1		/*  IEU0	Group			*/;	\
+	inc		%x2			/*  IEU1				*/;	\
+	srl		%x2, 1, %x2		/*  IEU0	Group			*/;	\
+	add		%sum, %x1, %sum		/*  IEU1				*/;	\
+	sub		%sum, %x2, %sum		/*  IEU0	Group			*/;	\
+	addcc		%sum, %x8, %sum		/*  IEU		Group			*/;	\
+	bcs,a,pn	%xcc, 33f		/*  CTI					*/;	\
+	 add		%sum, 1, %sum		/*  IEU0				*/;	\
+33:						/*  That's it				*/;
+
+	.text
+	.globl		csum_partial_copy_user_vis
+	.align		32
+/* %asi should be either ASI_P or ASI_AIUS for csum_partial_copy resp. csum_partial_copy_from_user */
+/* This assumes that !((%src^%dst)&3) && !((%src|%dst)&1) && %len >= 256 */
+csum_partial_copy_user_vis:
+	andcc		%dst, 7, %g0		/*  IEU1	Group			*/
+	be,pt		%icc, 4f		/*  CTI					*/
+	 and		%dst, 0x38, %o4		/*  IEU0				*/
+	mov		1, %g5			/*  IEU0	Group			*/
+	andcc		%dst, 2, %g0		/*  IEU1				*/
+	be,pt		%icc, 1f		/*  CTI					*/
+	 and		%dst, 4, %g7		/*  IEU0	Group			*/
+	lduh		[%src], %g2		/*  Load				*/
+	sub		%len, 2, %len		/*  IEU0	Group			*/
+	add		%dst, 2, %dst		/*  IEU1				*/
+	andcc		%dst, 4, %g7		/*  IEU1	Group			*/
+	sll		%g5, 16, %g5		/*  IEU0				*/
+	stha		%g2, [%dst - 2]	%asi	/*  Store	Group			*/
+	sll		%g2, 16, %g2		/*  IEU0	 			*/
+	add		%src, 2, %src		/*  IEU1				*/
+	addcc		%g2, %sum, %sum		/*  IEU1	Group			*/
+	bcs,a,pn	%icc, 1f		/*  CTI					*/
+	 add		%sum, %g5, %sum		/*  IEU0				*/
+1:	lduw		[%src], %g2		/*  Load				*/
+	brz,a,pn	%g7, 4f			/*  CTI+IEU1	Group			*/
+	 and		%dst, 0x38, %o4		/*  IEU0				*/
+	add		%dst, 4, %dst		/*  IEU0	Group			*/
+	sub		%len, 4, %len		/*  IEU1				*/
+	addcc		%g2, %sum, %sum		/*  IEU1	Group			*/
+	bcs,a,pn	%icc, 1f		/*  CTI					*/
+	 add		%sum, 1, %sum		/*  IEU0				*/
+1:	and		%dst, 0x38, %o4		/*  IEU0	Group			*/
+	stwa		%g2, [%dst - 4] %asi	/*  Store				*/
+	add		%src, 4, %src		/*  IEU1				*/
+4:
+#ifdef __KERNEL__
+	VISEntry
+#endif
+	mov		%src, %g7		/*  IEU1	Group			*/
+	fzero		%f48			/*  FPA					*/
+	alignaddr	%src, %g0, %src		/*  Single	Group			*/
+	subcc		%g7, %src, %g7		/*  IEU1	Group			*/
+	be,pt		%xcc, 1f		/*  CTI					*/
+	 mov		0x40, %g1		/*  IEU0				*/
+	lduw		[%src], %g2		/*  Load	Group			*/
+	subcc		%sum, %g2, %sum		/*  IEU1	Group+load stall	*/
+	bcs,a,pn	%icc, 1f		/*  CTI					*/
+	 sub		%sum, 1, %sum		/*  IEU0				*/
+1:	srl		%sum, 0, %sum		/*  IEU0	Group			*/
+	clr		%g5			/*  IEU1				*/
+	brz,pn		%o4, 3f			/*  CTI+IEU1	Group			*/
+	 sub		%g1, %o4, %g1		/*  IEU0				*/
+	ldd		[%src], %f0		/*  Load				*/
+	clr		%o4			/*  IEU0	Group			*/
+	andcc		%dst, 8, %g0		/*  IEU1				*/
+	be,pn		%icc, 1f		/*  CTI					*/
+	 ldd		[%src + 8], %f2		/*  Load	Group			*/
+	add		%src, 8, %src		/*  IEU0				*/
+	sub		%len, 8, %len		/*  IEU1				*/
+	fpadd32		%f0, %f48, %f50		/*  FPA					*/
+	addcc		%dst, 8, %dst		/*  IEU1	Group			*/
+	faligndata	%f0, %f2, %f16		/*  FPA					*/
+	fcmpgt32	%f48, %f50, %o4		/*  FPM		Group			*/
+	fmovd		%f2, %f0		/*  FPA		Group			*/
+	ldd		[%src + 8], %f2		/*  Load				*/
+	stda		%f16, [%dst - 8] %asi	/*  Store				*/
+	fmovd		%f50, %f48		/*  FPA					*/
+1:	andcc		%g1, 0x10, %g0		/*  IEU1	Group			*/
+	be,pn		%icc, 1f		/*  CTI					*/
+	 and		%g1, 0x20, %g1		/*  IEU0				*/
+	fpadd32		%f0, %f48, %f50		/*  FPA					*/
+	ldd		[%src + 16], %f4	/*  Load	Group			*/
+	add		%src, 16, %src		/*  IEU0				*/
+	add		%dst, 16, %dst		/*  IEU1				*/
+	faligndata	%f0, %f2, %f16		/*  FPA					*/
+	fcmpgt32	%f48, %f50, %g5		/*  FPM		Group			*/
+	sub		%len, 16, %len		/*  IEU0				*/
+	inc		%o4			/*  IEU1				*/
+	stda		%f16, [%dst - 16] %asi	/*  Store	Group			*/
+	fpadd32		%f2, %f50, %f48		/*  FPA					*/
+	srl		%o4, 1, %o5		/*  IEU0				*/
+	faligndata	%f2, %f4, %f18		/*  FPA		Group			*/
+	stda		%f18, [%dst - 8] %asi	/*  Store				*/
+	fcmpgt32	%f50, %f48, %o4		/*  FPM		Group			*/
+	add		%o5, %sum, %sum		/*  IEU0				*/
+	ldd		[%src + 8], %f2		/*  Load				*/
+	fmovd		%f4, %f0		/*  FPA					*/
+1:	brz,a,pn	%g1, 4f			/*  CTI+IEU1	Group			*/
+	 rd		%asi, %g2		/*  LSU		Group + 4 bubbles	*/
+	inc		%g5			/*  IEU0				*/
+	fpadd32		%f0, %f48, %f50		/*  FPA					*/
+	ldd		[%src + 16], %f4	/*  Load	Group			*/
+	srl		%g5, 1, %g5		/*  IEU0				*/
+	add		%dst, 32, %dst		/*  IEU1				*/
+	faligndata	%f0, %f2, %f16		/*  FPA					*/
+	fcmpgt32	%f48, %f50, %o5		/*  FPM		Group			*/
+	inc		%o4			/*  IEU0				*/
+	ldd		[%src + 24], %f6	/*  Load				*/
+	srl		%o4, 1, %o4		/*  IEU0	Group			*/
+	add		%g5, %sum, %sum		/*  IEU1				*/
+	ldd		[%src + 32], %f8	/*  Load				*/
+	fpadd32		%f2, %f50, %f48		/*  FPA					*/
+	faligndata	%f2, %f4, %f18		/*  FPA		Group			*/
+	sub		%len, 32, %len		/*  IEU0				*/
+	stda		%f16, [%dst - 32] %asi	/*  Store				*/
+	fcmpgt32	%f50, %f48, %g3		/*  FPM		Group			*/
+	inc		%o5			/*  IEU0				*/
+	add		%o4, %sum, %sum		/*  IEU1				*/
+	fpadd32		%f4, %f48, %f50		/*  FPA					*/
+	faligndata	%f4, %f6, %f20		/*  FPA		Group			*/
+	srl		%o5, 1, %o5		/*  IEU0				*/
+	fcmpgt32	%f48, %f50, %g5		/*  FPM		Group			*/
+	add		%o5, %sum, %sum		/*  IEU0				*/
+	stda		%f18, [%dst - 24] %asi	/*  Store				*/
+	fpadd32		%f6, %f50, %f48		/*  FPA					*/
+	inc		%g3			/*  IEU0	Group			*/
+	stda		%f20, [%dst - 16] %asi	/*  Store				*/
+	add		%src, 32, %src		/*  IEU1				*/
+	faligndata	%f6, %f8, %f22		/*  FPA					*/
+	fcmpgt32	%f50, %f48, %o4		/*  FPM		Group			*/
+	srl		%g3, 1, %g3		/*  IEU0				*/
+	stda		%f22, [%dst - 8] %asi	/*  Store				*/	
+	add		%g3, %sum, %sum		/*  IEU0	Group			*/
+3:	rd		%asi, %g2		/*  LSU		Group + 4 bubbles	*/
+#ifdef __KERNEL__
+4:	sethi		%hi(vis0s), %g7		/*  IEU0	Group			*/
+	or		%g2, ASI_BLK_OR, %g2	/*  IEU1				*/
+#else
+4:	rd		%pc, %g7		/*  LSU		Group + 4 bubbles	*/
+#endif
+	inc		%g5			/*  IEU0	Group			*/
+	and		%src, 0x38, %g3		/*  IEU1				*/	
+	membar		#StoreLoad		/*  LSU		Group			*/
+	srl		%g5, 1, %g5		/*  IEU0				*/
+	inc		%o4			/*  IEU1				*/
+	sll		%g3, 8, %g3		/*  IEU0	Group			*/
+	sub		%len, 0xc0, %len	/*  IEU1				*/
+	addcc		%g5, %sum, %sum		/*  IEU1	Group			*/
+	srl		%o4, 1, %o4		/*  IEU0				*/
+	add		%g7, %g3, %g7		/*  IEU0	Group			*/
+	add		%o4, %sum, %sum		/*  IEU1				*/
+#ifdef __KERNEL__
+	jmpl		%g7 + %lo(vis0s), %g0	/*  CTI+IEU1	Group			*/
+#else
+	jmpl		%g7 + (vis0s - 4b), %g0	/*  CTI+IEU1	Group			*/
+#endif
+	 fzero		%f32			/*  FPA					*/
+
+	.align		2048
+vis0s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group			*/
+	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	fmovd		%f48, %f62		/*  FPA		Group	f0 available	*/
+	faligndata	%f0, %f2, %f48		/*  FPA		Group	f2 available	*/
+	fcmpgt32	%f32, %f2, %x1		/*  FPM		Group	f4 available	*/
+	fpadd32		%f0, %f62, %f0		/*  FPA					*/
+	fcmpgt32	%f32, %f4, %x2		/*  FPM		Group	f6 available	*/
+	faligndata	%f2, %f4, %f50		/*  FPA					*/
+	fcmpgt32	%f62, %f0, %x3		/*  FPM		Group	f8 available	*/
+	faligndata	%f4, %f6, %f52		/*  FPA					*/
+	fcmpgt32	%f32, %f6, %x4		/*  FPM		Group	f10 available	*/
+	inc		%x1			/*  IEU0				*/
+	faligndata	%f6, %f8, %f54		/*  FPA					*/
+	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group	f12 available	*/
+	srl		%x1, 1, %x1		/*  IEU0				*/
+	inc		%x2			/*  IEU1				*/
+	faligndata	%f8, %f10, %f56		/*  FPA					*/
+	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group	f14 available	*/
+	srl		%x2, 1, %x2		/*  IEU0				*/
+	add		%sum, %x1, %sum		/*  IEU1				*/
+	faligndata	%f10, %f12, %f58	/*  FPA					*/
+	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group			*/
+	inc		%x3			/*  IEU0				*/
+	add		%sum, %x2, %sum		/*  IEU1				*/
+	faligndata	%f12, %f14, %f60	/*  FPA					*/
+	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group			*/
+	srl		%x3, 1, %x3		/*  IEU0				*/
+	inc		%x4			/*  IEU1				*/
+	fmovd		%f14, %f62		/*  FPA					*/
+	srl		%x4, 1, %x4		/*  IEU0	Group			*/
+	add		%sum, %x3, %sum		/*  IEU1				*/
+vis0:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
+			,f48,f50,f52,f54,f56,f58,f60,f62,f62,								
+			,LDBLK(f32),	STBLK,,,,,,,,									
+			,bcs,pn %icc, vis0e1)
+	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f48,f50,f52,f54,f56,f58,f60,f62,f62,								
+			,LDBLK(f0),	STBLK,,,,,,,,									
+			,bcs,pn %icc, vis0e2)
+	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f48,f50,f52,f54,f56,f58,f60,f62,f62,
+			,LDBLK(f16),	STBLK,,,,,,,,
+			,bcc,pt %icc, vis0)
+vis0e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f48,f50,f52,f54,f56,f58,f60,f62,f32,
+			,SYNC,		STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
+			,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e2)
+vis0e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f48,f50,f52,f54,f56,f58,f60,f62,f0,
+			,SYNC,		STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
+			,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e3)
+vis0e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f48,f50,f52,f54,f56,f58,f60,f62,f16,
+			,SYNC,		STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
+			,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e1)
+	.align		2048
+vis1s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group			*/
+	sub		%src, 8, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	fmovd		%f0, %f58		/*  FPA		Group			*/
+	fmovd		%f48, %f0		/*  FPA		Group			*/
+	fcmpgt32	%f32, %f2, %x2		/*  FPM		Group			*/
+	faligndata	%f2, %f4, %f48		/*  FPA					*/
+	fcmpgt32	%f32, %f4, %x3		/*  FPM		Group			*/
+	faligndata	%f4, %f6, %f50		/*  FPA					*/
+	fcmpgt32	%f32, %f6, %x4		/*  FPM		Group			*/
+	faligndata	%f6, %f8, %f52		/*  FPA					*/
+	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group			*/
+	inc		%x2			/*  IEU1				*/
+	faligndata	%f8, %f10, %f54		/*  FPA					*/
+	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group			*/
+	srl		%x2, 1, %x2		/*  IEU0				*/
+	faligndata	%f10, %f12, %f56	/*  FPA					*/
+	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group			*/
+	inc		%x3			/*  IEU0				*/
+	add		%sum, %x2, %sum		/*  IEU1				*/
+	faligndata	%f12, %f14, %f58	/*  FPA					*/
+	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group			*/
+	srl		%x3, 1, %x3		/*  IEU0				*/
+	inc		%x4			/*  IEU1				*/
+	fmovd		%f14, %f60		/*  FPA					*/
+	srl		%x4, 1, %x4		/*  IEU0	Group			*/
+	add		%sum, %x3, %sum		/*  IEU1				*/
+vis1:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f62,f48,f50,f52,f54,f56,f58,f60,f60,
+			,LDBLK(f32),	,STBLK,,,,,,,
+			,bcs,pn %icc, vis1e1)
+	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f62,f48,f50,f52,f54,f56,f58,f60,f60,
+			,LDBLK(f0),	,STBLK,,,,,,,
+			,bcs,pn %icc, vis1e2)
+	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f62,f48,f50,f52,f54,f56,f58,f60,f60,
+			,LDBLK(f16),	,STBLK,,,,,,,
+			,bcc,pt %icc, vis1)
+vis1e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f62,f48,f50,f52,f54,f56,f58,f60,f32,
+			,SYNC,		,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
+			,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e2)
+vis1e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f62,f48,f50,f52,f54,f56,f58,f60,f0,
+			,SYNC,		,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
+			,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e3)
+vis1e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f62,f48,f50,f52,f54,f56,f58,f60,f16,
+			,SYNC,		,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
+			,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e1)
+	.align		2048
+vis2s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group			*/
+	sub		%src, 16, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	fmovd		%f0, %f56		/*  FPA		Group			*/
+	fmovd		%f48, %f0		/*  FPA		Group			*/	
+	sub		%dst, 64, %dst		/*  IEU0				*/
+	fpsub32		%f2, %f2, %f2		/*  FPA		Group			*/
+	fcmpgt32	%f32, %f4, %x3		/*  FPM		Group			*/
+	faligndata	%f4, %f6, %f48		/*  FPA					*/
+	fcmpgt32	%f32, %f6, %x4		/*  FPM		Group			*/
+	faligndata	%f6, %f8, %f50		/*  FPA					*/
+	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group			*/
+	faligndata	%f8, %f10, %f52		/*  FPA					*/
+	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group			*/
+	faligndata	%f10, %f12, %f54	/*  FPA					*/
+	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group			*/
+	inc		%x3			/*  IEU0				*/
+	faligndata	%f12, %f14, %f56	/*  FPA					*/
+	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group			*/
+	srl		%x3, 1, %x3		/*  IEU0				*/
+	inc		%x4			/*  IEU1				*/
+	fmovd		%f14, %f58		/*  FPA					*/
+	srl		%x4, 1, %x4		/*  IEU0	Group			*/
+	add		%sum, %x3, %sum		/*  IEU1				*/
+vis2:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f60,f62,f48,f50,f52,f54,f56,f58,f58,
+			,LDBLK(f32),	,,STBLK,,,,,,
+			,bcs,pn %icc, vis2e1)
+	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f60,f62,f48,f50,f52,f54,f56,f58,f58,
+			,LDBLK(f0),	,,STBLK,,,,,,
+			,bcs,pn %icc, vis2e2)
+	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f60,f62,f48,f50,f52,f54,f56,f58,f58,
+			,LDBLK(f16),	,,STBLK,,,,,,
+			,bcc,pt %icc, vis2)
+vis2e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f60,f62,f48,f50,f52,f54,f56,f58,f32,
+			,SYNC,		,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
+			,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e2)
+vis2e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f60,f62,f48,f50,f52,f54,f56,f58,f0,
+			,SYNC,		,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
+			,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e3)
+vis2e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f60,f62,f48,f50,f52,f54,f56,f58,f16,
+			,SYNC,		,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
+			,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e1)
+	.align		2048
+vis3s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group			*/
+	sub		%src, 24, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	fmovd		%f0, %f54		/*  FPA		Group			*/
+	fmovd		%f48, %f0		/*  FPA		Group			*/
+	sub		%dst, 64, %dst		/*  IEU0				*/
+	fpsub32		%f2, %f2, %f2		/*  FPA		Group			*/
+	fpsub32		%f4, %f4, %f4		/*  FPA		Group			*/
+	fcmpgt32	%f32, %f6, %x4		/*  FPM		Group			*/
+	faligndata	%f6, %f8, %f48		/*  FPA					*/
+	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group			*/
+	faligndata	%f8, %f10, %f50		/*  FPA					*/
+	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group			*/
+	faligndata	%f10, %f12, %f52	/*  FPA					*/
+	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group			*/
+	faligndata	%f12, %f14, %f54	/*  FPA					*/
+	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group			*/
+	fmovd		%f14, %f56		/*  FPA					*/
+	inc		%x4			/*  IEU0				*/
+	srl		%x4, 1, %x4		/*  IEU0	Group			*/
+vis3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f58,f60,f62,f48,f50,f52,f54,f56,f56,
+			,LDBLK(f32),	,,,STBLK,,,,,
+			,bcs,pn %icc, vis3e1)
+	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f58,f60,f62,f48,f50,f52,f54,f56,f56,
+			,LDBLK(f0),	,,,STBLK,,,,,
+			,bcs,pn %icc, vis3e2)
+	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f58,f60,f62,f48,f50,f52,f54,f56,f56,
+			,LDBLK(f16),	,,,STBLK,,,,,
+			,bcc,pt %icc, vis3)
+vis3e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f58,f60,f62,f48,f50,f52,f54,f56,f32,
+			,SYNC,		,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
+			,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e2)
+vis3e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f58,f60,f62,f48,f50,f52,f54,f56,f0,
+			,SYNC,		,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
+			,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e3)
+vis3e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f58,f60,f62,f48,f50,f52,f54,f56,f16,
+			,SYNC,		,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
+			,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e1)
+	.align		2048
+vis4s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group			*/
+	sub		%src, 32, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	fmovd		%f0, %f52		/*  FPA		Group			*/
+	fmovd		%f48, %f0		/*  FPA		Group			*/
+	sub		%dst, 64, %dst		/*  IEU0				*/
+	fpsub32		%f2, %f2, %f2		/*  FPA		Group			*/
+	fpsub32		%f4, %f4, %f4		/*  FPA		Group			*/
+	fpsub32		%f6, %f6, %f6		/*  FPA		Group			*/
+	clr		%x4			/*  IEU0				*/
+	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group			*/
+	faligndata	%f8, %f10, %f48		/*  FPA					*/
+	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group			*/
+	faligndata	%f10, %f12, %f50	/*  FPA					*/
+	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group			*/
+	faligndata	%f12, %f14, %f52	/*  FPA					*/
+	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group			*/
+	fmovd		%f14, %f54		/*  FPA					*/
+vis4:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f56,f58,f60,f62,f48,f50,f52,f54,f54,
+			,LDBLK(f32),	,,,,STBLK,,,,
+			,bcs,pn %icc, vis4e1)
+	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f56,f58,f60,f62,f48,f50,f52,f54,f54,
+			,LDBLK(f0),	,,,,STBLK,,,,
+			,bcs,pn %icc, vis4e2)
+	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f56,f58,f60,f62,f48,f50,f52,f54,f54,
+			,LDBLK(f16),	,,,,STBLK,,,,
+			,bcc,pt %icc, vis4)
+vis4e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f56,f58,f60,f62,f48,f50,f52,f54,f32,
+			,SYNC,		,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
+			,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e2)
+vis4e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f56,f58,f60,f62,f48,f50,f52,f54,f0,
+			,SYNC,		,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
+			,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e3)
+vis4e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f56,f58,f60,f62,f48,f50,f52,f54,f16,
+			,SYNC,		,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
+			,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e1)
+	.align		2048
+vis5s:	ldd		[%src+0], %f10		/*  Load	Group			*/
+	ldd		[%src+8], %f12		/*  Load	Group			*/
+	ldd		[%src+16], %f14		/*  Load	Group			*/
+	add		%src, 24, %src		/*  IEU0	Group			*/
+	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group			*/
+	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	fmovd		%f48, %f0		/*  FPA		Group			*/
+	fmuld		%f32, %f32, %f2		/*  FPM					*/
+	clr		%x4			/*  IEU0				*/
+	faddd		%f32, %f32, %f4		/*  FPA		Group			*/
+	fmuld		%f32, %f32, %f6		/*  FPM					*/
+	clr		%x5			/*  IEU0				*/
+	faddd		%f32, %f32, %f8		/*  FPA		Group			*/
+	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group			*/
+	sub		%dst, 64, %dst		/*  IEU0				*/
+	faligndata	%f10, %f12, %f48	/*  FPA					*/
+	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group			*/
+	faligndata	%f12, %f14, %f50	/*  FPA					*/
+	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group			*/
+	fmovd		%f14, %f52		/*  FPA					*/
+vis5:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f54,f56,f58,f60,f62,f48,f50,f52,f52,
+			,LDBLK(f32),	,,,,,STBLK,,,
+			,bcs,pn %icc, vis5e1)
+	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f54,f56,f58,f60,f62,f48,f50,f52,f52,
+			,LDBLK(f0),	,,,,,STBLK,,,
+			,bcs,pn %icc, vis5e2)
+	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f54,f56,f58,f60,f62,f48,f50,f52,f52,
+			,LDBLK(f16),	,,,,,STBLK,,,
+			,bcc,pt %icc, vis5)
+vis5e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f54,f56,f58,f60,f62,f48,f50,f52,f32,
+			,SYNC,		,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
+			,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e2)
+vis5e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f54,f56,f58,f60,f62,f48,f50,f52,f0,
+			,SYNC,		,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
+			,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e3)
+vis5e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f54,f56,f58,f60,f62,f48,f50,f52,f16,
+			,SYNC,		,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
+			,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e1)
+	.align		2048
+vis6s:	ldd		[%src+0], %f12		/*  Load	Group			*/
+	ldd		[%src+8], %f14		/*  Load	Group			*/
+	add		%src, 16, %src		/*  IEU0	Group			*/
+	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group			*/
+	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	fmovd		%f48, %f0		/*  FPA		Group			*/
+	fmuld		%f32, %f32, %f2		/*  FPM					*/
+	clr		%x4			/*  IEU0				*/
+	faddd		%f32, %f32, %f4		/*  FPA		Group			*/
+	fmuld		%f32, %f32, %f6		/*  FPM					*/
+	clr		%x5			/*  IEU0				*/
+	faddd		%f32, %f32, %f8		/*  FPA		Group			*/
+	fmuld		%f32, %f32, %f10	/*  FPM					*/
+	clr		%x6			/*  IEU0				*/
+	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group			*/
+	sub		%dst, 64, %dst		/*  IEU0				*/
+	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group			*/
+	faligndata	%f12, %f14, %f48	/*  FPA					*/
+	fmovd		%f14, %f50		/*  FPA		Group			*/
+vis6:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f52,f54,f56,f58,f60,f62,f48,f50,f50,
+			,LDBLK(f32),	,,,,,,STBLK,,
+			,bcs,pn %icc, vis6e1)
+	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f52,f54,f56,f58,f60,f62,f48,f50,f50,
+			,LDBLK(f0),	,,,,,,STBLK,,
+			,bcs,pn %icc, vis6e2)
+	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f52,f54,f56,f58,f60,f62,f48,f50,f50,
+			,LDBLK(f16),	,,,,,,STBLK,,
+			,bcc,pt %icc, vis6)
+vis6e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f52,f54,f56,f58,f60,f62,f48,f50,f32,
+			,SYNC,		,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
+			,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e2)
+vis6e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f52,f54,f56,f58,f60,f62,f48,f50,f0,
+			,SYNC,		,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
+			,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e3)
+vis6e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f52,f54,f56,f58,f60,f62,f48,f50,f16,
+			,SYNC,		,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
+			,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e1)
+	.align		2048
+vis7s:	ldd		[%src+0], %f14		/*  Load	Group			*/
+	add		%src, 8, %src		/*  IEU0	Group			*/
+	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group			*/
+	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group			*/
+	add		%src, 64, %src		/*  IEU0	Group			*/
+	fmovd		%f48, %f0		/*  FPA		Group			*/
+	fmuld		%f32, %f32, %f2		/*  FPM					*/
+	clr		%x4			/*  IEU0				*/
+	faddd		%f32, %f32, %f4		/*  FPA		Group			*/
+	fmuld		%f32, %f32, %f6		/*  FPM					*/
+	clr		%x5			/*  IEU0				*/
+	faddd		%f32, %f32, %f8		/*  FPA		Group			*/
+	fmuld		%f32, %f32, %f10	/*  FPM					*/
+	clr		%x6			/*  IEU0				*/
+	faddd		%f32, %f32, %f12	/*  FPA		Group			*/
+	clr		%x7			/*  IEU0				*/
+	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group			*/
+	sub		%dst, 64, %dst		/*  IEU0				*/
+	fmovd		%f14, %f48		/*  FPA					*/
+vis7:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f50,f52,f54,f56,f58,f60,f62,f48,f48,
+			,LDBLK(f32),	,,,,,,,STBLK,
+			,bcs,pn %icc, vis7e1)
+	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f50,f52,f54,f56,f58,f60,f62,f48,f48,
+			,LDBLK(f0),	,,,,,,,STBLK,
+			,bcs,pn %icc, vis7e2)
+	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f50,f52,f54,f56,f58,f60,f62,f48,f48,
+			,LDBLK(f16),	,,,,,,,STBLK,
+			,bcc,pt %icc, vis7)
+vis7e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
+			,f50,f52,f54,f56,f58,f60,f62,f48,f32,
+			,SYNC,		,,,,,,,STBLK_XORASI(x7,x8),
+			,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e2)
+vis7e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
+			,f50,f52,f54,f56,f58,f60,f62,f48,f0,
+			,SYNC,		,,,,,,,STBLK_XORASI(x7,x8),
+			,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e3)
+vis7e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
+			,f50,f52,f54,f56,f58,f60,f62,f48,f16,
+			,SYNC,		,,,,,,,STBLK_XORASI(x7,x8),
+			,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e1)
+e1:	END_THE_TRICK1(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f6)
+e2:	END_THE_TRICK1(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f6)
+e3:	END_THE_TRICK1(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f6)
+ett:	rd		%gsr, %x3		/*  LSU		Group+4bubbles		*/
+	andcc		%x3, 7, %x3		/*  IEU1	Group			*/
+	add		%dst, 8, %dst		/*  IEU0				*/
+	bne,pn		%icc, 1f		/*  CTI					*/
+	 fzero		%f10			/*  FPA					*/
+	brz,a,pn	%len, 2f		/*  CTI+IEU1	Group			*/
+	 stda		%f6, [%dst - 8] %asi	/*  Store				*/
+1:	cmp		%len, 8			/*  IEU1				*/
+	blu,pn		%icc, 3f		/*  CTI					*/
+	 sub		%src, 64, %src		/*  IEU0	Group			*/
+1:	ldd		[%src], %f2		/*  Load	Group			*/
+	fpadd32		%f10, %f2, %f12		/*  FPA		Group+load stall	*/
+	add		%src, 8, %src		/*  IEU0				*/
+	add		%dst, 8, %dst		/*  IEU1				*/
+	faligndata	%f6, %f2, %f14		/*  FPA		Group			*/
+	fcmpgt32	%f10, %f12, %x5		/*  FPM		Group			*/
+	stda		%f14, [%dst - 16] %asi	/*  Store				*/
+	fmovd		%f2, %f6		/*  FPA					*/
+	fmovd		%f12, %f10		/*  FPA		Group			*/
+	sub		%len, 8, %len		/*  IEU1				*/
+	fzero		%f16			/*  FPA		Group - FPU nop		*/
+	fzero		%f18			/*  FPA		Group - FPU nop		*/
+	inc		%x5			/*  IEU0				*/
+	srl		%x5, 1, %x5		/*  IEU0	Group (regdep)		*/
+	cmp		%len, 8			/*  IEU1				*/
+	bgeu,pt		%icc, 1b		/*  CTI					*/
+	 add		%x5, %sum, %sum		/*  IEU0	Group			*/
+3:	brz,a,pt	%x3, 2f			/*  CTI+IEU1				*/
+	 stda		%f6, [%dst - 8] %asi	/*  Store	Group			*/
+	sta		%f7, [%dst - 8]	%asi	/*  Store	Group			*/
+	sub		%dst, 4, %dst		/*  IEU0				*/
+	add		%len, 4, %len		/*  IEU1				*/
+2:
+#ifdef __KERNEL__
+	sub		%sp, 8, %sp		/*  IEU0	Group			*/
+#endif
+	END_THE_TRICK2(	f48,f50,f52,f54,f56,f58,f60,f10,f12,f62)
+	membar		#Sync			/*  LSU		Group			*/
+#ifdef __KERNEL__
+	VISExit
+	add		%sp, 8, %sp		/*  IEU0	Group			*/
+#endif
+23:	brnz,pn		%len, 26f		/*  CTI+IEU1	Group			*/
+24:	 sllx		%sum, 32, %g1		/*  IEU0				*/
+25:	addcc		%sum, %g1, %src		/*  IEU1	Group			*/
+	srlx		%src, 32, %src		/*  IEU0	Group (regdep)		*/
+	bcs,a,pn	%xcc, 1f		/*  CTI					*/
+	 add		%src, 1, %src		/*  IEU1				*/
+#ifndef __KERNEL__
+1:	retl					/*  CTI		Group brk forced	*/
+	 srl		%src, 0, %src		/*  IEU0				*/
+#else
+1:	sethi		%uhi(PAGE_OFFSET), %g4	/*  IEU0	Group			*/
+	retl					/*  CTI		Group brk forced	*/
+	 sllx		%g4, 32, %g4		/*  IEU0				*/
+#endif
+26:	andcc		%len, 8, %g0		/*  IEU1	Group			*/
+	be,pn		%icc, 1f		/*  CTI					*/
+	 lduw		[%src], %o4		/*  Load				*/
+	lduw		[%src+4], %g2		/*  Load	Group			*/
+	add		%src, 8, %src		/*  IEU0				*/
+	add		%dst, 8, %dst		/*  IEU1				*/
+	sllx		%o4, 32, %g5		/*  IEU0	Group			*/
+	stwa		%o4, [%dst - 8] %asi	/*  Store				*/
+	or		%g5, %g2, %g5		/*  IEU0	Group			*/
+	stwa		%g2, [%dst - 4]	%asi	/*  Store				*/
+	addcc		%g5, %sum, %sum		/*  IEU1	Group			*/
+	bcs,a,pn	%xcc, 1f		/*  CTI					*/
+	 add		%sum, 1, %sum		/*  IEU0				*/
+1:	andcc		%len, 4, %g0		/*  IEU1	Group			*/
+	be,a,pn		%icc, 1f		/*  CTI					*/
+	 clr		%g2			/*  IEU0				*/
+	lduw		[%src], %g7		/*  Load				*/
+	add		%src, 4, %src		/*  IEU0	Group			*/
+	add		%dst, 4, %dst		/*  IEU1				*/
+	sllx		%g7, 32, %g2		/*  IEU0	Group			*/
+	stwa		%g7, [%dst - 4] %asi	/*  Store				*/
+1:	andcc		%len, 2, %g0		/*  IEU1				*/
+	be,a,pn		%icc, 1f		/*  CTI					*/
+	 clr		%g3			/*  IEU0	Group			*/
+	lduh		[%src], %g7		/*  Load				*/
+	add		%src, 2, %src		/*  IEU1				*/
+	add		%dst, 2, %dst		/*  IEU0	Group			*/
+	sll		%g7, 16, %g3		/*  IEU0	Group			*/
+	stha		%g7, [%dst - 2] %asi	/*  Store				*/
+1:	andcc		%len, 1, %g0		/*  IEU1				*/
+	be,a,pn		%icc, 1f		/*  CTI					*/
+	 clr		%o5			/*  IEU0	Group			*/
+	ldub		[%src], %g7		/*  Load				*/
+	sll		%g7, 8, %o5		/*  IEU0	Group			*/
+	stba		%g7, [%dst] %asi		/*  Store				*/
+1:	or		%g2, %g3, %g3		/*  IEU1				*/
+	or		%o5, %g3, %g3		/*  IEU0	Group (regdep)		*/
+	addcc		%g3, %sum, %sum		/*  IEU1	Group (regdep)		*/
+	bcs,a,pn	%xcc, 1f		/*  CTI					*/
+	 add		%sum, 1, %sum		/*  IEU0				*/
+1:	ba,pt		%xcc, 25b		/*  CTI		Group			*/
+	 sllx		%sum, 32, %g1		/*  IEU0				*/
+
+#ifdef __KERNEL__
+end:
+
+	.section	__ex_table
+	.align		4
+	.word		csum_partial_copy_user_vis, 0, end, cpc_handler
+#endif