path: root/arch/sparc64/lib/VIScsumcopyusr.S

/* $Id: VIScsumcopyusr.S,v 1.2 2000/02/20 23:21:40 davem Exp $
 * VIScsumcopyusr.S: High bandwidth IP checksumming with simultaneous
 *                   copying utilizing the UltraSparc Visual Instruction Set.
 *
 * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
 * Copyright (C) 2000 David S. Miller (davem@redhat.com)
 *
 * Based on older sparc32/sparc64 checksum.S, which is:
 *
 *      Copyright(C) 1995 Linus Torvalds
 *      Copyright(C) 1995 Miguel de Icaza
 *      Copyright(C) 1996,1997 David S. Miller
 *    derived from:
 *	  Linux/Alpha checksum c-code
 *        Linux/ix86 inline checksum assembly
 *        RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code)
 *	  David Mosberger-Tang for optimized reference c-code
 *	  BSD4.4 portable checksum routine
 */

#ifdef __sparc_v9__
#define STACKOFF	0x7ff+128
#else
#define STACKOFF	64
#endif

#ifdef __KERNEL__
#include <asm/head.h>
#include <asm/asi.h>
#include <asm/page.h>
#include <asm/visasm.h>
#define ASI_BLK_XOR	0
#define ASI_BLK_XOR1	(ASI_BLK_P ^ (ASI_BLK_P >> 3) ^ ASI_P)
#define ASI_BLK_OR	(ASI_BLK_P & ~ASI_P)
#else
#define ASI_P		0x80
#define ASI_BLK_P	0xf0
#define FRPS_FEF	0x04
#define FPRS_DU		0x02
#define FPRS_DL		0x01
#define ASI_BLK_XOR	(ASI_BLK_P ^ ASI_P)
#endif

#define src		o0
#define dst		o1
#define	len		o2
#define sum		o3
#define x1		g1
#define x2		g2
#define x3		o4
#define x4		g4
#define x5		g5
#define x6		g7
#define x7		g3
#define x8		o5

/* Dobrou noc, SunSoft engineers. Spete sladce.
 * This has a couple of tricks in and those
 * tricks are UltraLinux trade secrets :))
 * Once AGAIN, the SunSoft engineers are caught
 * asleep at the keyboard :)).
 * The main loop does about 20 superscalar cycles
 * per 64bytes checksummed/copied.
 */

#define LDBLK(O0)										\
	ldda		[%src] ASI_BLK_P, %O0 	/*  Load	Group			*/

#define STBLK											\
	stda		%f48, [%dst] %asi	/*  Store				*/

#ifdef __KERNEL__
#define STBLK_XORASI(tmpreg1,tmpreg2)								\
	stda		%f48, [%dst] %asi	/*  Store				*/;	\
	rd		%asi, %tmpreg1;								\
	srl		%tmpreg1, 3, %tmpreg2;							\
	xor		%tmpreg1, ASI_BLK_XOR1, %tmpreg1;					\
	wr		%tmpreg1, %tmpreg2, %asi;
#else
#define STBLK_XORASI(tmpreg1,tmpreg2)								\
	stda		%f48, [%dst] %asi	/*  Store				*/;	\
	rd		%asi, %tmpreg1;								\
	wr		%tmpreg1, ASI_BLK_XOR, %asi;
#endif

#define ST(fx,off)										\
	stda		%fx, [%dst + off] %asi	/*  Store				*/

#define SYNC											\
	membar		#Sync


#define DO_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,F0,F2,F4,F6,F8,F10,F12,F14,DUMMY1,A0,A2,A4,A6,A8,A10,A12,A14,B14,DUMMY2,LOAD,STORE1,STORE2,STORE3,STORE4,STORE5,STORE6,STORE7,STORE8,DUMMY3,BRANCH...)	\
	LOAD					/*  Load	(Group)		*/;	\
	faligndata	%A14, %F0, %A14		/*  FPA		Group		*/;	\
	inc		%x5			/*  IEU0			*/;	\
	STORE1					/*  Store (optional)		*/;	\
	faligndata	%F0, %F2, %A0		/*  FPA		Group		*/;	\
	srl		%x5, 1, %x5		/*  IEU0			*/;	\
	add		%sum, %x4, %sum		/*  IEU1			*/;	\
	fpadd32		%F0, %f0, %F0		/*  FPA		Group		*/;	\
	inc		%x6			/*  IEU0			*/;	\
	STORE2					/*  Store (optional)		*/;	\
	faligndata	%F2, %F4, %A2		/*  FPA		Group		*/;	\
	srl		%x6, 1, %x6		/*  IEU0			*/;	\
	add		%sum, %x5, %sum		/*  IEU1			*/;	\
	fpadd32		%F2, %f2, %F2		/*  FPA		Group		*/;	\
	add		%src, 64, %src		/*  IEU0			*/;	\
	fcmpgt32	%f0, %F0, %x1		/*  FPM				*/;	\
	add		%dst, 64, %dst		/*  IEU1	Group		*/;	\
	inc		%x7			/*  IEU0			*/;	\
	STORE3					/*  Store (optional)		*/;	\
	faligndata	%F4, %F6, %A4		/*  FPA				*/;	\
	fpadd32		%F4, %f4, %F4		/*  FPA		Group		*/;	\
	add		%sum, %x6, %sum		/*  IEU1			*/;	\
	fcmpgt32	%f2, %F2, %x2		/*  FPM				*/;	\
	srl		%x7, 1, %x7		/*  IEU0	Group		*/;	\
	inc		%x8			/*  IEU1			*/;	\
	STORE4					/*  Store (optional)		*/;	\
	faligndata	%F6, %F8, %A6		/*  FPA				*/;	\
	fpadd32		%F6, %f6, %F6		/*  FPA		Group		*/;	\
	srl		%x8, 1, %x8		/*  IEU0			*/;	\
	fcmpgt32	%f4, %F4, %x3		/*  FPM				*/;	\
	add		%sum, %x7, %sum		/*  IEU0	Group		*/;	\
	inc		%x1			/*  IEU1			*/;	\
	STORE5					/*  Store (optional)		*/;	\
	faligndata	%F8, %F10, %A8		/*  FPA				*/;	\
	fpadd32		%F8, %f8, %F8		/*  FPA		Group		*/;	\
	srl		%x1, 1, %x1		/*  IEU0			*/;	\
	fcmpgt32	%f6, %F6, %x4		/*  FPM				*/;	\
	add		%sum, %x8, %sum		/*  IEU0	Group		*/;	\
	inc		%x2			/*  IEU1			*/;	\
	STORE6					/*  Store (optional)		*/;	\
	faligndata	%F10, %F12, %A10	/*  FPA				*/;	\
	fpadd32		%F10, %f10, %F10	/*  FPA		Group		*/;	\
	srl		%x2, 1, %x2		/*  IEU0			*/;	\
	fcmpgt32	%f8, %F8, %x5		/*  FPM				*/;	\
	add		%sum, %x1, %sum		/*  IEU0	Group		*/;	\
	inc		%x3			/*  IEU1			*/;	\
	STORE7					/*  Store (optional)		*/;	\
	faligndata	%F12, %F14, %A12	/*  FPA				*/;	\
	fpadd32		%F12, %f12, %F12	/*  FPA		Group		*/;	\
	srl		%x3, 1, %x3		/*  IEU0			*/;	\
	fcmpgt32	%f10, %F10, %x6		/*  FPM				*/;	\
	add		%sum, %x2, %sum		/*  IEU0	Group		*/;	\
	inc		%x4			/*  IEU1			*/;	\
	STORE8					/*  Store (optional)		*/;	\
	fmovd		%F14, %B14		/*  FPA				*/;	\
	fpadd32		%F14, %f14, %F14	/*  FPA		Group		*/;	\
	srl		%x4, 1, %x4		/*  IEU0			*/;	\
	fcmpgt32	%f12, %F12, %x7		/*  FPM				*/;	\
	add		%sum, %x3, %sum		/*  IEU0	Group		*/;	\
	subcc		%len, 64, %len		/*  IEU1			*/;	\
	BRANCH					/*  CTI				*/;	\
	fcmpgt32	%f14, %F14, %x8		/*  FPM		Group		*/;

#define END_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB,S0,S1,S2,S3,T0,T1,U0,fz) \
	inc		%x5			/*  IEU0	Group		*/;	\
	fpadd32		%f2, %f0, %S0		/*  FPA				*/;	\
	add		%sum, %x4, %sum		/*  IEU1			*/;	\
	srl		%x5, 1, %x5		/*  IEU0	Group		*/;	\
	fpadd32		%f6, %f4, %S1		/*  FPA				*/;	\
	inc		%x6			/*  IEU1			*/;	\
	fpadd32		%f10, %f8, %S2		/*  FPA		Group		*/;	\
	add		%sum, %x5, %sum		/*  IEU0			*/;	\
	fcmpgt32	%f0, %S0, %x1		/*  FPM				*/;	\
	fpadd32		%f14, %f12, %S3		/*  FPA		Group		*/;	\
	srl		%x6, 1, %x6		/*  IEU0			*/;	\
	fcmpgt32	%f4, %S1, %x2		/*  FPM				*/;	\
	add		%sum, %x6, %sum		/*  IEU0	Group		*/;	\
	fzero		%fz			/*  FPA				*/;	\
	fcmpgt32	%f8, %S2, %x3		/*  FPM				*/;	\
	inc		%x7			/*  IEU0	Group		*/;	\
	inc		%x8			/*  IEU1			*/;	\
	srl		%x7, 1, %x7		/*  IEU0	Group		*/;	\
	inc		%x1			/*  IEU1			*/;	\
	fpadd32		%S0, %S1, %T0		/*  FPA				*/;	\
	fpadd32		%S2, %S3, %T1		/*  FPA		Group		*/;	\
	add		%sum, %x7, %sum		/*  IEU0			*/;	\
	fcmpgt32	%f12, %S3, %x4		/*  FPM				*/;	\
	srl		%x8, 1, %x8		/*  IEU0	Group		*/;	\
	inc		%x2			/*  IEU1			*/;	\
	srl		%x1, 1, %x1		/*  IEU0	Group		*/;	\
	add		%sum, %x8, %sum		/*  IEU1			*/;	\
	add		%sum, %x1, %sum		/*  IEU0	Group		*/;	\
	fcmpgt32	%S0, %T0, %x5		/*  FPM				*/;	\
	srl		%x2, 1, %x2		/*  IEU0	Group		*/;	\
	fcmpgt32	%S2, %T1, %x6		/*  FPM				*/;	\
	inc		%x3			/*  IEU0	Group		*/;	\
	add		%sum, %x2, %sum		/*  IEU1			*/;	\
	srl		%x3, 1, %x3		/*  IEU0	Group		*/;	\
	inc		%x4			/*  IEU1			*/;	\
	fpadd32		%T0, %T1, %U0		/*  FPA		Group		*/;	\
	add		%sum, %x3, %sum		/*  IEU0			*/;	\
	fcmpgt32	%fz, %f2, %x7		/*  FPM				*/;	\
	srl		%x4, 1, %x4		/*  IEU0	Group		*/;	\
	fcmpgt32	%fz, %f6, %x8		/*  FPM				*/;	\
	inc		%x5			/*  IEU0	Group		*/;	\
	add		%sum, %x4, %sum		/*  IEU1			*/;	\
	srl		%x5, 1, %x5		/*  IEU0	Group		*/;	\
	fcmpgt32	%fz, %f10, %x1		/*  FPM				*/;	\
	inc		%x6			/*  IEU0	Group		*/;	\
	add		%sum, %x5, %sum		/*  IEU1			*/;	\
	fmovd		%FA, %FB		/*  FPA		Group		*/;	\
	fcmpgt32	%fz, %f14, %x2		/*  FPM				*/;	\
	srl		%x6, 1, %x6		/*  IEU0	Group		*/;	\
	ba,pt		%xcc, ett		/*  CTI				*/;	\
	 inc		%x7			/*  IEU1			*/;

#define END_THE_TRICK1(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB) 				\
	END_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB,f48,f50,f52,f54,f56,f58,f60,f62)

#define END_THE_TRICK2(S0,S1,S2,S3,T0,T1,U0,U1,V0,fz) 					\
	fpadd32		%U0, %U1, %V0		/*  FPA		Group		*/;	\
	srl		%x7, 1, %x7		/*  IEU0			*/;	\
	add		%sum, %x6, %sum		/*  IEU1			*/;	\
	std		%V0, [%sp + STACKOFF]	/*  Store	Group		*/;	\
	inc		%x8			/*  IEU0			*/;	\
	sub		%sum, %x7, %sum		/*  IEU1			*/;	\
	srl		%x8, 1, %x8		/*  IEU0	Group		*/;	\
	fcmpgt32	%fz, %S1, %x3		/*  FPM				*/;	\
	inc		%x1			/*  IEU0	Group		*/;	\
	fcmpgt32	%fz, %S3, %x4		/*  FPM				*/;	\
	srl		%x1, 1, %x1		/*  IEU0	Group		*/;	\
	sub		%sum, %x8, %sum		/*  IEU1			*/;	\
	ldx		[%sp + STACKOFF], %x8	/*  Load	Group		*/;	\
	inc		%x2			/*  IEU0			*/;	\
	sub		%sum, %x1, %sum		/*  IEU1			*/;	\
	srl		%x2, 1, %x2		/*  IEU0	Group		*/;	\
	fcmpgt32	%fz, %T1, %x5		/*  FPM				*/;	\
	inc		%x3			/*  IEU0	Group		*/;	\
	fcmpgt32	%T0, %U0, %x6		/*  FPM				*/;	\
	srl		%x3, 1, %x3		/*  IEU0	Group		*/;	\
	sub		%sum, %x2, %sum		/*  IEU1			*/;	\
	inc		%x4			/*  IEU0	Group		*/;	\
	sub		%sum, %x3, %sum		/*  IEU1			*/;	\
	srl		%x4, 1, %x4		/*  IEU0	Group		*/;	\
	fcmpgt32	%fz, %U1, %x7		/*  FPM				*/;	\
	inc		%x5			/*  IEU0	Group		*/;	\
	fcmpgt32	%U0, %V0, %x1		/*  FPM				*/;	\
	srl		%x5, 1, %x5		/*  IEU0	Group		*/;	\
	sub		%sum, %x4, %sum		/*  IEU1			*/;	\
	sub		%sum, %x5, %sum		/*  IEU0	Group		*/;	\
	fcmpgt32	%fz, %V0, %x2		/*  FPM				*/;	\
	inc		%x6			/*  IEU0	Group		*/;	\
	inc		%x7			/*  IEU1			*/;	\
	srl		%x6, 1, %x6		/*  IEU0	Group		*/;	\
	inc		%x1			/*  IEU1			*/;	\
	srl		%x7, 1, %x7		/*  IEU0	Group		*/;	\
	add		%sum, %x6, %sum		/*  IEU1			*/;	\
	srl		%x1, 1, %x1		/*  IEU0	Group		*/;	\
	sub		%sum, %x7, %sum		/*  IEU1			*/;	\
	inc		%x2			/*  IEU0	Group		*/;	\
	add		%sum, %x1, %sum		/*  IEU1			*/;	\
	srl		%x2, 1, %x2		/*  IEU0	Group		*/;	\
	sub		%sum, %x2, %sum		/*  IEU0	Group		*/;	\
	addcc		%sum, %x8, %sum		/*  IEU1	Group		*/;	\
	bcs,a,pn	%xcc, 33f		/*  CTI				*/;	\
	 add		%sum, 1, %sum		/*  IEU0	(Group)		*/;	\
33:						/*  That's it			*/;

	.text
	.globl		csum_partial_copy_user_vis
	.align		32
/* %asi should be either ASI_P or ASI_AIUS for csum_partial_copy resp.
 * csum_partial_copy_from_user
 * This assumes that !((%src^%dst)&3) && !((%src|%dst)&1) && %len >= 256
 */
csum_partial_copy_user_vis:
	andcc		%dst, 7, %g0		/*  IEU1	Group		*/
	be,pt		%icc, 4f		/*  CTI				*/
	 and		%dst, 0x38, %o4		/*  IEU0			*/
	mov		1, %g5			/*  IEU0	Group		*/
	andcc		%dst, 2, %g0		/*  IEU1			*/
	be,pt		%icc, 1f		/*  CTI				*/
	 and		%dst, 4, %g7		/*  IEU0	Group		*/
	lduh		[%src], %g2		/*  Load			*/
	sub		%len, 2, %len		/*  IEU0	Group		*/
	add		%dst, 2, %dst		/*  IEU1			*/
	andcc		%dst, 4, %g7		/*  IEU1	Group		*/
	sll		%g5, 16, %g5		/*  IEU0			*/
	stha		%g2, [%dst - 2]	%asi	/*  Store	Group		*/
	sll		%g2, 16, %g2		/*  IEU0	 		*/
	add		%src, 2, %src		/*  IEU1			*/
	addcc		%g2, %sum, %sum		/*  IEU1	Group		*/
	bcs,a,pn	%icc, 1f		/*  CTI				*/
	 add		%sum, %g5, %sum		/*  IEU0			*/
1:	lduw		[%src], %g2		/*  Load			*/
	brz,a,pn	%g7, 4f			/*  CTI+IEU1	Group		*/
	 and		%dst, 0x38, %o4		/*  IEU0			*/
	add		%dst, 4, %dst		/*  IEU0	Group		*/
	sub		%len, 4, %len		/*  IEU1			*/
	addcc		%g2, %sum, %sum		/*  IEU1	Group		*/
	bcs,a,pn	%icc, 1f		/*  CTI				*/
	 add		%sum, 1, %sum		/*  IEU0			*/
1:	and		%dst, 0x38, %o4		/*  IEU0	Group		*/
	stwa		%g2, [%dst - 4] %asi	/*  Store			*/
	add		%src, 4, %src		/*  IEU1			*/
4:
#ifdef __KERNEL__
	VISEntry
#endif
	mov		%src, %g7		/*  IEU1	Group		*/
	fzero		%f48			/*  FPA				*/
	alignaddr	%src, %g0, %src		/*  Single	Group		*/
	subcc		%g7, %src, %g7		/*  IEU1	Group		*/
	be,pt		%xcc, 1f		/*  CTI				*/
	 mov		0x40, %g1		/*  IEU0			*/
	lduw		[%src], %g2		/*  Load	Group		*/
	subcc		%sum, %g2, %sum		/*  IEU1	Group+load stall*/
	bcs,a,pn	%icc, 1f		/*  CTI				*/
	 sub		%sum, 1, %sum		/*  IEU0			*/
1:	srl		%sum, 0, %sum		/*  IEU0	Group		*/
	clr		%g5			/*  IEU1			*/
	brz,pn		%o4, 3f			/*  CTI+IEU1	Group		*/
	 sub		%g1, %o4, %g1		/*  IEU0			*/
	ldd		[%src], %f0		/*  Load			*/
	clr		%o4			/*  IEU0	Group		*/
	andcc		%dst, 8, %g0		/*  IEU1			*/
	be,pn		%icc, 1f		/*  CTI				*/
	 ldd		[%src + 8], %f2		/*  Load	Group		*/
	add		%src, 8, %src		/*  IEU0			*/
	sub		%len, 8, %len		/*  IEU1			*/
	fpadd32		%f0, %f48, %f50		/*  FPA				*/
	addcc		%dst, 8, %dst		/*  IEU1	Group		*/
	faligndata	%f0, %f2, %f16		/*  FPA				*/
	fcmpgt32	%f48, %f50, %o4		/*  FPM		Group		*/
	fmovd		%f2, %f0		/*  FPA		Group		*/
	ldd		[%src + 8], %f2		/*  Load			*/
	stda		%f16, [%dst - 8] %asi	/*  Store			*/
	fmovd		%f50, %f48		/*  FPA				*/
1:	andcc		%g1, 0x10, %g0		/*  IEU1	Group		*/
	be,pn		%icc, 1f		/*  CTI				*/
	 and		%g1, 0x20, %g1		/*  IEU0			*/
	fpadd32		%f0, %f48, %f50		/*  FPA				*/
	ldd		[%src + 16], %f4	/*  Load	Group		*/
	add		%src, 16, %src		/*  IEU0			*/
	add		%dst, 16, %dst		/*  IEU1			*/
	faligndata	%f0, %f2, %f16		/*  FPA				*/
	fcmpgt32	%f48, %f50, %g5		/*  FPM		Group		*/
	sub		%len, 16, %len		/*  IEU0			*/
	inc		%o4			/*  IEU1			*/
	stda		%f16, [%dst - 16] %asi	/*  Store	Group		*/
	fpadd32		%f2, %f50, %f48		/*  FPA				*/
	srl		%o4, 1, %o5		/*  IEU0			*/
	faligndata	%f2, %f4, %f18		/*  FPA		Group		*/
	stda		%f18, [%dst - 8] %asi	/*  Store			*/
	fcmpgt32	%f50, %f48, %o4		/*  FPM		Group		*/
	add		%o5, %sum, %sum		/*  IEU0			*/
	ldd		[%src + 8], %f2		/*  Load			*/
	fmovd		%f4, %f0		/*  FPA				*/
1:	brz,a,pn	%g1, 4f			/*  CTI+IEU1	Group		*/
	 rd		%asi, %g2		/*  LSU		Group + 4 bubbles*/
	inc		%g5			/*  IEU0			*/
	fpadd32		%f0, %f48, %f50		/*  FPA				*/
	ldd		[%src + 16], %f4	/*  Load	Group		*/
	srl		%g5, 1, %g5		/*  IEU0			*/
	add		%dst, 32, %dst		/*  IEU1			*/
	faligndata	%f0, %f2, %f16		/*  FPA				*/
	fcmpgt32	%f48, %f50, %o5		/*  FPM		Group		*/
	inc		%o4			/*  IEU0			*/
	ldd		[%src + 24], %f6	/*  Load			*/
	srl		%o4, 1, %o4		/*  IEU0	Group		*/
	add		%g5, %sum, %sum		/*  IEU1			*/
	ldd		[%src + 32], %f8	/*  Load			*/
	fpadd32		%f2, %f50, %f48		/*  FPA				*/
	faligndata	%f2, %f4, %f18		/*  FPA		Group		*/
	sub		%len, 32, %len		/*  IEU0			*/
	stda		%f16, [%dst - 32] %asi	/*  Store			*/
	fcmpgt32	%f50, %f48, %g3		/*  FPM		Group		*/
	inc		%o5			/*  IEU0			*/
	add		%o4, %sum, %sum		/*  IEU1			*/
	fpadd32		%f4, %f48, %f50		/*  FPA				*/
	faligndata	%f4, %f6, %f20		/*  FPA		Group		*/
	srl		%o5, 1, %o5		/*  IEU0			*/
	fcmpgt32	%f48, %f50, %g5		/*  FPM		Group		*/
	add		%o5, %sum, %sum		/*  IEU0			*/
	stda		%f18, [%dst - 24] %asi	/*  Store			*/
	fpadd32		%f6, %f50, %f48		/*  FPA				*/
	inc		%g3			/*  IEU0	Group		*/
	stda		%f20, [%dst - 16] %asi	/*  Store			*/
	add		%src, 32, %src		/*  IEU1			*/
	faligndata	%f6, %f8, %f22		/*  FPA				*/
	fcmpgt32	%f50, %f48, %o4		/*  FPM		Group		*/
	srl		%g3, 1, %g3		/*  IEU0			*/
	stda		%f22, [%dst - 8] %asi	/*  Store			*/	
	add		%g3, %sum, %sum		/*  IEU0	Group		*/
3:	rd		%asi, %g2		/*  LSU		Group + 4 bubbles*/
#ifdef __KERNEL__
4:	sethi		%hi(vis0s), %g7		/*  IEU0	Group		*/
	or		%g2, ASI_BLK_OR, %g2	/*  IEU1			*/
#else
4:	rd		%pc, %g7		/*  LSU		Group + 4 bubbles*/
#endif
	inc		%g5			/*  IEU0	Group		*/
	and		%src, 0x38, %g3		/*  IEU1			*/	
	membar		#StoreLoad		/*  LSU		Group		*/
	srl		%g5, 1, %g5		/*  IEU0			*/
	inc		%o4			/*  IEU1			*/
	sll		%g3, 8, %g3		/*  IEU0	Group		*/
	sub		%len, 0xc0, %len	/*  IEU1			*/
	addcc		%g5, %sum, %sum		/*  IEU1	Group		*/
	srl		%o4, 1, %o4		/*  IEU0			*/
	add		%g7, %g3, %g7		/*  IEU0	Group		*/
	add		%o4, %sum, %sum		/*  IEU1			*/
#ifdef __KERNEL__
	jmpl		%g7 + %lo(vis0s), %g0	/*  CTI+IEU1	Group		*/
#else
	jmpl		%g7 + (vis0s - 4b), %g0	/*  CTI+IEU1	Group		*/
#endif
	 fzero		%f32			/*  FPA				*/

	.align		2048
vis0s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group		*/
	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	fmovd		%f48, %f62		/*  FPA		Group	f0 available*/
	faligndata	%f0, %f2, %f48		/*  FPA		Group	f2 available*/
	fcmpgt32	%f32, %f2, %x1		/*  FPM		Group	f4 available*/
	fpadd32		%f0, %f62, %f0		/*  FPA				*/
	fcmpgt32	%f32, %f4, %x2		/*  FPM		Group	f6 available*/
	faligndata	%f2, %f4, %f50		/*  FPA				*/
	fcmpgt32	%f62, %f0, %x3		/*  FPM		Group	f8 available*/
	faligndata	%f4, %f6, %f52		/*  FPA				*/
	fcmpgt32	%f32, %f6, %x4		/*  FPM		Group	f10 available*/
	inc		%x1			/*  IEU0			*/
	faligndata	%f6, %f8, %f54		/*  FPA				*/
	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group	f12 available*/
	srl		%x1, 1, %x1		/*  IEU0			*/
	inc		%x2			/*  IEU1			*/
	faligndata	%f8, %f10, %f56		/*  FPA				*/
	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group	f14 available*/
	srl		%x2, 1, %x2		/*  IEU0			*/
	add		%sum, %x1, %sum		/*  IEU1			*/
	faligndata	%f10, %f12, %f58	/*  FPA				*/
	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group		*/
	inc		%x3			/*  IEU0			*/
	add		%sum, %x2, %sum		/*  IEU1			*/
	faligndata	%f12, %f14, %f60	/*  FPA				*/
	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group		*/
	srl		%x3, 1, %x3		/*  IEU0			*/
	inc		%x4			/*  IEU1			*/
	fmovd		%f14, %f62		/*  FPA				*/
	srl		%x4, 1, %x4		/*  IEU0	Group		*/
	add		%sum, %x3, %sum		/*  IEU1			*/
vis0:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
			,f48,f50,f52,f54,f56,f58,f60,f62,f62,								
			,LDBLK(f32),	STBLK,,,,,,,,									
			,bcs,pn %icc, vis0e1)
	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f48,f50,f52,f54,f56,f58,f60,f62,f62,								
			,LDBLK(f0),	STBLK,,,,,,,,									
			,bcs,pn %icc, vis0e2)
	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f48,f50,f52,f54,f56,f58,f60,f62,f62,
			,LDBLK(f16),	STBLK,,,,,,,,
			,bcc,pt %icc, vis0)
vis0e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f48,f50,f52,f54,f56,f58,f60,f62,f32,
			,SYNC,		STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
			,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e2)
vis0e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f48,f50,f52,f54,f56,f58,f60,f62,f0,
			,SYNC,		STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
			,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e3)
vis0e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f48,f50,f52,f54,f56,f58,f60,f62,f16,
			,SYNC,		STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
			,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e1)
	.align		2048
vis1s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group		*/
	sub		%src, 8, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	fmovd		%f0, %f58		/*  FPA		Group		*/
	fmovd		%f48, %f0		/*  FPA		Group		*/
	fcmpgt32	%f32, %f2, %x2		/*  FPM		Group		*/
	faligndata	%f2, %f4, %f48		/*  FPA				*/
	fcmpgt32	%f32, %f4, %x3		/*  FPM		Group		*/
	faligndata	%f4, %f6, %f50		/*  FPA				*/
	fcmpgt32	%f32, %f6, %x4		/*  FPM		Group		*/
	faligndata	%f6, %f8, %f52		/*  FPA				*/
	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group		*/
	inc		%x2			/*  IEU1			*/
	faligndata	%f8, %f10, %f54		/*  FPA				*/
	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group		*/
	srl		%x2, 1, %x2		/*  IEU0			*/
	faligndata	%f10, %f12, %f56	/*  FPA				*/
	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group		*/
	inc		%x3			/*  IEU0			*/
	add		%sum, %x2, %sum		/*  IEU1			*/
	faligndata	%f12, %f14, %f58	/*  FPA				*/
	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group		*/
	srl		%x3, 1, %x3		/*  IEU0			*/
	inc		%x4			/*  IEU1			*/
	fmovd		%f14, %f60		/*  FPA				*/
	srl		%x4, 1, %x4		/*  IEU0	Group		*/
	add		%sum, %x3, %sum		/*  IEU1			*/
vis1:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f62,f48,f50,f52,f54,f56,f58,f60,f60,
			,LDBLK(f32),	,STBLK,,,,,,,
			,bcs,pn %icc, vis1e1)
	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f62,f48,f50,f52,f54,f56,f58,f60,f60,
			,LDBLK(f0),	,STBLK,,,,,,,
			,bcs,pn %icc, vis1e2)
	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f62,f48,f50,f52,f54,f56,f58,f60,f60,
			,LDBLK(f16),	,STBLK,,,,,,,
			,bcc,pt %icc, vis1)
vis1e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f62,f48,f50,f52,f54,f56,f58,f60,f32,
			,SYNC,		,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
			,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e2)
vis1e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f62,f48,f50,f52,f54,f56,f58,f60,f0,
			,SYNC,		,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
			,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e3)
vis1e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f62,f48,f50,f52,f54,f56,f58,f60,f16,
			,SYNC,		,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
			,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e1)
	.align		2048
vis2s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group		*/
	sub		%src, 16, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	fmovd		%f0, %f56		/*  FPA		Group		*/
	fmovd		%f48, %f0		/*  FPA		Group		*/	
	sub		%dst, 64, %dst		/*  IEU0			*/
	fpsub32		%f2, %f2, %f2		/*  FPA		Group		*/
	fcmpgt32	%f32, %f4, %x3		/*  FPM		Group		*/
	faligndata	%f4, %f6, %f48		/*  FPA				*/
	fcmpgt32	%f32, %f6, %x4		/*  FPM		Group		*/
	faligndata	%f6, %f8, %f50		/*  FPA				*/
	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group		*/
	faligndata	%f8, %f10, %f52		/*  FPA				*/
	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group		*/
	faligndata	%f10, %f12, %f54	/*  FPA				*/
	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group		*/
	inc		%x3			/*  IEU0			*/
	faligndata	%f12, %f14, %f56	/*  FPA				*/
	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group		*/
	srl		%x3, 1, %x3		/*  IEU0			*/
	inc		%x4			/*  IEU1			*/
	fmovd		%f14, %f58		/*  FPA				*/
	srl		%x4, 1, %x4		/*  IEU0	Group		*/
	add		%sum, %x3, %sum		/*  IEU1			*/
vis2:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f60,f62,f48,f50,f52,f54,f56,f58,f58,
			,LDBLK(f32),	,,STBLK,,,,,,
			,bcs,pn %icc, vis2e1)
	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f60,f62,f48,f50,f52,f54,f56,f58,f58,
			,LDBLK(f0),	,,STBLK,,,,,,
			,bcs,pn %icc, vis2e2)
	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f60,f62,f48,f50,f52,f54,f56,f58,f58,
			,LDBLK(f16),	,,STBLK,,,,,,
			,bcc,pt %icc, vis2)
vis2e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f60,f62,f48,f50,f52,f54,f56,f58,f32,
			,SYNC,		,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
			,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e2)
vis2e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f60,f62,f48,f50,f52,f54,f56,f58,f0,
			,SYNC,		,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
			,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e3)
vis2e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f60,f62,f48,f50,f52,f54,f56,f58,f16,
			,SYNC,		,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
			,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e1)
	.align		2048
vis3s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group		*/
	sub		%src, 24, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	fmovd		%f0, %f54		/*  FPA		Group		*/
	fmovd		%f48, %f0		/*  FPA		Group		*/
	sub		%dst, 64, %dst		/*  IEU0			*/
	fpsub32		%f2, %f2, %f2		/*  FPA		Group		*/
	fpsub32		%f4, %f4, %f4		/*  FPA		Group		*/
	fcmpgt32	%f32, %f6, %x4		/*  FPM		Group		*/
	faligndata	%f6, %f8, %f48		/*  FPA				*/
	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group		*/
	faligndata	%f8, %f10, %f50		/*  FPA				*/
	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group		*/
	faligndata	%f10, %f12, %f52	/*  FPA				*/
	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group		*/
	faligndata	%f12, %f14, %f54	/*  FPA				*/
	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group		*/
	fmovd		%f14, %f56		/*  FPA				*/
	inc		%x4			/*  IEU0			*/
	srl		%x4, 1, %x4		/*  IEU0	Group		*/
vis3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f58,f60,f62,f48,f50,f52,f54,f56,f56,
			,LDBLK(f32),	,,,STBLK,,,,,
			,bcs,pn %icc, vis3e1)
	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f58,f60,f62,f48,f50,f52,f54,f56,f56,
			,LDBLK(f0),	,,,STBLK,,,,,
			,bcs,pn %icc, vis3e2)
	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f58,f60,f62,f48,f50,f52,f54,f56,f56,
			,LDBLK(f16),	,,,STBLK,,,,,
			,bcc,pt %icc, vis3)
vis3e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f58,f60,f62,f48,f50,f52,f54,f56,f32,
			,SYNC,		,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
			,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e2)
vis3e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f58,f60,f62,f48,f50,f52,f54,f56,f0,
			,SYNC,		,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
			,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e3)
vis3e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f58,f60,f62,f48,f50,f52,f54,f56,f16,
			,SYNC,		,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
			,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e1)
	.align		2048
vis4s:	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group		*/
	sub		%src, 32, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f0	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	fmovd		%f0, %f52		/*  FPA		Group		*/
	fmovd		%f48, %f0		/*  FPA		Group		*/
	sub		%dst, 64, %dst		/*  IEU0			*/
	fpsub32		%f2, %f2, %f2		/*  FPA		Group		*/
	fpsub32		%f4, %f4, %f4		/*  FPA		Group		*/
	fpsub32		%f6, %f6, %f6		/*  FPA		Group		*/
	clr		%x4			/*  IEU0			*/
	fcmpgt32	%f32, %f8, %x5		/*  FPM		Group		*/
	faligndata	%f8, %f10, %f48		/*  FPA				*/
	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group		*/
	faligndata	%f10, %f12, %f50	/*  FPA				*/
	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group		*/
	faligndata	%f12, %f14, %f52	/*  FPA				*/
	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group		*/
	fmovd		%f14, %f54		/*  FPA				*/
vis4:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f56,f58,f60,f62,f48,f50,f52,f54,f54,
			,LDBLK(f32),	,,,,STBLK,,,,
			,bcs,pn %icc, vis4e1)
	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f56,f58,f60,f62,f48,f50,f52,f54,f54,
			,LDBLK(f0),	,,,,STBLK,,,,
			,bcs,pn %icc, vis4e2)
	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f56,f58,f60,f62,f48,f50,f52,f54,f54,
			,LDBLK(f16),	,,,,STBLK,,,,
			,bcc,pt %icc, vis4)
vis4e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f56,f58,f60,f62,f48,f50,f52,f54,f32,
			,SYNC,		,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
			,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e2)
vis4e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f56,f58,f60,f62,f48,f50,f52,f54,f0,
			,SYNC,		,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
			,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e3)
vis4e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f56,f58,f60,f62,f48,f50,f52,f54,f16,
			,SYNC,		,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
			,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e1)
	.align		2048
vis5s:	ldd		[%src+0], %f10		/*  Load	Group		*/
	ldd		[%src+8], %f12		/*  Load	Group		*/
	ldd		[%src+16], %f14		/*  Load	Group		*/
	add		%src, 24, %src		/*  IEU0	Group		*/
	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group		*/
	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	fmovd		%f48, %f0		/*  FPA		Group		*/
	fmuld		%f32, %f32, %f2		/*  FPM				*/
	clr		%x4			/*  IEU0			*/
	faddd		%f32, %f32, %f4		/*  FPA		Group		*/
	fmuld		%f32, %f32, %f6		/*  FPM				*/
	clr		%x5			/*  IEU0			*/
	faddd		%f32, %f32, %f8		/*  FPA		Group		*/
	fcmpgt32	%f32, %f10, %x6		/*  FPM		Group		*/
	sub		%dst, 64, %dst		/*  IEU0			*/
	faligndata	%f10, %f12, %f48	/*  FPA				*/
	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group		*/
	faligndata	%f12, %f14, %f50	/*  FPA				*/
	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group		*/
	fmovd		%f14, %f52		/*  FPA				*/
vis5:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f54,f56,f58,f60,f62,f48,f50,f52,f52,
			,LDBLK(f32),	,,,,,STBLK,,,
			,bcs,pn %icc, vis5e1)
	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f54,f56,f58,f60,f62,f48,f50,f52,f52,
			,LDBLK(f0),	,,,,,STBLK,,,
			,bcs,pn %icc, vis5e2)
	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f54,f56,f58,f60,f62,f48,f50,f52,f52,
			,LDBLK(f16),	,,,,,STBLK,,,
			,bcc,pt %icc, vis5)
vis5e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f54,f56,f58,f60,f62,f48,f50,f52,f32,
			,SYNC,		,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
			,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e2)
vis5e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f54,f56,f58,f60,f62,f48,f50,f52,f0,
			,SYNC,		,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
			,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e3)
vis5e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f54,f56,f58,f60,f62,f48,f50,f52,f16,
			,SYNC,		,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
			,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e1)
	.align		2048
vis6s:	ldd		[%src+0], %f12		/*  Load	Group		*/
	ldd		[%src+8], %f14		/*  Load	Group		*/
	add		%src, 16, %src		/*  IEU0	Group		*/
	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group		*/
	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	fmovd		%f48, %f0		/*  FPA		Group		*/
	fmuld		%f32, %f32, %f2		/*  FPM				*/
	clr		%x4			/*  IEU0			*/
	faddd		%f32, %f32, %f4		/*  FPA		Group		*/
	fmuld		%f32, %f32, %f6		/*  FPM				*/
	clr		%x5			/*  IEU0			*/
	faddd		%f32, %f32, %f8		/*  FPA		Group		*/
	fmuld		%f32, %f32, %f10	/*  FPM				*/
	clr		%x6			/*  IEU0			*/
	fcmpgt32	%f32, %f12, %x7		/*  FPM		Group		*/
	sub		%dst, 64, %dst		/*  IEU0			*/
	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group		*/
	faligndata	%f12, %f14, %f48	/*  FPA				*/
	fmovd		%f14, %f50		/*  FPA		Group		*/
vis6:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f52,f54,f56,f58,f60,f62,f48,f50,f50,
			,LDBLK(f32),	,,,,,,STBLK,,
			,bcs,pn %icc, vis6e1)
	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f52,f54,f56,f58,f60,f62,f48,f50,f50,
			,LDBLK(f0),	,,,,,,STBLK,,
			,bcs,pn %icc, vis6e2)
	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f52,f54,f56,f58,f60,f62,f48,f50,f50,
			,LDBLK(f16),	,,,,,,STBLK,,
			,bcc,pt %icc, vis6)
vis6e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f52,f54,f56,f58,f60,f62,f48,f50,f32,
			,SYNC,		,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
			,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e2)
vis6e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f52,f54,f56,f58,f60,f62,f48,f50,f0,
			,SYNC,		,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
			,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e3)
vis6e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f52,f54,f56,f58,f60,f62,f48,f50,f16,
			,SYNC,		,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
			,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e1)
	.align		2048
vis7s:	ldd		[%src+0], %f14		/*  Load	Group		*/
	add		%src, 8, %src		/*  IEU0	Group		*/
	wr		%g2, ASI_BLK_XOR, %asi	/*  LSU		Group		*/
	ldda		[%src] ASI_BLK_P, %f16	/*  Load	Group		*/
	add		%src, 64, %src		/*  IEU0	Group		*/
	fmovd		%f48, %f0		/*  FPA		Group		*/
	fmuld		%f32, %f32, %f2		/*  FPM				*/
	clr		%x4			/*  IEU0			*/
	faddd		%f32, %f32, %f4		/*  FPA		Group		*/
	fmuld		%f32, %f32, %f6		/*  FPM				*/
	clr		%x5			/*  IEU0			*/
	faddd		%f32, %f32, %f8		/*  FPA		Group		*/
	fmuld		%f32, %f32, %f10	/*  FPM				*/
	clr		%x6			/*  IEU0			*/
	faddd		%f32, %f32, %f12	/*  FPA		Group		*/
	clr		%x7			/*  IEU0			*/
	fcmpgt32	%f32, %f14, %x8		/*  FPM		Group		*/
	sub		%dst, 64, %dst		/*  IEU0			*/
	fmovd		%f14, %f48		/*  FPA				*/
vis7:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f50,f52,f54,f56,f58,f60,f62,f48,f48,
			,LDBLK(f32),	,,,,,,,STBLK,
			,bcs,pn %icc, vis7e1)
	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f50,f52,f54,f56,f58,f60,f62,f48,f48,
			,LDBLK(f0),	,,,,,,,STBLK,
			,bcs,pn %icc, vis7e2)
	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f50,f52,f54,f56,f58,f60,f62,f48,f48,
			,LDBLK(f16),	,,,,,,,STBLK,
			,bcc,pt %icc, vis7)
vis7e3:	DO_THE_TRICK(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,	
			,f50,f52,f54,f56,f58,f60,f62,f48,f32,
			,SYNC,		,,,,,,,STBLK_XORASI(x7,x8),
			,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e2)
vis7e1:	DO_THE_TRICK(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
			,f50,f52,f54,f56,f58,f60,f62,f48,f0,
			,SYNC,		,,,,,,,STBLK_XORASI(x7,x8),
			,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e3)
vis7e2:	DO_THE_TRICK(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,	
			,f50,f52,f54,f56,f58,f60,f62,f48,f16,
			,SYNC,		,,,,,,,STBLK_XORASI(x7,x8),
			,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e1)
e1:	END_THE_TRICK1(	f0,f2,f4,f6,f8,f10,f12,f14,f16,f6)
e2:	END_THE_TRICK1(	f16,f18,f20,f22,f24,f26,f28,f30,f32,f6)
e3:	END_THE_TRICK1(	f32,f34,f36,f38,f40,f42,f44,f46,f0,f6)
ett:	rd		%gsr, %x3		/*  LSU		Group+4bubbles	*/
	andcc		%x3, 7, %x3		/*  IEU1	Group		*/
	add		%dst, 8, %dst		/*  IEU0			*/
	bne,pn		%icc, 1f		/*  CTI				*/
	 fzero		%f10			/*  FPA				*/
	brz,a,pn	%len, 2f		/*  CTI+IEU1	Group		*/
	 stda		%f6, [%dst - 8] %asi	/*  Store			*/
1:	cmp		%len, 8			/*  IEU1			*/
	blu,pn		%icc, 3f		/*  CTI				*/
	 sub		%src, 64, %src		/*  IEU0	Group		*/
1:	ldd		[%src], %f2		/*  Load	Group		*/
	fpadd32		%f10, %f2, %f12		/*  FPA		Group+load stall*/
	add		%src, 8, %src		/*  IEU0			*/
	add		%dst, 8, %dst		/*  IEU1			*/
	faligndata	%f6, %f2, %f14		/*  FPA		Group		*/
	fcmpgt32	%f10, %f12, %x5		/*  FPM		Group		*/
	stda		%f14, [%dst - 16] %asi	/*  Store			*/
	fmovd		%f2, %f6		/*  FPA				*/
	fmovd		%f12, %f10		/*  FPA		Group		*/
	sub		%len, 8, %len		/*  IEU1			*/
	fzero		%f16			/*  FPA		Group - FPU nop	*/
	fzero		%f18			/*  FPA		Group - FPU nop	*/
	inc		%x5			/*  IEU0			*/
	srl		%x5, 1, %x5		/*  IEU0	Group (regdep)	*/
	cmp		%len, 8			/*  IEU1			*/
	bgeu,pt		%icc, 1b		/*  CTI				*/
	 add		%x5, %sum, %sum		/*  IEU0	Group		*/
3:	brz,a,pt	%x3, 2f			/*  CTI+IEU1			*/
	 stda		%f6, [%dst - 8] %asi	/*  Store	Group		*/
	sta		%f7, [%dst - 8]	%asi	/*  Store	Group		*/
	sub		%dst, 4, %dst		/*  IEU0			*/
	add		%len, 4, %len		/*  IEU1			*/
2:
#ifdef __KERNEL__
	sub		%sp, 8, %sp		/*  IEU0	Group		*/
#endif
	END_THE_TRICK2(	f48,f50,f52,f54,f56,f58,f60,f10,f12,f62)
	membar		#Sync			/*  LSU		Group		*/
#ifdef __KERNEL__
	VISExit
	add		%sp, 8, %sp		/*  IEU0	Group		*/
#endif
23:	brnz,pn		%len, 26f		/*  CTI+IEU1	Group		*/
24:	 sllx		%sum, 32, %g1		/*  IEU0			*/
25:	addcc		%sum, %g1, %src		/*  IEU1	Group		*/
	srlx		%src, 32, %src		/*  IEU0	Group (regdep)	*/
	bcs,a,pn	%xcc, 1f		/*  CTI				*/
	 add		%src, 1, %src		/*  IEU1			*/
#ifndef __KERNEL__
1:	retl					/*  CTI		Group brk forced*/
	 srl		%src, 0, %src		/*  IEU0			*/
#else
1:	sethi		%uhi(PAGE_OFFSET), %g4	/*  IEU0	Group		*/
	retl					/*  CTI		Group brk forced*/
	 sllx		%g4, 32, %g4		/*  IEU0			*/
#endif
26:	andcc		%len, 8, %g0		/*  IEU1	Group		*/
	be,pn		%icc, 1f		/*  CTI				*/
	 lduw		[%src], %o4		/*  Load			*/
	lduw		[%src+4], %g2		/*  Load	Group		*/
	add		%src, 8, %src		/*  IEU0			*/
	add		%dst, 8, %dst		/*  IEU1			*/
	sllx		%o4, 32, %g5		/*  IEU0	Group		*/
	stwa		%o4, [%dst - 8] %asi	/*  Store			*/
	or		%g5, %g2, %g5		/*  IEU0	Group		*/
	stwa		%g2, [%dst - 4]	%asi	/*  Store			*/
	addcc		%g5, %sum, %sum		/*  IEU1	Group		*/
	bcs,a,pn	%xcc, 1f		/*  CTI				*/
	 add		%sum, 1, %sum		/*  IEU0			*/
1:	andcc		%len, 4, %g0		/*  IEU1	Group		*/
	be,a,pn		%icc, 1f		/*  CTI				*/
	 clr		%g2			/*  IEU0			*/
	lduw		[%src], %g7		/*  Load			*/
	add		%src, 4, %src		/*  IEU0	Group		*/
	add		%dst, 4, %dst		/*  IEU1			*/
	sllx		%g7, 32, %g2		/*  IEU0	Group		*/
	stwa		%g7, [%dst - 4] %asi	/*  Store			*/
1:	andcc		%len, 2, %g0		/*  IEU1			*/
	be,a,pn		%icc, 1f		/*  CTI				*/
	 clr		%g3			/*  IEU0	Group		*/
	lduh		[%src], %g7		/*  Load			*/
	add		%src, 2, %src		/*  IEU1			*/
	add		%dst, 2, %dst		/*  IEU0	Group		*/
	sll		%g7, 16, %g3		/*  IEU0	Group		*/
	stha		%g7, [%dst - 2] %asi	/*  Store			*/
1:	andcc		%len, 1, %g0		/*  IEU1			*/
	be,a,pn		%icc, 1f		/*  CTI				*/
	 clr		%o5			/*  IEU0	Group		*/
	ldub		[%src], %g7		/*  Load			*/
	sll		%g7, 8, %o5		/*  IEU0	Group		*/
	stba		%g7, [%dst] %asi	/*  Store			*/
1:	or		%g2, %g3, %g3		/*  IEU1			*/
	or		%o5, %g3, %g3		/*  IEU0	Group (regdep)	*/
	addcc		%g3, %sum, %sum		/*  IEU1	Group (regdep)	*/
	bcs,a,pn	%xcc, 1f		/*  CTI				*/
	 add		%sum, 1, %sum		/*  IEU0			*/
1:	ba,pt		%xcc, 25b		/*  CTI		Group		*/
	 sllx		%sum, 32, %g1		/*  IEU0			*/

#ifdef __KERNEL__
end:

	.section	__ex_table
	.align		4
	.word		csum_partial_copy_user_vis, 0, end, cpc_handler
#endif