Merge with Linux 2.3.51.

7 files changed, 265 insertions, 174 deletions

diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c
index 00f635ab3..d7267880a 100644
--- a/arch/sparc64/kernel/pci_iommu.c
+++ b/arch/sparc64/kernel/pci_iommu.c
@@ -1,4 +1,4 @@
-/* $Id: pci_iommu.c,v 1.10 2000/02/18 13:48:54 davem Exp $
+/* $Id: pci_iommu.c,v 1.11 2000/03/10 02:42:15 davem Exp $
  * pci_iommu.c: UltraSparc PCI controller IOM/STC support.
  *
  * Copyright (C) 1999 David S. Miller (davem@redhat.com)
@@ -34,46 +34,96 @@
 			     : "r" (__val), "r" (__reg), \
 			       "i" (ASI_PHYS_BYPASS_EC_E))
 
+/* Must be invoked under the IOMMU lock. */
+static void __iommu_flushall(struct pci_iommu *iommu)
+{
+	unsigned long tag;
+	int entry;
+
+	tag = iommu->iommu_flush + (0xa580UL - 0x0210UL);
+	for (entry = 0; entry < 16; entry++) {
+		pci_iommu_write(tag, 0);
+		tag += 8;
+	}
+
+	/* Ensure completion of previous PIO writes. */
+	(void) pci_iommu_read(iommu->write_complete_reg);
+
+	/* Now update everyone's flush point. */
+	for (entry = 0; entry < PBM_NCLUSTERS; entry++) {
+		iommu->alloc_info[entry].flush =
+			iommu->alloc_info[entry].next;
+	}
+}
+
 static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages)
 {
-	iopte_t *iopte;
-	unsigned long cnum, ent;
+	iopte_t *iopte, *limit;
+	unsigned long cnum, ent, flush_point;
 
 	cnum = 0;
 	while ((1UL << cnum) < npages)
 		cnum++;
-	iopte  = iommu->page_table + (cnum << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS));
-	iopte += ((ent = iommu->lowest_free[cnum]) << cnum);
+	iopte  = (iommu->page_table +
+		  (cnum << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
 
-	if (iopte_val(iopte[(1UL << cnum)]) == 0UL) {
-		/* Fast path. */
-		iommu->lowest_free[cnum] = ent + 1;
-	} else {
-		unsigned long pte_off = 1;
+	if (cnum == 0)
+		limit = (iommu->page_table +
+			 iommu->lowest_consistent_map);
+	else
+		limit = (iopte +
+			 (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
 
-		ent += 1;
-		do {
-			pte_off++;
-			ent++;
-		} while (iopte_val(iopte[(pte_off << cnum)]) != 0UL);
-		iommu->lowest_free[cnum] = ent;
+	iopte += ((ent = iommu->alloc_info[cnum].next) << cnum);
+	flush_point = iommu->alloc_info[cnum].flush;
+	
+	for (;;) {
+		if (iopte_val(*iopte) == 0UL) {
+			if ((iopte + (1 << cnum)) >= limit)
+				ent = 0;
+			else
+				ent = ent + 1;
+			iommu->alloc_info[cnum].next = ent;
+			if (ent == flush_point)
+				__iommu_flushall(iommu);
+			break;
+		}
+		iopte += (1 << cnum);
+		ent++;
+		if (iopte >= limit) {
+			iopte = (iommu->page_table +
+				 (cnum <<
+				  (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
+			ent = 0;
+		}
+		if (ent == flush_point)
+			__iommu_flushall(iommu);
 	}
 
 	/* I've got your streaming cluster right here buddy boy... */
 	return iopte;
 }
 
-static inline void free_streaming_cluster(struct pci_iommu *iommu, dma_addr_t base, unsigned long npages)
+static void free_streaming_cluster(struct pci_iommu *iommu, dma_addr_t base,
+				   unsigned long npages, unsigned long ctx)
 {
 	unsigned long cnum, ent;
 
 	cnum = 0;
 	while ((1UL << cnum) < npages)
 		cnum++;
+
 	ent = (base << (32 - PAGE_SHIFT + PBM_LOGCLUSTERS - iommu->page_table_sz_bits))
 		>> (32 + PBM_LOGCLUSTERS + cnum - iommu->page_table_sz_bits);
-	if (ent < iommu->lowest_free[cnum])
-		iommu->lowest_free[cnum] = ent;
+
+	/* If the global flush might not have caught this entry,
+	 * adjust the flush point such that we will flush before
+	 * ever trying to reuse it.
+	 */
+#define between(X,Y,Z)	(((Z) - (Y)) >= ((X) - (Y)))
+	if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush))
+		iommu->alloc_info[cnum].flush = ent;
+#undef between
 }
 
 /* We allocate consistent mappings from the end of cluster zero. */
@@ -92,8 +142,13 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long
 				if (iopte_val(*iopte) & IOPTE_VALID)
 					break;
 			}
-			if (tmp == 0)
+			if (tmp == 0) {
+				u32 entry = (iopte - iommu->page_table);
+
+				if (entry < iommu->lowest_consistent_map)
+					iommu->lowest_consistent_map = entry;
 				return iopte;
+			}
 		}
 	}
 	return NULL;
@@ -182,7 +237,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
 	struct pcidev_cookie *pcp;
 	struct pci_iommu *iommu;
 	iopte_t *iopte;
-	unsigned long flags, order, npages, i;
+	unsigned long flags, order, npages, i, ctx;
 
 	npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 	pcp = pdev->sysdata;
@@ -192,15 +247,45 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
 
 	spin_lock_irqsave(&iommu->lock, flags);
 
+	if ((iopte - iommu->page_table) ==
+	    iommu->lowest_consistent_map) {
+		iopte_t *walk = iopte + npages;
+		iopte_t *limit;
+
+		limit = (iommu->page_table +
+			 (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
+		while (walk < limit) {
+			if (iopte_val(*walk) != IOPTE_INVALID)
+				break;
+			walk++;
+		}
+		iommu->lowest_consistent_map =
+			(walk - iommu->page_table);
+	}
+
 	/* Data for consistent mappings cannot enter the streaming
-	 * buffers, so we only need to update the TSB.  Flush of the
-	 * IOTLB is done later when these ioptes are used for a new
-	 * allocation.
+	 * buffers, so we only need to update the TSB.  We flush
+	 * the IOMMU here as well to prevent conflicts with the
+	 * streaming mapping deferred tlb flush scheme.
 	 */
 
+	ctx = 0;
+	if (iommu->iommu_ctxflush)
+		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
+
 	for (i = 0; i < npages; i++, iopte++)
 		iopte_val(*iopte) = IOPTE_INVALID;
 
+	if (iommu->iommu_ctxflush) {
+		pci_iommu_write(iommu->iommu_ctxflush, ctx);
+	} else {
+		for (i = 0; i < npages; i++) {
+			u32 daddr = dvma + (i << PAGE_SHIFT);
+
+			pci_iommu_write(iommu->iommu_flush, daddr);
+		}
+	}
+
 	spin_unlock_irqrestore(&iommu->lock, flags);
 
 	order = get_order(size);
@@ -253,14 +338,6 @@ dma_addr_t pci_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direct
 	for (i = 0; i < npages; i++, base++, base_paddr += PAGE_SIZE)
 		iopte_val(*base) = iopte_protection | base_paddr;
 
-	/* Flush the IOMMU TLB. */
-	if (iommu->iommu_ctxflush) {
-		pci_iommu_write(iommu->iommu_ctxflush, ctx);
-	} else {
-		for (i = 0; i < npages; i++, bus_addr += PAGE_SIZE)
-			pci_iommu_write(iommu->iommu_flush, bus_addr);
-	}
-
 	spin_unlock_irqrestore(&iommu->lock, flags);
 
 	return ret;
@@ -294,15 +371,15 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
 
 	spin_lock_irqsave(&iommu->lock, flags);
 
+	/* Record the context, if any. */
+	ctx = 0;
+	if (iommu->iommu_ctxflush)
+		ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
+
 	/* Step 1: Kick data out of streaming buffers if necessary. */
 	if (strbuf->strbuf_enabled) {
 		u32 vaddr = bus_addr;
 
-		/* Record the context, if any. */
-		ctx = 0;
-		if (iommu->iommu_ctxflush)
-			ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
-
 		PCI_STC_FLUSHFLAG_INIT(strbuf);
 		if (strbuf->strbuf_ctxflush &&
 		    iommu->iommu_ctxflush) {
@@ -327,10 +404,8 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
 	/* Step 2: Clear out first TSB entry. */
 	iopte_val(*base) = IOPTE_INVALID;
 
-	free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, npages);
-
-	/* Step 3: Ensure completion of previous PIO writes. */
-	(void) pci_iommu_read(iommu->write_complete_reg);
+	free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
+			       npages, ctx);
 
 	spin_unlock_irqrestore(&iommu->lock, flags);
 }
@@ -415,7 +490,7 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int
 	struct pcidev_cookie *pcp;
 	struct pci_iommu *iommu;
 	struct pci_strbuf *strbuf;
-	unsigned long flags, ctx, i, npages, iopte_protection;
+	unsigned long flags, ctx, npages, iopte_protection;
 	iopte_t *base;
 	u32 dma_base;
 	struct scatterlist *sgtmp;
@@ -474,14 +549,6 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int
 	verify_sglist(sglist, nelems, base, npages);
 #endif
 
-	/* Step 6: Flush the IOMMU TLB. */
-	if (iommu->iommu_ctxflush) {
-		pci_iommu_write(iommu->iommu_ctxflush, ctx);
-	} else {
-		for (i = 0; i < npages; i++, dma_base += PAGE_SIZE)
-			pci_iommu_write(iommu->iommu_flush, dma_base);
-	}
-
 	spin_unlock_irqrestore(&iommu->lock, flags);
 
 	return used;
@@ -522,15 +589,15 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
 
 	spin_lock_irqsave(&iommu->lock, flags);
 
+	/* Record the context, if any. */
+	ctx = 0;
+	if (iommu->iommu_ctxflush)
+		ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
+
 	/* Step 1: Kick data out of streaming buffers if necessary. */
 	if (strbuf->strbuf_enabled) {
 		u32 vaddr = bus_addr;
 
-		/* Record the context, if any. */
-		ctx = 0;
-		if (iommu->iommu_ctxflush)
-			ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
-
 		PCI_STC_FLUSHFLAG_INIT(strbuf);
 		if (strbuf->strbuf_ctxflush &&
 		    iommu->iommu_ctxflush) {
@@ -555,10 +622,8 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
 	/* Step 2: Clear out first TSB entry. */
 	iopte_val(*base) = IOPTE_INVALID;
 
-	free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, npages);
-
-	/* Step 3: Ensure completion of previous PIO writes. */
-	(void) pci_iommu_read(iommu->write_complete_reg);
+	free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
+			       npages, ctx);
 
 	spin_unlock_irqrestore(&iommu->lock, flags);
 }
diff --git a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c
index b3248de39..1c8f59f3f 100644
--- a/arch/sparc64/kernel/pci_psycho.c
+++ b/arch/sparc64/kernel/pci_psycho.c
@@ -1,4 +1,4 @@
-/* $Id: pci_psycho.c,v 1.13 2000/02/18 13:48:54 davem Exp $
+/* $Id: pci_psycho.c,v 1.14 2000/03/10 02:42:15 davem Exp $
  * pci_psycho.c: PSYCHO/U2P specific PCI controller support.
  *
  * Copyright (C) 1997, 1998, 1999 David S. Miller (davem@caipfs.rutgers.edu)
@@ -1246,11 +1246,14 @@ static void __init psycho_iommu_init(struct pci_controller_info *p)
 	control = psycho_read(p->controller_regs + PSYCHO_IOMMU_CONTROL);
 	control |= PSYCHO_IOMMU_CTRL_DENAB;
 	psycho_write(p->controller_regs + PSYCHO_IOMMU_CONTROL, control);
-	for(i = 0; i < 16; i++)
+	for(i = 0; i < 16; i++) {
+		psycho_write(p->controller_regs + PSYCHO_IOMMU_TAG + (i * 8UL), 0);
 		psycho_write(p->controller_regs + PSYCHO_IOMMU_DATA + (i * 8UL), 0);
+	}
 
-	control &= ~(PSYCHO_IOMMU_CTRL_DENAB);
-	psycho_write(p->controller_regs + PSYCHO_IOMMU_CONTROL, control);
+	/* Leave diag mode enabled for full-flushing done
+	 * in pci_iommu.c
+	 */
 
 	/* Using assumed page size 8K with 128K entries we need 1MB iommu page
 	 * table (128K ioptes * 8 bytes per iopte).  This is
@@ -1267,9 +1270,14 @@ static void __init psycho_iommu_init(struct pci_controller_info *p)
 	p->iommu.dma_addr_mask = 0xffffffff;
 	memset((char *)tsbbase, 0, PAGE_SIZE << 7);
 
-	/* Make sure DMA address 0 is never returned just to allow catching
-	   of buggy drivers.  */
-	p->iommu.lowest_free[0] = 1;
+	/* We start with no consistent mappings. */
+	p->iommu.lowest_consistent_map =
+		1 << (p->iommu.page_table_sz_bits - PBM_LOGCLUSTERS);
+
+	for (i = 0; i < PBM_NCLUSTERS; i++) {
+		p->iommu.alloc_info[i].flush = 0;
+		p->iommu.alloc_info[i].next = 0;
+	}
 
 	psycho_write(p->controller_regs + PSYCHO_IOMMU_TSBBASE, __pa(tsbbase));
 
diff --git a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c
index e96af490d..a10f5f072 100644
--- a/arch/sparc64/kernel/pci_sabre.c
+++ b/arch/sparc64/kernel/pci_sabre.c
@@ -1,4 +1,4 @@
-/* $Id: pci_sabre.c,v 1.14 2000/02/18 13:48:55 davem Exp $
+/* $Id: pci_sabre.c,v 1.15 2000/03/10 02:42:16 davem Exp $
  * pci_sabre.c: Sabre specific PCI controller support.
  *
  * Copyright (C) 1997, 1998, 1999 David S. Miller (davem@caipfs.rutgers.edu)
@@ -1128,11 +1128,14 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
 	control |= SABRE_IOMMUCTRL_DENAB;
 	sabre_write(p->controller_regs + SABRE_IOMMU_CONTROL, control);
 
-	for(i = 0; i < 16; i++)
+	for(i = 0; i < 16; i++) {
+		sabre_write(p->controller_regs + SABRE_IOMMU_TAG + (i * 8UL), 0);
 		sabre_write(p->controller_regs + SABRE_IOMMU_DATA + (i * 8UL), 0);
+	}
 
-	control &= ~(SABRE_IOMMUCTRL_DENAB);
-	sabre_write(p->controller_regs + SABRE_IOMMU_CONTROL, control);
+	/* Leave diag mode enabled for full-flushing done
+	 * in pci_iommu.c
+	 */
 
 	tsbbase = __get_free_pages(GFP_KERNEL, order = get_order(tsbsize * 1024 * 8));
 	if (!tsbbase) {
@@ -1144,10 +1147,6 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
 	p->iommu.dma_addr_mask = dma_mask;
 	memset((char *)tsbbase, 0, PAGE_SIZE << order);
 
-	/* Make sure DMA address 0 is never returned just to allow catching
-	   of buggy drivers.  */
-	p->iommu.lowest_free[0] = 1;
-
 	sabre_write(p->controller_regs + SABRE_IOMMU_TSBBASE, __pa(tsbbase));
 
 	control = sabre_read(p->controller_regs + SABRE_IOMMU_CONTROL);
@@ -1168,6 +1167,15 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
 		break;
 	}
 	sabre_write(p->controller_regs + SABRE_IOMMU_CONTROL, control);
+
+	/* We start with no consistent mappings. */
+	p->iommu.lowest_consistent_map =
+		1 << (p->iommu.page_table_sz_bits - PBM_LOGCLUSTERS);
+
+	for (i = 0; i < PBM_NCLUSTERS; i++) {
+		p->iommu.alloc_info[i].flush = 0;
+		p->iommu.alloc_info[i].next = 0;
+	}
 }
 
 static void __init pbm_register_toplevel_resources(struct pci_controller_info *p,
diff --git a/arch/sparc64/kernel/sbus.c b/arch/sparc64/kernel/sbus.c
index 1b454fa2c..c9a0d4a59 100644
--- a/arch/sparc64/kernel/sbus.c
+++ b/arch/sparc64/kernel/sbus.c
@@ -1,4 +1,4 @@
-/* $Id: sbus.c,v 1.9 2000/02/18 13:48:57 davem Exp $
+/* $Id: sbus.c,v 1.10 2000/03/10 07:52:08 davem Exp $
  * sbus.c: UltraSparc SBUS controller support.
  *
  * Copyright (C) 1999 David S. Miller (davem@redhat.com)
@@ -53,13 +53,18 @@ struct sbus_iommu {
 	 * you must increase the size of the type of
 	 * these counters.  You have been duly warned. -DaveM
 	 */
-/*0x30*/u16			lowest_free[NCLUSTERS];
+/*0x30*/struct {
+		u16	next;
+		u16	flush;
+	} alloc_info[NCLUSTERS];
+
+	/* The lowest used consistent mapping entry.  Since
+	 * we allocate consistent maps out of cluster 0 this
+	 * is relative to the beginning of closter 0.
+	 */
+/*0x50*/u32		lowest_consistent_map;
 };
 
-/* Flushing heuristics */
-#define IOMMU_DIAG_LIM	16
-#define STRBUF_DIAG_LIM	32
-
 /* Offsets from iommu_regs */
 #define SYSIO_IOMMUREG_BASE	0x2400UL
 #define IOMMU_CONTROL	(0x2400UL - 0x2400UL)	/* IOMMU control register */
@@ -73,49 +78,29 @@ struct sbus_iommu {
 
 #define IOMMU_DRAM_VALID	(1UL << 30UL)
 
-static void __iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
+static void __iommu_flushall(struct sbus_iommu *iommu)
 {
-	int hit = 0;
-
-	if (npages <= IOMMU_DIAG_LIM) {
-		while (npages--)
-			upa_writeq(base + (npages << PAGE_SHIFT),
-				   iommu->iommu_regs + IOMMU_FLUSH);
-		hit = 1;
-	} else {
-		u32 limit = base + ((npages << PAGE_SHIFT) - 1UL);
-		unsigned long dram = iommu->iommu_regs + IOMMU_DRAMDIAG;
-		unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG;
-		int entry;
+	unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG;
+	int entry;
 
-		for (entry = 0; entry < 16; entry++, dram += 8, tag += 8) {
-			u32 addr = ((u32)upa_readq(tag) << PAGE_SHIFT);
-			if (addr >= base && addr <= limit) {
-				u64 val = upa_readq(dram);
+	for (entry = 0; entry < 16; entry++) {
+		upa_writeq(0, tag);
+		tag += 8UL;
+	}
+	upa_readq(iommu->sbus_control_reg);
 
-				if (val & IOMMU_DRAM_VALID) {
-					upa_writeq(addr,
-						   iommu->iommu_regs + IOMMU_FLUSH);
-					hit = 1;
-				}
-			}
-		}
+	for (entry = 0; entry < NCLUSTERS; entry++) {
+		iommu->alloc_info[entry].flush =
+			iommu->alloc_info[entry].next;
 	}
-	if (hit != 0)
-		upa_readq(iommu->sbus_control_reg);
 }
 
-/* In an effort to keep latency under control, we special
- * case single page IOMMU flushes.
- */
-static __inline__ void iommu_flush(struct sbus_iommu *iommu,
-				   u32 base, unsigned long npages)
+static void iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
 {
-	if (npages == 1) {
-		upa_writeq(base, iommu->iommu_regs + IOMMU_FLUSH);
-		upa_readq(iommu->sbus_control_reg);
-	} else
-		__iommu_flush(iommu, base, npages);
+	while (npages--)
+		upa_writeq(base + (npages << PAGE_SHIFT),
+			   iommu->iommu_regs + IOMMU_FLUSH);
+	upa_readq(iommu->sbus_control_reg);
 }
 
 /* Offsets from strbuf_regs */
@@ -132,65 +117,57 @@ static __inline__ void iommu_flush(struct sbus_iommu *iommu,
 
 static void strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
 {
-	int hit = 0;
-
 	iommu->strbuf_flushflag = 0UL;
-	if (npages <= STRBUF_DIAG_LIM) {
-		while (npages--)
-			upa_writeq(base + (npages << PAGE_SHIFT),
-				   iommu->strbuf_regs + STRBUF_PFLUSH);
-		hit = 1;
-	} else {
-		u32 limit = base + ((npages << PAGE_SHIFT) - 1UL);
-		unsigned long tag = iommu->strbuf_regs + STRBUF_PTAGDIAG;
-		int entry;
-
-		for (entry = 0; entry < 16; entry++, tag += 8) {
-			u64 val = upa_readq(tag);
-
-			if (val & STRBUF_TAG_VALID) {
-				u32 addr = ((u32)(val & ~3UL)) << (PAGE_SHIFT - 2UL);
-				if (addr >= base && addr <= limit) {
-					upa_writeq(addr,
-						   iommu->strbuf_regs + STRBUF_PFLUSH);
-					hit = 1;
-				}
-			}
-		}
-	}
-	if (hit != 0) {
-		/* Whoopee cushion! */
-		upa_writeq(__pa(&iommu->strbuf_flushflag),
-			   iommu->strbuf_regs + STRBUF_FSYNC);
-		upa_readq(iommu->sbus_control_reg);
-		while (iommu->strbuf_flushflag == 0UL)
-			membar("#LoadLoad");
-	}
+	while (npages--)
+		upa_writeq(base + (npages << PAGE_SHIFT),
+			   iommu->strbuf_regs + STRBUF_PFLUSH);
+
+	/* Whoopee cushion! */
+	upa_writeq(__pa(&iommu->strbuf_flushflag),
+		   iommu->strbuf_regs + STRBUF_FSYNC);
+	upa_readq(iommu->sbus_control_reg);
+	while (iommu->strbuf_flushflag == 0UL)
+		membar("#LoadLoad");
 }
 
 static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages)
 {
-	iopte_t *iopte;
-	unsigned long cnum, ent;
+	iopte_t *iopte, *limit;
+	unsigned long cnum, ent, flush_point;
 
 	cnum = 0;
 	while ((1UL << cnum) < npages)
 		cnum++;
 	iopte  = iommu->page_table + (cnum * CLUSTER_NPAGES);
-	iopte += ((ent = iommu->lowest_free[cnum]) << cnum);
-
-	if (iopte_val(iopte[(1UL << cnum)]) == 0UL) {
-		/* Fast path. */
-		iommu->lowest_free[cnum] = ent + 1;
-	} else {
-		unsigned long pte_off = 1;
 
-		ent += 1;
-		do {
-			pte_off++;
-			ent++;
-		} while (iopte_val(iopte[(pte_off << cnum)]) != 0UL);
-		iommu->lowest_free[cnum] = ent;
+	if (cnum == 0)
+		limit = (iommu->page_table +
+			 iommu->lowest_consistent_map);
+	else
+		limit = (iopte + CLUSTER_NPAGES);
+
+	iopte += ((ent = iommu->alloc_info[cnum].next) << cnum);
+	flush_point = iommu->alloc_info[cnum].flush;
+
+	for (;;) {
+		if (iopte_val(*iopte) == 0UL) {
+			if ((iopte + (1 << cnum)) >= limit)
+				ent = 0;
+			else
+				ent = ent + 1;
+			iommu->alloc_info[cnum].next = ent;
+			if (ent == flush_point)
+				__iommu_flushall(iommu);
+			break;
+		}
+		iopte += (1 << cnum);
+		ent++;
+		if (iopte >= limit) {
+			iopte = (iommu->page_table + (cnum * CLUSTER_NPAGES));
+			ent = 0;
+		}
+		if (ent == flush_point)
+			__iommu_flushall(iommu);
 	}
 
 	/* I've got your streaming cluster right here buddy boy... */
@@ -208,8 +185,15 @@ static void free_streaming_cluster(struct sbus_iommu *iommu, u32 base, unsigned
 	ent = (base & CLUSTER_MASK) >> (PAGE_SHIFT + cnum);
 	iopte = iommu->page_table + ((base - MAP_BASE) >> PAGE_SHIFT);
 	iopte_val(*iopte) = 0UL;
-	if (ent < iommu->lowest_free[cnum])
-		iommu->lowest_free[cnum] = ent;
+
+	/* If the global flush might not have caught this entry,
+	 * adjust the flush point such that we will flush before
+	 * ever trying to reuse it.
+	 */
+#define between(X,Y,Z)	(((Z) - (Y)) >= ((X) - (Y)))
+	if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush))
+		iommu->alloc_info[cnum].flush = ent;
+#undef between
 }
 
 /* We allocate consistent mappings from the end of cluster zero. */
@@ -228,8 +212,13 @@ static iopte_t *alloc_consistent_cluster(struct sbus_iommu *iommu, unsigned long
 				if (iopte_val(*iopte) & IOPTE_VALID)
 					break;
 			}
-			if (tmp == 0)
+			if (tmp == 0) {
+				u32 entry = (iopte - iommu->page_table);
+
+				if (entry < iommu->lowest_consistent_map)
+					iommu->lowest_consistent_map = entry;
 				return iopte;
+			}
 		}
 	}
 	return NULL;
@@ -239,6 +228,20 @@ static void free_consistent_cluster(struct sbus_iommu *iommu, u32 base, unsigned
 {
 	iopte_t *iopte = iommu->page_table + ((base - MAP_BASE) >> PAGE_SHIFT);
 
+	if ((iopte - iommu->page_table) == iommu->lowest_consistent_map) {
+		iopte_t *walk = iopte + npages;
+		iopte_t *limit;
+
+		limit = iommu->page_table + CLUSTER_NPAGES;
+		while (walk < limit) {
+			if (iopte_val(*walk) != 0UL)
+				break;
+			walk++;
+		}
+		iommu->lowest_consistent_map =
+			(walk - iommu->page_table);
+	}
+
 	while (npages--)
 		*iopte++ = __iopte(0UL);
 }
@@ -301,6 +304,7 @@ void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_add
 
 	spin_lock_irq(&iommu->lock);
 	free_consistent_cluster(iommu, dvma, npages);
+	iommu_flush(iommu, dvma, npages);
 	spin_unlock_irq(&iommu->lock);
 
 	order = get_order(size);
@@ -337,7 +341,6 @@ dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t size, int di
 		phys_base += PAGE_SIZE;
 	}
 	npages = size >> PAGE_SHIFT;
-	iommu_flush(iommu, dma_base, npages);
 	spin_unlock_irqrestore(&iommu->lock, flags);
 
 	return (dma_base | offset);
@@ -472,7 +475,6 @@ int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int di
 #ifdef VERIFY_SG
 	verify_sglist(sg, nents, iopte, npages);
 #endif
-	iommu_flush(iommu, dma_base, npages);
 	spin_unlock_irqrestore(&iommu->lock, flags);
 
 	return used;
@@ -1061,9 +1063,13 @@ void __init sbus_iommu_init(int prom_node, struct sbus_bus *sbus)
 
 	memset(iommu, 0, sizeof(*iommu));
 
-	/* Make sure DMA address 0 is never returned just to allow catching
-	   of buggy drivers.  */
-	iommu->lowest_free[0] = 1;
+	/* We start with no consistent mappings. */
+	iommu->lowest_consistent_map = CLUSTER_NPAGES;
+
+	for (i = 0; i < NCLUSTERS; i++) {
+		iommu->alloc_info[i].flush = 0;
+		iommu->alloc_info[i].next = 0;
+	}
 
 	/* Setup spinlock. */
 	spin_lock_init(&iommu->lock);
@@ -1110,9 +1116,12 @@ void __init sbus_iommu_init(int prom_node, struct sbus_bus *sbus)
 	 */
 	for (i = 0; i < 16; i++) {
 		unsigned long dram = iommu->iommu_regs + IOMMU_DRAMDIAG;
+		unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG;
 
 		dram += (unsigned long)i * 8UL;
+		tag += (unsigned long)i * 8UL;
 		upa_writeq(0, dram);
+		upa_writeq(0, tag);
 	}
 	upa_readq(iommu->sbus_control_reg);
 
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index 96360b010..b4ee5625e 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -328,6 +328,7 @@ again:
 	stxa	%6, [%0+%8] %3
 	membar	#Sync
 	stxa	%%g0, [%7] %3
+	membar	#Sync
 	mov	0x20, %%g1
 	ldxa	[%%g1] 0x7f, %%g0
 	membar	#Sync"
diff --git a/arch/sparc64/kernel/sys_sparc32.c b/arch/sparc64/kernel/sys_sparc32.c
index d3f02ae54..1fc0b1ba5 100644
--- a/arch/sparc64/kernel/sys_sparc32.c
+++ b/arch/sparc64/kernel/sys_sparc32.c
@@ -1,4 +1,4 @@
-/* $Id: sys_sparc32.c,v 1.133 2000/03/01 02:53:33 davem Exp $
+/* $Id: sys_sparc32.c,v 1.134 2000/03/07 22:27:30 davem Exp $
  * sys_sparc32.c: Conversion between 32bit and 64bit native syscalls.
  *
  * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
diff --git a/arch/sparc64/kernel/sys_sunos32.c b/arch/sparc64/kernel/sys_sunos32.c
index d977c7952..9673cdd36 100644
--- a/arch/sparc64/kernel/sys_sunos32.c
+++ b/arch/sparc64/kernel/sys_sunos32.c
@@ -1,4 +1,4 @@
-/* $Id: sys_sunos32.c,v 1.39 2000/02/16 07:31:37 davem Exp $
+/* $Id: sys_sunos32.c,v 1.40 2000/03/07 22:27:31 davem Exp $
  * sys_sunos32.c: SunOS binary compatability layer on sparc64.
  *
  * Copyright (C) 1995, 1996, 1997 David S. Miller (davem@caip.rutgers.edu)