Parallel initialization: each hub now tries to discover the xbow

connected to it, and sees whether it should become the master of the
attached bridges. Also, maintain information per bridge as to which
nasid/cpu to deliver interrupts to.

3 files changed, 74 insertions, 54 deletions

diff --git a/arch/mips64/sgi-ip27/ip27-init.c b/arch/mips64/sgi-ip27/ip27-init.c
index 6d3e70298..ed7d05c6b 100644
--- a/arch/mips64/sgi-ip27/ip27-init.c
+++ b/arch/mips64/sgi-ip27/ip27-init.c
@@ -283,6 +283,7 @@ void sn_mp_setup(void)
 
 void per_hub_init(cnodeid_t cnode)
 {
+	extern void pcibr_setup(cnodeid_t);
 	cnodemask_t	done;
 
 	spin_lock(&hub_mask_lock);
@@ -299,6 +300,7 @@ void per_hub_init(cnodeid_t cnode)
 	 */
 	if (!done) {
 		hub_rtc_init(cnode);
+		pcibr_setup(cnode); 
 	}
 }
 
@@ -343,6 +345,7 @@ void per_cpu_init(void)
 		set_cp0_status(ST0_KX|ST0_SX|ST0_UX, ST0_KX|ST0_SX|ST0_UX);
 		sti();
 		load_mmu();
+		atomic_inc(&numstarted);
 	}
 	if (is_slave == 0)
 		is_slave = 1;
@@ -391,7 +394,6 @@ void cboot(void)
 #endif
 	_flush_tlb_all();
 	flush_cache_all();
-	atomic_inc(&numstarted); 
 	start_secondary();
 }
 
diff --git a/arch/mips64/sgi-ip27/ip27-irq.c b/arch/mips64/sgi-ip27/ip27-irq.c
index ccff1f09f..7de2b8549 100644
--- a/arch/mips64/sgi-ip27/ip27-irq.c
+++ b/arch/mips64/sgi-ip27/ip27-irq.c
@@ -63,7 +63,7 @@
 irq_cpustat_t irq_stat [NR_CPUS];
 
 extern asmlinkage void ip27_irq(void);
-extern int irq_to_bus[], irq_to_slot[];
+extern int irq_to_bus[], irq_to_slot[], bus_to_cpu[];
 int (*irq_cannonicalize)(int irq);
 int intr_connect_level(cpuid_t cpu, int bit);
 int intr_disconnect_level(cpuid_t cpu, int bit);
@@ -82,10 +82,13 @@ unsigned long spurious_count = 0;
  * use these macros to get the encoded nasid and widget id
  * from the irq value
  */
+#define IRQ_TO_BUS(i)			irq_to_bus[(i)]
+#define IRQ_TO_CPU(i) \
+	((i) == IOC3_ETH_INT ? 0 : bus_to_cpu[IRQ_TO_BUS(i)])
 #define NASID_FROM_PCI_IRQ(i) \
-		(((i) == IOC3_ETH_INT) ? 0 : bus_to_nid[irq_to_bus[(i)]])
+		(((i) == IOC3_ETH_INT) ? 0 : bus_to_nid[IRQ_TO_BUS(i)])
 #define WID_FROM_PCI_IRQ(i) \
-		(((i) == IOC3_ETH_INT) ? 8 : bus_to_wid[irq_to_bus[(i)]])
+		(((i) == IOC3_ETH_INT) ? 8 : bus_to_wid[IRQ_TO_BUS(i)])
 #define	SLOT_FROM_PCI_IRQ(i)		irq_to_slot[i]
 
 void disable_irq(unsigned int irq_nr)
@@ -106,7 +109,7 @@ int get_irq_list(char *buf)
 	int i, len = 0;
 	struct irqaction * action;
 
-	for (i = 0 ; i < 32 ; i++) {
+	for (i = 0 ; i < NR_IRQS ; i++) {
 		action = irq_action[i];
 		if (!action) 
 			continue;
@@ -220,12 +223,12 @@ static unsigned int bridge_startup(unsigned int irq)
 {
         bridge_t *bridge;
         int pin, swlevel;
-	cpuid_t cpu = 0;
+	cpuid_t cpu;
+	nasid_t master = NASID_FROM_PCI_IRQ(irq);
 
-        bridge = (bridge_t *) NODE_SWIN_BASE(NASID_FROM_PCI_IRQ(irq), 
-							WID_FROM_PCI_IRQ(irq));
-
-	pin = SLOT_FROM_PCI_IRQ(irq); 
+        bridge = (bridge_t *) NODE_SWIN_BASE(master, WID_FROM_PCI_IRQ(irq));
+	pin = SLOT_FROM_PCI_IRQ(irq);
+	cpu = IRQ_TO_CPU(irq);
 
 	DBG("bridge_startup(): irq= 0x%x  real_irq= %d pin=%d\n", irq, real_irq, pin);
         /*
@@ -235,7 +238,7 @@ static unsigned int bridge_startup(unsigned int irq)
         swlevel = IRQ_TO_SWLEVEL(cpu, irq);
         intr_connect_level(cpu, swlevel);
 
-        bridge->b_int_addr[pin].addr = 0x20000 | swlevel;
+        bridge->b_int_addr[pin].addr = (0x20000 | swlevel | (master << 8));
         bridge->b_int_enable |= (1 << pin);
 	/* set more stuff in int_enable reg */
 	bridge->b_int_enable |= 0x7ffffe00;
@@ -296,30 +299,6 @@ static unsigned int bridge_shutdown(unsigned int irq)
         return 0;       /* Never anything pending.  */
 }
 
-static void bridge_init(void)
-{
-	bridge_t *bridge;
-        int     bus;
-
-        for (bus=0; bus<num_bridges; bus++) {
-          bridge = (bridge_t *) NODE_SWIN_BASE(bus_to_nid[bus],bus_to_wid[bus]);          
-          /* Hmm...  IRIX sets additional bits in the address which are
-             documented as reserved in the bridge docs ...  */
-          bridge->b_int_mode = 0x0;                     /* Don't clear ints */
-#if 0
-          bridge->b_wid_int_upper = 0x000a8000;           /* Ints to node 0 */
-          bridge->b_wid_int_lower = 0x01000090;
-          bridge->b_dir_map = 0xa00000;                 /* DMA */
-#endif /* shouldn't lower= 0x01800090 ??? */
-          bridge->b_wid_int_upper = 0x000a8000;           /* Ints to widget A */
-          bridge->b_wid_int_lower = 0x01800090;
-          bridge->b_dir_map = 0xa00000;                   /* DMA */
-
-          bridge->b_int_enable = 0;
-          bridge->b_widget.w_tflush;                    /* Flush */
-        }
-}
-
 void irq_debug(void)
 {
 	bridge_t *bridge = (bridge_t *) 0x9200000008000000;
@@ -447,7 +426,6 @@ void __init init_IRQ(void)
 {
 	irq_cannonicalize = indy_irq_cannonicalize;
 
-	bridge_init();
 	set_except_vector(0, ip27_irq);
 }
 
diff --git a/arch/mips64/sgi-ip27/ip27-setup.c b/arch/mips64/sgi-ip27/ip27-setup.c
index 89f58c13d..e76f696e1 100644
--- a/arch/mips64/sgi-ip27/ip27-setup.c
+++ b/arch/mips64/sgi-ip27/ip27-setup.c
@@ -12,6 +12,9 @@
 #include <linux/config.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/sched.h>
+#include <linux/smp.h>
 #include <asm/sn/types.h>
 #include <asm/sn/sn0/addrs.h>
 #include <asm/sn/sn0/hubni.h>
@@ -102,27 +105,32 @@ static void __init verify_mode(void)
 #define XXBOW_WIDGET_PART_NUM   0xd000          /* Xbridge */
 #define BASE_XBOW_PORT  	8     /* Lowest external port */
 
-static void __init pcibr_setup(void)
+unsigned int bus_to_cpu[256];
+
+void __init pcibr_setup(cnodeid_t nid)
 {
-	int 			i;
+	int 			i, start, num, masterwid;
 	bridge_t 		*bridge; 
 	volatile u64 		hubreg;
-	nasid_t	 		nasid;
+	nasid_t	 		nasid, masternasid;
 	xwidget_part_num_t	partnum;
 	widgetreg_t 		widget_id;
+	static spinlock_t	pcibr_setup_lock = SPIN_LOCK_UNLOCKED;
 
-	num_bridges = 0;
 	/*
 	 * find what's on our local node
 	 */
-	nasid = 0;
-	hubreg = LOCAL_HUB_L(IIO_LLP_CSR);
+	spin_lock(&pcibr_setup_lock);
+	start = num_bridges;		/* Remember where we start from */
+	nasid = COMPACT_TO_NASID_NODEID(nid);
+	hubreg = REMOTE_HUB_L(nasid, IIO_LLP_CSR);
 	if (hubreg & IIO_LLP_CSR_IS_UP) {
 		/* link is up */
 		widget_id = *(volatile widgetreg_t *)
                         (RAW_NODE_SWIN_BASE(nasid, 0x0) + WIDGET_ID);
 		partnum = XWIDGET_PART_NUM(widget_id);
-		printk("pcibr_setup(): found partnum= 0x%x ", partnum);
+		printk("Cpu %d, Nasid 0x%lx, pcibr_setup(): found partnum= 0x%x",
+					smp_processor_id(), nasid, partnum);
 		if (partnum == BRIDGE_WIDGET_PART_NUM) {
 			/*
 			 * found direct connected bridge so must be Origin200
@@ -131,8 +139,7 @@ static void __init pcibr_setup(void)
 			num_bridges = 1;
         		bus_to_wid[0] = 0x8;
 			bus_to_nid[0] = 0;
-		}
-		if (partnum == XBOW_WIDGET_PART_NUM) {
+		} else if (partnum == XBOW_WIDGET_PART_NUM) {
 			lboard_t *brd;
 			klxbow_t *xbow_p;
 			/*
@@ -150,6 +157,27 @@ static void __init pcibr_setup(void)
 			     find_component(brd, NULL, KLSTRUCT_XBOW)) == NULL)
 				printk("argh\n");
 			else {
+			   /*
+			    * Okay, here's a xbow. Lets arbitrate and find
+			    * out if we should initialize it. Set hub connected
+			    * at highest or lowest widget as master.
+			    * This algo needs to change a little for headless
+			    * nodes.
+			    */
+#ifdef WIDGET_A
+			   i = HUB_WIDGET_ID_MAX + 1;
+			   do {
+				i--;
+			   } while (!XBOW_PORT_TYPE_HUB(xbow_p, i));
+#else
+			   i = HUB_WIDGET_ID_MIN - 1;
+			   do {
+				i++;
+			   } while (!XBOW_PORT_TYPE_HUB(xbow_p, i));
+#endif
+			   masterwid = i;
+			   masternasid = XBOW_PORT_NASID(xbow_p, i);
+			   if (nasid == masternasid)
 			   for (i=HUB_WIDGET_ID_MIN; i<=HUB_WIDGET_ID_MAX; i++) {
 				if (!XBOW_PORT_IS_ENABLED(xbow_p, i))
 					continue;
@@ -166,8 +194,7 @@ static void __init pcibr_setup(void)
 				}
 			   }
 			}
-		}
-		if (partnum == XXBOW_WIDGET_PART_NUM) {
+		} else if (partnum == XXBOW_WIDGET_PART_NUM) {
 			/*
 			 * found xbridge, assume ibrick for now 
 			 */
@@ -183,13 +210,17 @@ static void __init pcibr_setup(void)
 			num_bridges = 3;
 		}
 	}
+	num = num_bridges - start;
+	spin_unlock(&pcibr_setup_lock);
 	/*
          * set bridge registers
          */
-	for (i=0; i<num_bridges; i++) {
+	for (i = start; i < (start + num); i++) {
+
 		DBG("pcibr_setup: bus= %d  bus_to_wid[%2d]= %d  bus_to_nid[%2d]= %d\n",
                         i, i, bus_to_wid[i], i, bus_to_nid[i]);
 
+		bus_to_cpu[i] = smp_processor_id();
 		/*
 		 * point to this bridge
 		 */
@@ -208,6 +239,19 @@ static void __init pcibr_setup(void)
 		bridge->b_wid_control |= BRIDGE_CTRL_IO_SWAP;
 		bridge->b_wid_control |= BRIDGE_CTRL_MEM_SWAP;
 
+		/*
+		 * Hmm...  IRIX sets additional bits in the address which 
+		 * are documented as reserved in the bridge docs.
+		 * We waste time programming b_wid_int_upper/b_wid_int_lower,
+		 * since bridge_startup will set up the widget->nasid intr
+		 * path anyway.
+		 */
+		bridge->b_int_mode = 0x0;		/* Don't clear ints */
+		bridge->b_wid_int_upper = 0x000a8000;	/* Ints to widget A */
+		bridge->b_wid_int_lower = 0x01800090;
+		bridge->b_dir_map = 0xa00000;		/* DMA */
+		bridge->b_int_enable = 0;
+
 		bridge->b_wid_tflush;     /* wait until Bridge PIO complete */
 	}
 }
@@ -217,11 +261,11 @@ void __init ip27_setup(void)
 	nasid_t nid;
 	hubreg_t p, e;
 
+	num_bridges = 0;
 	/*
 	 * hub_rtc init and cpu clock intr enabled for later calibrate_delay.
 	 */
 	DBG("ip27_setup(): Entered.\n");
-	per_cpu_init();
 	nid = get_nasid();
 	printk("IP27: Running on node %d.\n", nid);
 
@@ -240,9 +284,5 @@ void __init ip27_setup(void)
 	verify_mode();
 	ioc3_sio_init();
 	ioc3_eth_init();
-
-	DBG("ip27_setup(): calling pcibr_setup\n");
-	/* set some bridge registers */
-	pcibr_setup();
-	DBG("ip27_setup(): Exit.\n");
+	per_cpu_init();
 }