Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
qib_iba6120.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
3  * All rights reserved.
4  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses. You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  * Redistribution and use in source and binary forms, with or
13  * without modification, are permitted provided that the following
14  * conditions are met:
15  *
16  * - Redistributions of source code must retain the above
17  * copyright notice, this list of conditions and the following
18  * disclaimer.
19  *
20  * - Redistributions in binary form must reproduce the above
21  * copyright notice, this list of conditions and the following
22  * disclaimer in the documentation and/or other materials
23  * provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 /*
35  * This file contains all of the code that is specific to the
36  * QLogic_IB 6120 PCIe chip.
37  */
38 
39 #include <linux/interrupt.h>
40 #include <linux/pci.h>
41 #include <linux/delay.h>
42 #include <rdma/ib_verbs.h>
43 
44 #include "qib.h"
45 #include "qib_6120_regs.h"
46 
47 static void qib_6120_setup_setextled(struct qib_pportdata *, u32);
48 static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op);
49 static u8 qib_6120_phys_portstate(u64);
50 static u32 qib_6120_iblink_state(u64);
51 
52 /*
53  * This file contains all the chip-specific register information and
54  * access functions for the QLogic QLogic_IB PCI-Express chip.
55  *
56  */
57 
58 /* KREG_IDX uses machine-generated #defines */
59 #define KREG_IDX(regname) (QIB_6120_##regname##_OFFS / sizeof(u64))
60 
61 /* Use defines to tie machine-generated names to lower-case names */
62 #define kr_extctrl KREG_IDX(EXTCtrl)
63 #define kr_extstatus KREG_IDX(EXTStatus)
64 #define kr_gpio_clear KREG_IDX(GPIOClear)
65 #define kr_gpio_mask KREG_IDX(GPIOMask)
66 #define kr_gpio_out KREG_IDX(GPIOOut)
67 #define kr_gpio_status KREG_IDX(GPIOStatus)
68 #define kr_rcvctrl KREG_IDX(RcvCtrl)
69 #define kr_sendctrl KREG_IDX(SendCtrl)
70 #define kr_partitionkey KREG_IDX(RcvPartitionKey)
71 #define kr_hwdiagctrl KREG_IDX(HwDiagCtrl)
72 #define kr_ibcstatus KREG_IDX(IBCStatus)
73 #define kr_ibcctrl KREG_IDX(IBCCtrl)
74 #define kr_sendbuffererror KREG_IDX(SendBufErr0)
75 #define kr_rcvbthqp KREG_IDX(RcvBTHQP)
76 #define kr_counterregbase KREG_IDX(CntrRegBase)
77 #define kr_palign KREG_IDX(PageAlign)
78 #define kr_rcvegrbase KREG_IDX(RcvEgrBase)
79 #define kr_rcvegrcnt KREG_IDX(RcvEgrCnt)
80 #define kr_rcvhdrcnt KREG_IDX(RcvHdrCnt)
81 #define kr_rcvhdrentsize KREG_IDX(RcvHdrEntSize)
82 #define kr_rcvhdrsize KREG_IDX(RcvHdrSize)
83 #define kr_rcvtidbase KREG_IDX(RcvTIDBase)
84 #define kr_rcvtidcnt KREG_IDX(RcvTIDCnt)
85 #define kr_scratch KREG_IDX(Scratch)
86 #define kr_sendctrl KREG_IDX(SendCtrl)
87 #define kr_sendpioavailaddr KREG_IDX(SendPIOAvailAddr)
88 #define kr_sendpiobufbase KREG_IDX(SendPIOBufBase)
89 #define kr_sendpiobufcnt KREG_IDX(SendPIOBufCnt)
90 #define kr_sendpiosize KREG_IDX(SendPIOSize)
91 #define kr_sendregbase KREG_IDX(SendRegBase)
92 #define kr_userregbase KREG_IDX(UserRegBase)
93 #define kr_control KREG_IDX(Control)
94 #define kr_intclear KREG_IDX(IntClear)
95 #define kr_intmask KREG_IDX(IntMask)
96 #define kr_intstatus KREG_IDX(IntStatus)
97 #define kr_errclear KREG_IDX(ErrClear)
98 #define kr_errmask KREG_IDX(ErrMask)
99 #define kr_errstatus KREG_IDX(ErrStatus)
100 #define kr_hwerrclear KREG_IDX(HwErrClear)
101 #define kr_hwerrmask KREG_IDX(HwErrMask)
102 #define kr_hwerrstatus KREG_IDX(HwErrStatus)
103 #define kr_revision KREG_IDX(Revision)
104 #define kr_portcnt KREG_IDX(PortCnt)
105 #define kr_serdes_cfg0 KREG_IDX(SerdesCfg0)
106 #define kr_serdes_cfg1 (kr_serdes_cfg0 + 1)
107 #define kr_serdes_stat KREG_IDX(SerdesStat)
108 #define kr_xgxs_cfg KREG_IDX(XGXSCfg)
109 
110 /* These must only be written via qib_write_kreg_ctxt() */
111 #define kr_rcvhdraddr KREG_IDX(RcvHdrAddr0)
112 #define kr_rcvhdrtailaddr KREG_IDX(RcvHdrTailAddr0)
113 
114 #define CREG_IDX(regname) ((QIB_6120_##regname##_OFFS - \
115  QIB_6120_LBIntCnt_OFFS) / sizeof(u64))
116 
117 #define cr_badformat CREG_IDX(RxBadFormatCnt)
118 #define cr_erricrc CREG_IDX(RxICRCErrCnt)
119 #define cr_errlink CREG_IDX(RxLinkProblemCnt)
120 #define cr_errlpcrc CREG_IDX(RxLPCRCErrCnt)
121 #define cr_errpkey CREG_IDX(RxPKeyMismatchCnt)
122 #define cr_rcvflowctrl_err CREG_IDX(RxFlowCtrlErrCnt)
123 #define cr_err_rlen CREG_IDX(RxLenErrCnt)
124 #define cr_errslen CREG_IDX(TxLenErrCnt)
125 #define cr_errtidfull CREG_IDX(RxTIDFullErrCnt)
126 #define cr_errtidvalid CREG_IDX(RxTIDValidErrCnt)
127 #define cr_errvcrc CREG_IDX(RxVCRCErrCnt)
128 #define cr_ibstatuschange CREG_IDX(IBStatusChangeCnt)
129 #define cr_lbint CREG_IDX(LBIntCnt)
130 #define cr_invalidrlen CREG_IDX(RxMaxMinLenErrCnt)
131 #define cr_invalidslen CREG_IDX(TxMaxMinLenErrCnt)
132 #define cr_lbflowstall CREG_IDX(LBFlowStallCnt)
133 #define cr_pktrcv CREG_IDX(RxDataPktCnt)
134 #define cr_pktrcvflowctrl CREG_IDX(RxFlowPktCnt)
135 #define cr_pktsend CREG_IDX(TxDataPktCnt)
136 #define cr_pktsendflow CREG_IDX(TxFlowPktCnt)
137 #define cr_portovfl CREG_IDX(RxP0HdrEgrOvflCnt)
138 #define cr_rcvebp CREG_IDX(RxEBPCnt)
139 #define cr_rcvovfl CREG_IDX(RxBufOvflCnt)
140 #define cr_senddropped CREG_IDX(TxDroppedPktCnt)
141 #define cr_sendstall CREG_IDX(TxFlowStallCnt)
142 #define cr_sendunderrun CREG_IDX(TxUnderrunCnt)
143 #define cr_wordrcv CREG_IDX(RxDwordCnt)
144 #define cr_wordsend CREG_IDX(TxDwordCnt)
145 #define cr_txunsupvl CREG_IDX(TxUnsupVLErrCnt)
146 #define cr_rxdroppkt CREG_IDX(RxDroppedPktCnt)
147 #define cr_iblinkerrrecov CREG_IDX(IBLinkErrRecoveryCnt)
148 #define cr_iblinkdown CREG_IDX(IBLinkDownedCnt)
149 #define cr_ibsymbolerr CREG_IDX(IBSymbolErrCnt)
150 
151 #define SYM_RMASK(regname, fldname) ((u64) \
152  QIB_6120_##regname##_##fldname##_RMASK)
153 #define SYM_MASK(regname, fldname) ((u64) \
154  QIB_6120_##regname##_##fldname##_RMASK << \
155  QIB_6120_##regname##_##fldname##_LSB)
156 #define SYM_LSB(regname, fldname) (QIB_6120_##regname##_##fldname##_LSB)
157 
158 #define SYM_FIELD(value, regname, fldname) ((u64) \
159  (((value) >> SYM_LSB(regname, fldname)) & \
160  SYM_RMASK(regname, fldname)))
161 #define ERR_MASK(fldname) SYM_MASK(ErrMask, fldname##Mask)
162 #define HWE_MASK(fldname) SYM_MASK(HwErrMask, fldname##Mask)
163 
164 /* link training states, from IBC */
165 #define IB_6120_LT_STATE_DISABLED 0x00
166 #define IB_6120_LT_STATE_LINKUP 0x01
167 #define IB_6120_LT_STATE_POLLACTIVE 0x02
168 #define IB_6120_LT_STATE_POLLQUIET 0x03
169 #define IB_6120_LT_STATE_SLEEPDELAY 0x04
170 #define IB_6120_LT_STATE_SLEEPQUIET 0x05
171 #define IB_6120_LT_STATE_CFGDEBOUNCE 0x08
172 #define IB_6120_LT_STATE_CFGRCVFCFG 0x09
173 #define IB_6120_LT_STATE_CFGWAITRMT 0x0a
174 #define IB_6120_LT_STATE_CFGIDLE 0x0b
175 #define IB_6120_LT_STATE_RECOVERRETRAIN 0x0c
176 #define IB_6120_LT_STATE_RECOVERWAITRMT 0x0e
177 #define IB_6120_LT_STATE_RECOVERIDLE 0x0f
178 
179 /* link state machine states from IBC */
180 #define IB_6120_L_STATE_DOWN 0x0
181 #define IB_6120_L_STATE_INIT 0x1
182 #define IB_6120_L_STATE_ARM 0x2
183 #define IB_6120_L_STATE_ACTIVE 0x3
184 #define IB_6120_L_STATE_ACT_DEFER 0x4
185 
186 static const u8 qib_6120_physportstate[0x20] = {
214 };
215 
216 
217 struct qib_chip_specific {
219  u64 *cntrs;
220  u64 *portcntrs;
221  void *dummy_hdrq; /* used after ctxt close */
223  spinlock_t kernel_tid_lock; /* no back to back kernel TID writes */
224  spinlock_t user_tid_lock; /* no back to back user TID writes */
225  spinlock_t rcvmod_lock; /* protect rcvctrl shadow changes */
226  spinlock_t gpio_lock; /* RMW of shadows/regs for ExtCtrl and GPIO */
227  u64 hwerrmask;
228  u64 errormask;
229  u64 gpio_out; /* shadow of kr_gpio_out, for rmw ops */
230  u64 gpio_mask; /* shadow the gpio mask register */
231  u64 extctrl; /* shadow the gpio output enable, etc... */
232  /*
233  * these 5 fields are used to establish deltas for IB symbol
234  * errors and linkrecovery errors. They can be reported on
235  * some chips during link negotiation prior to INIT, and with
236  * DDR when faking DDR negotiations with non-IBTA switches.
237  * The chip counters are adjusted at driver unload if there is
238  * a non-zero delta.
239  */
245  u64 ibcctrl; /* shadow for kr_ibcctrl */
246  u32 lastlinkrecov; /* link recovery issue */
247  int irq;
250  u32 ncntrs;
251  u32 nportcntrs;
252  /* used with gpio interrupts to implement IB counters */
255  /*
256  * these count only cases where _successive_ LocalLinkIntegrity
257  * errors were seen in the receive headers of IB standard packets
258  */
262  u64 sword; /* total dwords sent (sample result) */
263  u64 rword; /* total dwords received (sample result) */
264  u64 spkts; /* total packets sent (sample result) */
265  u64 rpkts; /* total packets received (sample result) */
266  u64 xmit_wait; /* # of ticks no data sent (sample result) */
268  char emsgbuf[128];
269  char bitsmsgbuf[64];
271 };
272 
273 /* ibcctrl bits */
274 #define QLOGIC_IB_IBCC_LINKINITCMD_DISABLE 1
275 /* cycle through TS1/TS2 till OK */
276 #define QLOGIC_IB_IBCC_LINKINITCMD_POLL 2
277 /* wait for TS1, then go on */
278 #define QLOGIC_IB_IBCC_LINKINITCMD_SLEEP 3
279 #define QLOGIC_IB_IBCC_LINKINITCMD_SHIFT 16
280 
281 #define QLOGIC_IB_IBCC_LINKCMD_DOWN 1 /* move to 0x11 */
282 #define QLOGIC_IB_IBCC_LINKCMD_ARMED 2 /* move to 0x21 */
283 #define QLOGIC_IB_IBCC_LINKCMD_ACTIVE 3 /* move to 0x31 */
284 #define QLOGIC_IB_IBCC_LINKCMD_SHIFT 18
285 
286 /*
287  * We could have a single register get/put routine, that takes a group type,
288  * but this is somewhat clearer and cleaner. It also gives us some error
289  * checking. 64 bit register reads should always work, but are inefficient
290  * on opteron (the northbridge always generates 2 separate HT 32 bit reads),
291  * so we use kreg32 wherever possible. User register and counter register
292  * reads are always 32 bit reads, so only one form of those routines.
293  */
294 
305 static inline u32 qib_read_ureg32(const struct qib_devdata *dd,
306  enum qib_ureg regno, int ctxt)
307 {
308  if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
309  return 0;
310 
311  if (dd->userbase)
312  return readl(regno + (u64 __iomem *)
313  ((char __iomem *)dd->userbase +
314  dd->ureg_align * ctxt));
315  else
316  return readl(regno + (u64 __iomem *)
317  (dd->uregbase +
318  (char __iomem *)dd->kregbase +
319  dd->ureg_align * ctxt));
320 }
321 
331 static inline void qib_write_ureg(const struct qib_devdata *dd,
332  enum qib_ureg regno, u64 value, int ctxt)
333 {
334  u64 __iomem *ubase;
335  if (dd->userbase)
336  ubase = (u64 __iomem *)
337  ((char __iomem *) dd->userbase +
338  dd->ureg_align * ctxt);
339  else
340  ubase = (u64 __iomem *)
341  (dd->uregbase +
342  (char __iomem *) dd->kregbase +
343  dd->ureg_align * ctxt);
344 
345  if (dd->kregbase && (dd->flags & QIB_PRESENT))
346  writeq(value, &ubase[regno]);
347 }
348 
349 static inline u32 qib_read_kreg32(const struct qib_devdata *dd,
350  const u16 regno)
351 {
352  if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
353  return -1;
354  return readl((u32 __iomem *)&dd->kregbase[regno]);
355 }
356 
357 static inline u64 qib_read_kreg64(const struct qib_devdata *dd,
358  const u16 regno)
359 {
360  if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
361  return -1;
362 
363  return readq(&dd->kregbase[regno]);
364 }
365 
366 static inline void qib_write_kreg(const struct qib_devdata *dd,
367  const u16 regno, u64 value)
368 {
369  if (dd->kregbase && (dd->flags & QIB_PRESENT))
370  writeq(value, &dd->kregbase[regno]);
371 }
372 
380 static inline void qib_write_kreg_ctxt(const struct qib_devdata *dd,
381  const u16 regno, unsigned ctxt,
382  u64 value)
383 {
384  qib_write_kreg(dd, regno + ctxt, value);
385 }
386 
387 static inline void write_6120_creg(const struct qib_devdata *dd,
388  u16 regno, u64 value)
389 {
390  if (dd->cspec->cregbase && (dd->flags & QIB_PRESENT))
391  writeq(value, &dd->cspec->cregbase[regno]);
392 }
393 
394 static inline u64 read_6120_creg(const struct qib_devdata *dd, u16 regno)
395 {
396  if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
397  return 0;
398  return readq(&dd->cspec->cregbase[regno]);
399 }
400 
401 static inline u32 read_6120_creg32(const struct qib_devdata *dd, u16 regno)
402 {
403  if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
404  return 0;
405  return readl(&dd->cspec->cregbase[regno]);
406 }
407 
408 /* kr_control bits */
409 #define QLOGIC_IB_C_RESET 1U
410 
411 /* kr_intstatus, kr_intclear, kr_intmask bits */
412 #define QLOGIC_IB_I_RCVURG_MASK ((1U << 5) - 1)
413 #define QLOGIC_IB_I_RCVURG_SHIFT 0
414 #define QLOGIC_IB_I_RCVAVAIL_MASK ((1U << 5) - 1)
415 #define QLOGIC_IB_I_RCVAVAIL_SHIFT 12
416 
417 #define QLOGIC_IB_C_FREEZEMODE 0x00000002
418 #define QLOGIC_IB_C_LINKENABLE 0x00000004
419 #define QLOGIC_IB_I_ERROR 0x0000000080000000ULL
420 #define QLOGIC_IB_I_SPIOSENT 0x0000000040000000ULL
421 #define QLOGIC_IB_I_SPIOBUFAVAIL 0x0000000020000000ULL
422 #define QLOGIC_IB_I_GPIO 0x0000000010000000ULL
423 #define QLOGIC_IB_I_BITSEXTANT \
424  ((QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT) | \
425  (QLOGIC_IB_I_RCVAVAIL_MASK << \
426  QLOGIC_IB_I_RCVAVAIL_SHIFT) | \
427  QLOGIC_IB_I_ERROR | QLOGIC_IB_I_SPIOSENT | \
428  QLOGIC_IB_I_SPIOBUFAVAIL | QLOGIC_IB_I_GPIO)
429 
430 /* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
431 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK 0x000000000000003fULL
432 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT 0
433 #define QLOGIC_IB_HWE_PCIEPOISONEDTLP 0x0000000010000000ULL
434 #define QLOGIC_IB_HWE_PCIECPLTIMEOUT 0x0000000020000000ULL
435 #define QLOGIC_IB_HWE_PCIEBUSPARITYXTLH 0x0000000040000000ULL
436 #define QLOGIC_IB_HWE_PCIEBUSPARITYXADM 0x0000000080000000ULL
437 #define QLOGIC_IB_HWE_PCIEBUSPARITYRADM 0x0000000100000000ULL
438 #define QLOGIC_IB_HWE_COREPLL_FBSLIP 0x0080000000000000ULL
439 #define QLOGIC_IB_HWE_COREPLL_RFSLIP 0x0100000000000000ULL
440 #define QLOGIC_IB_HWE_PCIE1PLLFAILED 0x0400000000000000ULL
441 #define QLOGIC_IB_HWE_PCIE0PLLFAILED 0x0800000000000000ULL
442 #define QLOGIC_IB_HWE_SERDESPLLFAILED 0x1000000000000000ULL
443 
444 
445 /* kr_extstatus bits */
446 #define QLOGIC_IB_EXTS_FREQSEL 0x2
447 #define QLOGIC_IB_EXTS_SERDESSEL 0x4
448 #define QLOGIC_IB_EXTS_MEMBIST_ENDTEST 0x0000000000004000
449 #define QLOGIC_IB_EXTS_MEMBIST_FOUND 0x0000000000008000
450 
451 /* kr_xgxsconfig bits */
452 #define QLOGIC_IB_XGXS_RESET 0x5ULL
453 
454 #define _QIB_GPIO_SDA_NUM 1
455 #define _QIB_GPIO_SCL_NUM 0
456 
457 /* Bits in GPIO for the added IB link interrupts */
458 #define GPIO_RXUVL_BIT 3
459 #define GPIO_OVRUN_BIT 4
460 #define GPIO_LLI_BIT 5
461 #define GPIO_ERRINTR_MASK 0x38
462 
463 
464 #define QLOGIC_IB_RT_BUFSIZE_MASK 0xe0000000ULL
465 #define QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid) \
466  ((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) >> 29) + 11 - 1)
467 #define QLOGIC_IB_RT_BUFSIZE(tid) (1 << QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid))
468 #define QLOGIC_IB_RT_IS_VALID(tid) \
469  (((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) && \
470  ((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) != QLOGIC_IB_RT_BUFSIZE_MASK)))
471 #define QLOGIC_IB_RT_ADDR_MASK 0x1FFFFFFFULL /* 29 bits valid */
472 #define QLOGIC_IB_RT_ADDR_SHIFT 10
473 
474 #define QLOGIC_IB_R_INTRAVAIL_SHIFT 16
475 #define QLOGIC_IB_R_TAILUPD_SHIFT 31
476 #define IBA6120_R_PKEY_DIS_SHIFT 30
477 
478 #define PBC_6120_VL15_SEND_CTRL (1ULL << 31) /* pbc; VL15; link_buf only */
479 
480 #define IBCBUSFRSPCPARITYERR HWE_MASK(IBCBusFromSPCParityErr)
481 #define IBCBUSTOSPCPARITYERR HWE_MASK(IBCBusToSPCParityErr)
482 
483 #define SYM_MASK_BIT(regname, fldname, bit) ((u64) \
484  ((1ULL << (SYM_LSB(regname, fldname) + (bit)))))
485 
486 #define TXEMEMPARITYERR_PIOBUF \
487  SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 0)
488 #define TXEMEMPARITYERR_PIOPBC \
489  SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 1)
490 #define TXEMEMPARITYERR_PIOLAUNCHFIFO \
491  SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 2)
492 
493 #define RXEMEMPARITYERR_RCVBUF \
494  SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 0)
495 #define RXEMEMPARITYERR_LOOKUPQ \
496  SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 1)
497 #define RXEMEMPARITYERR_EXPTID \
498  SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 2)
499 #define RXEMEMPARITYERR_EAGERTID \
500  SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 3)
501 #define RXEMEMPARITYERR_FLAGBUF \
502  SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 4)
503 #define RXEMEMPARITYERR_DATAINFO \
504  SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 5)
505 #define RXEMEMPARITYERR_HDRINFO \
506  SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 6)
507 
508 /* 6120 specific hardware errors... */
509 static const struct qib_hwerror_msgs qib_6120_hwerror_msgs[] = {
510  /* generic hardware errors */
511  QLOGIC_IB_HWE_MSG(IBCBUSFRSPCPARITYERR, "QIB2IB Parity"),
512  QLOGIC_IB_HWE_MSG(IBCBUSTOSPCPARITYERR, "IB2QIB Parity"),
513 
515  "TXE PIOBUF Memory Parity"),
517  "TXE PIOPBC Memory Parity"),
519  "TXE PIOLAUNCHFIFO Memory Parity"),
520 
522  "RXE RCVBUF Memory Parity"),
524  "RXE LOOKUPQ Memory Parity"),
526  "RXE EAGERTID Memory Parity"),
528  "RXE EXPTID Memory Parity"),
530  "RXE FLAGBUF Memory Parity"),
532  "RXE DATAINFO Memory Parity"),
534  "RXE HDRINFO Memory Parity"),
535 
536  /* chip-specific hardware errors */
538  "PCIe Poisoned TLP"),
540  "PCIe completion timeout"),
541  /*
542  * In practice, it's unlikely wthat we'll see PCIe PLL, or bus
543  * parity or memory parity error failures, because most likely we
544  * won't be able to talk to the core of the chip. Nonetheless, we
545  * might see them, if they are in parts of the PCIe core that aren't
546  * essential.
547  */
549  "PCIePLL1"),
551  "PCIePLL0"),
553  "PCIe XTLH core parity"),
555  "PCIe ADM TX core parity"),
557  "PCIe ADM RX core parity"),
559  "SerDes PLL"),
560 };
561 
562 #define TXE_PIO_PARITY (TXEMEMPARITYERR_PIOBUF | TXEMEMPARITYERR_PIOPBC)
563 #define _QIB_PLL_FAIL (QLOGIC_IB_HWE_COREPLL_FBSLIP | \
564  QLOGIC_IB_HWE_COREPLL_RFSLIP)
565 
566  /* variables for sanity checking interrupt and errors */
567 #define IB_HWE_BITSEXTANT \
568  (HWE_MASK(RXEMemParityErr) | \
569  HWE_MASK(TXEMemParityErr) | \
570  (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK << \
571  QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) | \
572  QLOGIC_IB_HWE_PCIE1PLLFAILED | \
573  QLOGIC_IB_HWE_PCIE0PLLFAILED | \
574  QLOGIC_IB_HWE_PCIEPOISONEDTLP | \
575  QLOGIC_IB_HWE_PCIECPLTIMEOUT | \
576  QLOGIC_IB_HWE_PCIEBUSPARITYXTLH | \
577  QLOGIC_IB_HWE_PCIEBUSPARITYXADM | \
578  QLOGIC_IB_HWE_PCIEBUSPARITYRADM | \
579  HWE_MASK(PowerOnBISTFailed) | \
580  QLOGIC_IB_HWE_COREPLL_FBSLIP | \
581  QLOGIC_IB_HWE_COREPLL_RFSLIP | \
582  QLOGIC_IB_HWE_SERDESPLLFAILED | \
583  HWE_MASK(IBCBusToSPCParityErr) | \
584  HWE_MASK(IBCBusFromSPCParityErr))
585 
586 #define IB_E_BITSEXTANT \
587  (ERR_MASK(RcvFormatErr) | ERR_MASK(RcvVCRCErr) | \
588  ERR_MASK(RcvICRCErr) | ERR_MASK(RcvMinPktLenErr) | \
589  ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvLongPktLenErr) | \
590  ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvUnexpectedCharErr) | \
591  ERR_MASK(RcvUnsupportedVLErr) | ERR_MASK(RcvEBPErr) | \
592  ERR_MASK(RcvIBFlowErr) | ERR_MASK(RcvBadVersionErr) | \
593  ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr) | \
594  ERR_MASK(RcvBadTidErr) | ERR_MASK(RcvHdrLenErr) | \
595  ERR_MASK(RcvHdrErr) | ERR_MASK(RcvIBLostLinkErr) | \
596  ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendMaxPktLenErr) | \
597  ERR_MASK(SendUnderRunErr) | ERR_MASK(SendPktLenErr) | \
598  ERR_MASK(SendDroppedSmpPktErr) | \
599  ERR_MASK(SendDroppedDataPktErr) | \
600  ERR_MASK(SendPioArmLaunchErr) | \
601  ERR_MASK(SendUnexpectedPktNumErr) | \
602  ERR_MASK(SendUnsupportedVLErr) | ERR_MASK(IBStatusChanged) | \
603  ERR_MASK(InvalidAddrErr) | ERR_MASK(ResetNegated) | \
604  ERR_MASK(HardwareErr))
605 
606 #define QLOGIC_IB_E_PKTERRS ( \
607  ERR_MASK(SendPktLenErr) | \
608  ERR_MASK(SendDroppedDataPktErr) | \
609  ERR_MASK(RcvVCRCErr) | \
610  ERR_MASK(RcvICRCErr) | \
611  ERR_MASK(RcvShortPktLenErr) | \
612  ERR_MASK(RcvEBPErr))
613 
614 /* These are all rcv-related errors which we want to count for stats */
615 #define E_SUM_PKTERRS \
616  (ERR_MASK(RcvHdrLenErr) | ERR_MASK(RcvBadTidErr) | \
617  ERR_MASK(RcvBadVersionErr) | ERR_MASK(RcvHdrErr) | \
618  ERR_MASK(RcvLongPktLenErr) | ERR_MASK(RcvShortPktLenErr) | \
619  ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvMinPktLenErr) | \
620  ERR_MASK(RcvFormatErr) | ERR_MASK(RcvUnsupportedVLErr) | \
621  ERR_MASK(RcvUnexpectedCharErr) | ERR_MASK(RcvEBPErr))
622 
623 /* These are all send-related errors which we want to count for stats */
624 #define E_SUM_ERRS \
625  (ERR_MASK(SendPioArmLaunchErr) | \
626  ERR_MASK(SendUnexpectedPktNumErr) | \
627  ERR_MASK(SendDroppedDataPktErr) | \
628  ERR_MASK(SendDroppedSmpPktErr) | \
629  ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendUnsupportedVLErr) | \
630  ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) | \
631  ERR_MASK(InvalidAddrErr))
632 
633 /*
634  * this is similar to E_SUM_ERRS, but can't ignore armlaunch, don't ignore
635  * errors not related to freeze and cancelling buffers. Can't ignore
636  * armlaunch because could get more while still cleaning up, and need
637  * to cancel those as they happen.
638  */
639 #define E_SPKT_ERRS_IGNORE \
640  (ERR_MASK(SendDroppedDataPktErr) | \
641  ERR_MASK(SendDroppedSmpPktErr) | \
642  ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendMinPktLenErr) | \
643  ERR_MASK(SendPktLenErr))
644 
645 /*
646  * these are errors that can occur when the link changes state while
647  * a packet is being sent or received. This doesn't cover things
648  * like EBP or VCRC that can be the result of a sending having the
649  * link change state, so we receive a "known bad" packet.
650  */
651 #define E_SUM_LINK_PKTERRS \
652  (ERR_MASK(SendDroppedDataPktErr) | \
653  ERR_MASK(SendDroppedSmpPktErr) | \
654  ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) | \
655  ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvMinPktLenErr) | \
656  ERR_MASK(RcvUnexpectedCharErr))
657 
658 static void qib_6120_put_tid_2(struct qib_devdata *, u64 __iomem *,
659  u32, unsigned long);
660 
661 /*
662  * On platforms using this chip, and not having ordered WC stores, we
663  * can get TXE parity errors due to speculative reads to the PIO buffers,
664  * and this, due to a chip issue can result in (many) false parity error
665  * reports. So it's a debug print on those, and an info print on systems
666  * where the speculative reads don't occur.
667  */
668 static void qib_6120_txe_recover(struct qib_devdata *dd)
669 {
670  if (!qib_unordered_wc())
671  qib_devinfo(dd->pcidev,
672  "Recovering from TXE PIO parity error\n");
673 }
674 
675 /* enable/disable chip from delivering interrupts */
676 static void qib_6120_set_intr_state(struct qib_devdata *dd, u32 enable)
677 {
678  if (enable) {
679  if (dd->flags & QIB_BADINTR)
680  return;
681  qib_write_kreg(dd, kr_intmask, ~0ULL);
682  /* force re-interrupt of any pending interrupts. */
683  qib_write_kreg(dd, kr_intclear, 0ULL);
684  } else
685  qib_write_kreg(dd, kr_intmask, 0ULL);
686 }
687 
688 /*
689  * Try to cleanup as much as possible for anything that might have gone
690  * wrong while in freeze mode, such as pio buffers being written by user
691  * processes (causing armlaunch), send errors due to going into freeze mode,
692  * etc., and try to avoid causing extra interrupts while doing so.
693  * Forcibly update the in-memory pioavail register copies after cleanup
694  * because the chip won't do it while in freeze mode (the register values
695  * themselves are kept correct).
696  * Make sure that we don't lose any important interrupts by using the chip
697  * feature that says that writing 0 to a bit in *clear that is set in
698  * *status will cause an interrupt to be generated again (if allowed by
699  * the *mask value).
700  * This is in chip-specific code because of all of the register accesses,
701  * even though the details are similar on most chips
702  */
703 static void qib_6120_clear_freeze(struct qib_devdata *dd)
704 {
705  /* disable error interrupts, to avoid confusion */
706  qib_write_kreg(dd, kr_errmask, 0ULL);
707 
708  /* also disable interrupts; errormask is sometimes overwriten */
709  qib_6120_set_intr_state(dd, 0);
710 
711  qib_cancel_sends(dd->pport);
712 
713  /* clear the freeze, and be sure chip saw it */
714  qib_write_kreg(dd, kr_control, dd->control);
715  qib_read_kreg32(dd, kr_scratch);
716 
717  /* force in-memory update now we are out of freeze */
719 
720  /*
721  * force new interrupt if any hwerr, error or interrupt bits are
722  * still set, and clear "safe" send packet errors related to freeze
723  * and cancelling sends. Re-enable error interrupts before possible
724  * force of re-interrupt on pending interrupts.
725  */
726  qib_write_kreg(dd, kr_hwerrclear, 0ULL);
727  qib_write_kreg(dd, kr_errclear, E_SPKT_ERRS_IGNORE);
728  qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
729  qib_6120_set_intr_state(dd, 1);
730 }
731 
743 static void qib_handle_6120_hwerrors(struct qib_devdata *dd, char *msg,
744  size_t msgl)
745 {
746  u64 hwerrs;
747  u32 bits, ctrl;
748  int isfatal = 0;
749  char *bitsmsg;
750  int log_idx;
751 
752  hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
753  if (!hwerrs)
754  return;
755  if (hwerrs == ~0ULL) {
756  qib_dev_err(dd,
757  "Read of hardware error status failed (all bits set); ignoring\n");
758  return;
759  }
760  qib_stats.sps_hwerrs++;
761 
762  /* Always clear the error status register, except MEMBISTFAIL,
763  * regardless of whether we continue or stop using the chip.
764  * We want that set so we know it failed, even across driver reload.
765  * We'll still ignore it in the hwerrmask. We do this partly for
766  * diagnostics, but also for support */
767  qib_write_kreg(dd, kr_hwerrclear,
768  hwerrs & ~HWE_MASK(PowerOnBISTFailed));
769 
770  hwerrs &= dd->cspec->hwerrmask;
771 
772  /* We log some errors to EEPROM, check if we have any of those. */
773  for (log_idx = 0; log_idx < QIB_EEP_LOG_CNT; ++log_idx)
774  if (hwerrs & dd->eep_st_masks[log_idx].hwerrs_to_log)
775  qib_inc_eeprom_err(dd, log_idx, 1);
776 
777  /*
778  * Make sure we get this much out, unless told to be quiet,
779  * or it's occurred within the last 5 seconds.
780  */
781  if (hwerrs & ~(TXE_PIO_PARITY | RXEMEMPARITYERR_EAGERTID))
782  qib_devinfo(dd->pcidev,
783  "Hardware error: hwerr=0x%llx (cleared)\n",
784  (unsigned long long) hwerrs);
785 
786  if (hwerrs & ~IB_HWE_BITSEXTANT)
787  qib_dev_err(dd,
788  "hwerror interrupt with unknown errors %llx set\n",
789  (unsigned long long)(hwerrs & ~IB_HWE_BITSEXTANT));
790 
791  ctrl = qib_read_kreg32(dd, kr_control);
792  if ((ctrl & QLOGIC_IB_C_FREEZEMODE) && !dd->diag_client) {
793  /*
794  * Parity errors in send memory are recoverable,
795  * just cancel the send (if indicated in * sendbuffererror),
796  * count the occurrence, unfreeze (if no other handled
797  * hardware error bits are set), and continue. They can
798  * occur if a processor speculative read is done to the PIO
799  * buffer while we are sending a packet, for example.
800  */
801  if (hwerrs & TXE_PIO_PARITY) {
802  qib_6120_txe_recover(dd);
803  hwerrs &= ~TXE_PIO_PARITY;
804  }
805 
806  if (!hwerrs) {
807  static u32 freeze_cnt;
808 
809  freeze_cnt++;
810  qib_6120_clear_freeze(dd);
811  } else
812  isfatal = 1;
813  }
814 
815  *msg = '\0';
816 
817  if (hwerrs & HWE_MASK(PowerOnBISTFailed)) {
818  isfatal = 1;
819  strlcat(msg,
820  "[Memory BIST test failed, InfiniPath hardware unusable]",
821  msgl);
822  /* ignore from now on, so disable until driver reloaded */
823  dd->cspec->hwerrmask &= ~HWE_MASK(PowerOnBISTFailed);
824  qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
825  }
826 
827  qib_format_hwerrors(hwerrs, qib_6120_hwerror_msgs,
828  ARRAY_SIZE(qib_6120_hwerror_msgs), msg, msgl);
829 
830  bitsmsg = dd->cspec->bitsmsgbuf;
831  if (hwerrs & (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<
833  bits = (u32) ((hwerrs >>
836  snprintf(bitsmsg, sizeof dd->cspec->bitsmsgbuf,
837  "[PCIe Mem Parity Errs %x] ", bits);
838  strlcat(msg, bitsmsg, msgl);
839  }
840 
841  if (hwerrs & _QIB_PLL_FAIL) {
842  isfatal = 1;
843  snprintf(bitsmsg, sizeof dd->cspec->bitsmsgbuf,
844  "[PLL failed (%llx), InfiniPath hardware unusable]",
845  (unsigned long long) hwerrs & _QIB_PLL_FAIL);
846  strlcat(msg, bitsmsg, msgl);
847  /* ignore from now on, so disable until driver reloaded */
848  dd->cspec->hwerrmask &= ~(hwerrs & _QIB_PLL_FAIL);
849  qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
850  }
851 
852  if (hwerrs & QLOGIC_IB_HWE_SERDESPLLFAILED) {
853  /*
854  * If it occurs, it is left masked since the external
855  * interface is unused
856  */
857  dd->cspec->hwerrmask &= ~QLOGIC_IB_HWE_SERDESPLLFAILED;
858  qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
859  }
860 
861  if (hwerrs)
862  /*
863  * if any set that we aren't ignoring; only
864  * make the complaint once, in case it's stuck
865  * or recurring, and we get here multiple
866  * times.
867  */
868  qib_dev_err(dd, "%s hardware error\n", msg);
869  else
870  *msg = 0; /* recovered from all of them */
871 
872  if (isfatal && !dd->diag_client) {
873  qib_dev_err(dd,
874  "Fatal Hardware Error, no longer usable, SN %.16s\n",
875  dd->serial);
876  /*
877  * for /sys status file and user programs to print; if no
878  * trailing brace is copied, we'll know it was truncated.
879  */
880  if (dd->freezemsg)
881  snprintf(dd->freezemsg, dd->freezelen,
882  "{%s}", msg);
884  }
885 }
886 
887 /*
888  * Decode the error status into strings, deciding whether to always
889  * print * it or not depending on "normal packet errors" vs everything
890  * else. Return 1 if "real" errors, otherwise 0 if only packet
891  * errors, so caller can decide what to print with the string.
892  */
893 static int qib_decode_6120_err(struct qib_devdata *dd, char *buf, size_t blen,
894  u64 err)
895 {
896  int iserr = 1;
897 
898  *buf = '\0';
899  if (err & QLOGIC_IB_E_PKTERRS) {
900  if (!(err & ~QLOGIC_IB_E_PKTERRS))
901  iserr = 0;
902  if ((err & ERR_MASK(RcvICRCErr)) &&
903  !(err&(ERR_MASK(RcvVCRCErr)|ERR_MASK(RcvEBPErr))))
904  strlcat(buf, "CRC ", blen);
905  if (!iserr)
906  goto done;
907  }
908  if (err & ERR_MASK(RcvHdrLenErr))
909  strlcat(buf, "rhdrlen ", blen);
910  if (err & ERR_MASK(RcvBadTidErr))
911  strlcat(buf, "rbadtid ", blen);
912  if (err & ERR_MASK(RcvBadVersionErr))
913  strlcat(buf, "rbadversion ", blen);
914  if (err & ERR_MASK(RcvHdrErr))
915  strlcat(buf, "rhdr ", blen);
916  if (err & ERR_MASK(RcvLongPktLenErr))
917  strlcat(buf, "rlongpktlen ", blen);
918  if (err & ERR_MASK(RcvMaxPktLenErr))
919  strlcat(buf, "rmaxpktlen ", blen);
920  if (err & ERR_MASK(RcvMinPktLenErr))
921  strlcat(buf, "rminpktlen ", blen);
922  if (err & ERR_MASK(SendMinPktLenErr))
923  strlcat(buf, "sminpktlen ", blen);
924  if (err & ERR_MASK(RcvFormatErr))
925  strlcat(buf, "rformaterr ", blen);
926  if (err & ERR_MASK(RcvUnsupportedVLErr))
927  strlcat(buf, "runsupvl ", blen);
928  if (err & ERR_MASK(RcvUnexpectedCharErr))
929  strlcat(buf, "runexpchar ", blen);
930  if (err & ERR_MASK(RcvIBFlowErr))
931  strlcat(buf, "ribflow ", blen);
932  if (err & ERR_MASK(SendUnderRunErr))
933  strlcat(buf, "sunderrun ", blen);
934  if (err & ERR_MASK(SendPioArmLaunchErr))
935  strlcat(buf, "spioarmlaunch ", blen);
936  if (err & ERR_MASK(SendUnexpectedPktNumErr))
937  strlcat(buf, "sunexperrpktnum ", blen);
938  if (err & ERR_MASK(SendDroppedSmpPktErr))
939  strlcat(buf, "sdroppedsmppkt ", blen);
940  if (err & ERR_MASK(SendMaxPktLenErr))
941  strlcat(buf, "smaxpktlen ", blen);
942  if (err & ERR_MASK(SendUnsupportedVLErr))
943  strlcat(buf, "sunsupVL ", blen);
944  if (err & ERR_MASK(InvalidAddrErr))
945  strlcat(buf, "invalidaddr ", blen);
946  if (err & ERR_MASK(RcvEgrFullErr))
947  strlcat(buf, "rcvegrfull ", blen);
948  if (err & ERR_MASK(RcvHdrFullErr))
949  strlcat(buf, "rcvhdrfull ", blen);
950  if (err & ERR_MASK(IBStatusChanged))
951  strlcat(buf, "ibcstatuschg ", blen);
952  if (err & ERR_MASK(RcvIBLostLinkErr))
953  strlcat(buf, "riblostlink ", blen);
954  if (err & ERR_MASK(HardwareErr))
955  strlcat(buf, "hardware ", blen);
956  if (err & ERR_MASK(ResetNegated))
957  strlcat(buf, "reset ", blen);
958 done:
959  return iserr;
960 }
961 
962 /*
963  * Called when we might have an error that is specific to a particular
964  * PIO buffer, and may need to cancel that buffer, so it can be re-used.
965  */
966 static void qib_disarm_6120_senderrbufs(struct qib_pportdata *ppd)
967 {
968  unsigned long sbuf[2];
969  struct qib_devdata *dd = ppd->dd;
970 
971  /*
972  * It's possible that sendbuffererror could have bits set; might
973  * have already done this as a result of hardware error handling.
974  */
975  sbuf[0] = qib_read_kreg64(dd, kr_sendbuffererror);
976  sbuf[1] = qib_read_kreg64(dd, kr_sendbuffererror + 1);
977 
978  if (sbuf[0] || sbuf[1])
979  qib_disarm_piobufs_set(dd, sbuf,
980  dd->piobcnt2k + dd->piobcnt4k);
981 }
982 
983 static int chk_6120_linkrecovery(struct qib_devdata *dd, u64 ibcs)
984 {
985  int ret = 1;
986  u32 ibstate = qib_6120_iblink_state(ibcs);
987  u32 linkrecov = read_6120_creg32(dd, cr_iblinkerrrecov);
988 
989  if (linkrecov != dd->cspec->lastlinkrecov) {
990  /* and no more until active again */
991  dd->cspec->lastlinkrecov = 0;
993  ret = 0;
994  }
995  if (ibstate == IB_PORT_ACTIVE)
996  dd->cspec->lastlinkrecov =
997  read_6120_creg32(dd, cr_iblinkerrrecov);
998  return ret;
999 }
1000 
1001 static void handle_6120_errors(struct qib_devdata *dd, u64 errs)
1002 {
1003  char *msg;
1004  u64 ignore_this_time = 0;
1005  u64 iserr = 0;
1006  int log_idx;
1007  struct qib_pportdata *ppd = dd->pport;
1008  u64 mask;
1009 
1010  /* don't report errors that are masked */
1011  errs &= dd->cspec->errormask;
1012  msg = dd->cspec->emsgbuf;
1013 
1014  /* do these first, they are most important */
1015  if (errs & ERR_MASK(HardwareErr))
1016  qib_handle_6120_hwerrors(dd, msg, sizeof dd->cspec->emsgbuf);
1017  else
1018  for (log_idx = 0; log_idx < QIB_EEP_LOG_CNT; ++log_idx)
1019  if (errs & dd->eep_st_masks[log_idx].errs_to_log)
1020  qib_inc_eeprom_err(dd, log_idx, 1);
1021 
1022  if (errs & ~IB_E_BITSEXTANT)
1023  qib_dev_err(dd,
1024  "error interrupt with unknown errors %llx set\n",
1025  (unsigned long long) (errs & ~IB_E_BITSEXTANT));
1026 
1027  if (errs & E_SUM_ERRS) {
1028  qib_disarm_6120_senderrbufs(ppd);
1029  if ((errs & E_SUM_LINK_PKTERRS) &&
1030  !(ppd->lflags & QIBL_LINKACTIVE)) {
1031  /*
1032  * This can happen when trying to bring the link
1033  * up, but the IB link changes state at the "wrong"
1034  * time. The IB logic then complains that the packet
1035  * isn't valid. We don't want to confuse people, so
1036  * we just don't print them, except at debug
1037  */
1038  ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1039  }
1040  } else if ((errs & E_SUM_LINK_PKTERRS) &&
1041  !(ppd->lflags & QIBL_LINKACTIVE)) {
1042  /*
1043  * This can happen when SMA is trying to bring the link
1044  * up, but the IB link changes state at the "wrong" time.
1045  * The IB logic then complains that the packet isn't
1046  * valid. We don't want to confuse people, so we just
1047  * don't print them, except at debug
1048  */
1049  ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1050  }
1051 
1052  qib_write_kreg(dd, kr_errclear, errs);
1053 
1054  errs &= ~ignore_this_time;
1055  if (!errs)
1056  goto done;
1057 
1058  /*
1059  * The ones we mask off are handled specially below
1060  * or above.
1061  */
1062  mask = ERR_MASK(IBStatusChanged) | ERR_MASK(RcvEgrFullErr) |
1063  ERR_MASK(RcvHdrFullErr) | ERR_MASK(HardwareErr);
1064  qib_decode_6120_err(dd, msg, sizeof dd->cspec->emsgbuf, errs & ~mask);
1065 
1066  if (errs & E_SUM_PKTERRS)
1067  qib_stats.sps_rcverrs++;
1068  if (errs & E_SUM_ERRS)
1069  qib_stats.sps_txerrs++;
1070 
1071  iserr = errs & ~(E_SUM_PKTERRS | QLOGIC_IB_E_PKTERRS);
1072 
1073  if (errs & ERR_MASK(IBStatusChanged)) {
1074  u64 ibcs = qib_read_kreg64(dd, kr_ibcstatus);
1075  u32 ibstate = qib_6120_iblink_state(ibcs);
1076  int handle = 1;
1077 
1078  if (ibstate != IB_PORT_INIT && dd->cspec->lastlinkrecov)
1079  handle = chk_6120_linkrecovery(dd, ibcs);
1080  /*
1081  * Since going into a recovery state causes the link state
1082  * to go down and since recovery is transitory, it is better
1083  * if we "miss" ever seeing the link training state go into
1084  * recovery (i.e., ignore this transition for link state
1085  * special handling purposes) without updating lastibcstat.
1086  */
1087  if (handle && qib_6120_phys_portstate(ibcs) ==
1089  handle = 0;
1090  if (handle)
1091  qib_handle_e_ibstatuschanged(ppd, ibcs);
1092  }
1093 
1094  if (errs & ERR_MASK(ResetNegated)) {
1095  qib_dev_err(dd,
1096  "Got reset, requires re-init (unload and reload driver)\n");
1097  dd->flags &= ~QIB_INITTED; /* needs re-init */
1098  /* mark as having had error */
1100  *dd->pport->statusp &= ~QIB_STATUS_IB_CONF;
1101  }
1102 
1103  if (*msg && iserr)
1104  qib_dev_porterr(dd, ppd->port, "%s error\n", msg);
1105 
1106  if (ppd->state_wanted & ppd->lflags)
1108 
1109  /*
1110  * If there were hdrq or egrfull errors, wake up any processes
1111  * waiting in poll. We used to try to check which contexts had
1112  * the overflow, but given the cost of that and the chip reads
1113  * to support it, it's better to just wake everybody up if we
1114  * get an overflow; waiters can poll again if it's not them.
1115  */
1116  if (errs & (ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr))) {
1117  qib_handle_urcv(dd, ~0U);
1118  if (errs & ERR_MASK(RcvEgrFullErr))
1119  qib_stats.sps_buffull++;
1120  else
1121  qib_stats.sps_hdrfull++;
1122  }
1123 done:
1124  return;
1125 }
1126 
1137 static void qib_6120_init_hwerrors(struct qib_devdata *dd)
1138 {
1139  u64 val;
1140  u64 extsval;
1141 
1142  extsval = qib_read_kreg64(dd, kr_extstatus);
1143 
1144  if (!(extsval & QLOGIC_IB_EXTS_MEMBIST_ENDTEST))
1145  qib_dev_err(dd, "MemBIST did not complete!\n");
1146 
1147  /* init so all hwerrors interrupt, and enter freeze, ajdust below */
1148  val = ~0ULL;
1149  if (dd->minrev < 2) {
1150  /*
1151  * Avoid problem with internal interface bus parity
1152  * checking. Fixed in Rev2.
1153  */
1155  }
1156  /* avoid some intel cpu's speculative read freeze mode issue */
1157  val &= ~TXEMEMPARITYERR_PIOBUF;
1158 
1159  dd->cspec->hwerrmask = val;
1160 
1161  qib_write_kreg(dd, kr_hwerrclear, ~HWE_MASK(PowerOnBISTFailed));
1162  qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1163 
1164  /* clear all */
1165  qib_write_kreg(dd, kr_errclear, ~0ULL);
1166  /* enable errors that are masked, at least this first time. */
1167  qib_write_kreg(dd, kr_errmask, ~0ULL);
1168  dd->cspec->errormask = qib_read_kreg64(dd, kr_errmask);
1169  /* clear any interrupts up to this point (ints still not enabled) */
1170  qib_write_kreg(dd, kr_intclear, ~0ULL);
1171 
1172  qib_write_kreg(dd, kr_rcvbthqp,
1174  QIB_KD_QP);
1175 }
1176 
1177 /*
1178  * Disable and enable the armlaunch error. Used for PIO bandwidth testing
1179  * on chips that are count-based, rather than trigger-based. There is no
1180  * reference counting, but that's also fine, given the intended use.
1181  * Only chip-specific because it's all register accesses
1182  */
1183 static void qib_set_6120_armlaunch(struct qib_devdata *dd, u32 enable)
1184 {
1185  if (enable) {
1186  qib_write_kreg(dd, kr_errclear,
1187  ERR_MASK(SendPioArmLaunchErr));
1188  dd->cspec->errormask |= ERR_MASK(SendPioArmLaunchErr);
1189  } else
1190  dd->cspec->errormask &= ~ERR_MASK(SendPioArmLaunchErr);
1191  qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
1192 }
1193 
1194 /*
1195  * Formerly took parameter <which> in pre-shifted,
1196  * pre-merged form with LinkCmd and LinkInitCmd
1197  * together, and assuming the zero was NOP.
1198  */
1199 static void qib_set_ib_6120_lstate(struct qib_pportdata *ppd, u16 linkcmd,
1200  u16 linitcmd)
1201 {
1202  u64 mod_wd;
1203  struct qib_devdata *dd = ppd->dd;
1204  unsigned long flags;
1205 
1206  if (linitcmd == QLOGIC_IB_IBCC_LINKINITCMD_DISABLE) {
1207  /*
1208  * If we are told to disable, note that so link-recovery
1209  * code does not attempt to bring us back up.
1210  */
1211  spin_lock_irqsave(&ppd->lflags_lock, flags);
1212  ppd->lflags |= QIBL_IB_LINK_DISABLED;
1213  spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1214  } else if (linitcmd || linkcmd == QLOGIC_IB_IBCC_LINKCMD_DOWN) {
1215  /*
1216  * Any other linkinitcmd will lead to LINKDOWN and then
1217  * to INIT (if all is well), so clear flag to let
1218  * link-recovery code attempt to bring us back up.
1219  */
1220  spin_lock_irqsave(&ppd->lflags_lock, flags);
1221  ppd->lflags &= ~QIBL_IB_LINK_DISABLED;
1222  spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1223  }
1224 
1225  mod_wd = (linkcmd << QLOGIC_IB_IBCC_LINKCMD_SHIFT) |
1226  (linitcmd << QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1227 
1228  qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl | mod_wd);
1229  /* write to chip to prevent back-to-back writes of control reg */
1230  qib_write_kreg(dd, kr_scratch, 0);
1231 }
1232 
1237 static int qib_6120_bringup_serdes(struct qib_pportdata *ppd)
1238 {
1239  struct qib_devdata *dd = ppd->dd;
1240  u64 val, config1, prev_val, hwstat, ibc;
1241 
1242  /* Put IBC in reset, sends disabled */
1244  qib_write_kreg(dd, kr_control, 0ULL);
1245 
1246  dd->cspec->ibdeltainprog = 1;
1247  dd->cspec->ibsymsnap = read_6120_creg32(dd, cr_ibsymbolerr);
1248  dd->cspec->iblnkerrsnap = read_6120_creg32(dd, cr_iblinkerrrecov);
1249 
1250  /* flowcontrolwatermark is in units of KBytes */
1251  ibc = 0x5ULL << SYM_LSB(IBCCtrl, FlowCtrlWaterMark);
1252  /*
1253  * How often flowctrl sent. More or less in usecs; balance against
1254  * watermark value, so that in theory senders always get a flow
1255  * control update in time to not let the IB link go idle.
1256  */
1257  ibc |= 0x3ULL << SYM_LSB(IBCCtrl, FlowCtrlPeriod);
1258  /* max error tolerance */
1259  dd->cspec->lli_thresh = 0xf;
1260  ibc |= (u64) dd->cspec->lli_thresh << SYM_LSB(IBCCtrl, PhyerrThreshold);
1261  /* use "real" buffer space for */
1262  ibc |= 4ULL << SYM_LSB(IBCCtrl, CreditScale);
1263  /* IB credit flow control. */
1264  ibc |= 0xfULL << SYM_LSB(IBCCtrl, OverrunThreshold);
1265  /*
1266  * set initial max size pkt IBC will send, including ICRC; it's the
1267  * PIO buffer size in dwords, less 1; also see qib_set_mtu()
1268  */
1269  ibc |= ((u64)(ppd->ibmaxlen >> 2) + 1) << SYM_LSB(IBCCtrl, MaxPktLen);
1270  dd->cspec->ibcctrl = ibc; /* without linkcmd or linkinitcmd! */
1271 
1272  /* initially come up waiting for TS1, without sending anything. */
1273  val = dd->cspec->ibcctrl | (QLOGIC_IB_IBCC_LINKINITCMD_DISABLE <<
1275  qib_write_kreg(dd, kr_ibcctrl, val);
1276 
1277  val = qib_read_kreg64(dd, kr_serdes_cfg0);
1278  config1 = qib_read_kreg64(dd, kr_serdes_cfg1);
1279 
1280  /*
1281  * Force reset on, also set rxdetect enable. Must do before reading
1282  * serdesstatus at least for simulation, or some of the bits in
1283  * serdes status will come back as undefined and cause simulation
1284  * failures
1285  */
1286  val |= SYM_MASK(SerdesCfg0, ResetPLL) |
1287  SYM_MASK(SerdesCfg0, RxDetEnX) |
1288  (SYM_MASK(SerdesCfg0, L1PwrDnA) |
1289  SYM_MASK(SerdesCfg0, L1PwrDnB) |
1290  SYM_MASK(SerdesCfg0, L1PwrDnC) |
1291  SYM_MASK(SerdesCfg0, L1PwrDnD));
1292  qib_write_kreg(dd, kr_serdes_cfg0, val);
1293  /* be sure chip saw it */
1294  qib_read_kreg64(dd, kr_scratch);
1295  udelay(5); /* need pll reset set at least for a bit */
1296  /*
1297  * after PLL is reset, set the per-lane Resets and TxIdle and
1298  * clear the PLL reset and rxdetect (to get falling edge).
1299  * Leave L1PWR bits set (permanently)
1300  */
1301  val &= ~(SYM_MASK(SerdesCfg0, RxDetEnX) |
1302  SYM_MASK(SerdesCfg0, ResetPLL) |
1303  (SYM_MASK(SerdesCfg0, L1PwrDnA) |
1304  SYM_MASK(SerdesCfg0, L1PwrDnB) |
1305  SYM_MASK(SerdesCfg0, L1PwrDnC) |
1306  SYM_MASK(SerdesCfg0, L1PwrDnD)));
1307  val |= (SYM_MASK(SerdesCfg0, ResetA) |
1308  SYM_MASK(SerdesCfg0, ResetB) |
1309  SYM_MASK(SerdesCfg0, ResetC) |
1310  SYM_MASK(SerdesCfg0, ResetD)) |
1311  SYM_MASK(SerdesCfg0, TxIdeEnX);
1312  qib_write_kreg(dd, kr_serdes_cfg0, val);
1313  /* be sure chip saw it */
1314  (void) qib_read_kreg64(dd, kr_scratch);
1315  /* need PLL reset clear for at least 11 usec before lane
1316  * resets cleared; give it a few more to be sure */
1317  udelay(15);
1318  val &= ~((SYM_MASK(SerdesCfg0, ResetA) |
1319  SYM_MASK(SerdesCfg0, ResetB) |
1320  SYM_MASK(SerdesCfg0, ResetC) |
1321  SYM_MASK(SerdesCfg0, ResetD)) |
1322  SYM_MASK(SerdesCfg0, TxIdeEnX));
1323 
1324  qib_write_kreg(dd, kr_serdes_cfg0, val);
1325  /* be sure chip saw it */
1326  (void) qib_read_kreg64(dd, kr_scratch);
1327 
1328  val = qib_read_kreg64(dd, kr_xgxs_cfg);
1329  prev_val = val;
1330  if (val & QLOGIC_IB_XGXS_RESET)
1331  val &= ~QLOGIC_IB_XGXS_RESET;
1332  if (SYM_FIELD(val, XGXSCfg, polarity_inv) != ppd->rx_pol_inv) {
1333  /* need to compensate for Tx inversion in partner */
1334  val &= ~SYM_MASK(XGXSCfg, polarity_inv);
1335  val |= (u64)ppd->rx_pol_inv << SYM_LSB(XGXSCfg, polarity_inv);
1336  }
1337  if (val != prev_val)
1338  qib_write_kreg(dd, kr_xgxs_cfg, val);
1339 
1340  val = qib_read_kreg64(dd, kr_serdes_cfg0);
1341 
1342  /* clear current and de-emphasis bits */
1343  config1 &= ~0x0ffffffff00ULL;
1344  /* set current to 20ma */
1345  config1 |= 0x00000000000ULL;
1346  /* set de-emphasis to -5.68dB */
1347  config1 |= 0x0cccc000000ULL;
1348  qib_write_kreg(dd, kr_serdes_cfg1, config1);
1349 
1350  /* base and port guid same for single port */
1351  ppd->guid = dd->base_guid;
1352 
1353  /*
1354  * the process of setting and un-resetting the serdes normally
1355  * causes a serdes PLL error, so check for that and clear it
1356  * here. Also clearr hwerr bit in errstatus, but not others.
1357  */
1358  hwstat = qib_read_kreg64(dd, kr_hwerrstatus);
1359  if (hwstat) {
1360  /* should just have PLL, clear all set, in an case */
1361  qib_write_kreg(dd, kr_hwerrclear, hwstat);
1362  qib_write_kreg(dd, kr_errclear, ERR_MASK(HardwareErr));
1363  }
1364 
1366  dd->control &= ~QLOGIC_IB_C_FREEZEMODE;
1367  qib_write_kreg(dd, kr_control, dd->control);
1368 
1369  return 0;
1370 }
1371 
1377 static void qib_6120_quiet_serdes(struct qib_pportdata *ppd)
1378 {
1379  struct qib_devdata *dd = ppd->dd;
1380  u64 val;
1381 
1382  qib_set_ib_6120_lstate(ppd, 0, QLOGIC_IB_IBCC_LINKINITCMD_DISABLE);
1383 
1384  /* disable IBC */
1386  qib_write_kreg(dd, kr_control,
1387  dd->control | QLOGIC_IB_C_FREEZEMODE);
1388 
1389  if (dd->cspec->ibsymdelta || dd->cspec->iblnkerrdelta ||
1390  dd->cspec->ibdeltainprog) {
1391  u64 diagc;
1392 
1393  /* enable counter writes */
1394  diagc = qib_read_kreg64(dd, kr_hwdiagctrl);
1395  qib_write_kreg(dd, kr_hwdiagctrl,
1396  diagc | SYM_MASK(HwDiagCtrl, CounterWrEnable));
1397 
1398  if (dd->cspec->ibsymdelta || dd->cspec->ibdeltainprog) {
1399  val = read_6120_creg32(dd, cr_ibsymbolerr);
1400  if (dd->cspec->ibdeltainprog)
1401  val -= val - dd->cspec->ibsymsnap;
1402  val -= dd->cspec->ibsymdelta;
1403  write_6120_creg(dd, cr_ibsymbolerr, val);
1404  }
1405  if (dd->cspec->iblnkerrdelta || dd->cspec->ibdeltainprog) {
1406  val = read_6120_creg32(dd, cr_iblinkerrrecov);
1407  if (dd->cspec->ibdeltainprog)
1408  val -= val - dd->cspec->iblnkerrsnap;
1409  val -= dd->cspec->iblnkerrdelta;
1410  write_6120_creg(dd, cr_iblinkerrrecov, val);
1411  }
1412 
1413  /* and disable counter writes */
1414  qib_write_kreg(dd, kr_hwdiagctrl, diagc);
1415  }
1416 
1417  val = qib_read_kreg64(dd, kr_serdes_cfg0);
1418  val |= SYM_MASK(SerdesCfg0, TxIdeEnX);
1419  qib_write_kreg(dd, kr_serdes_cfg0, val);
1420 }
1421 
1445 static void qib_6120_setup_setextled(struct qib_pportdata *ppd, u32 on)
1446 {
1447  u64 extctl, val, lst, ltst;
1448  unsigned long flags;
1449  struct qib_devdata *dd = ppd->dd;
1450 
1451  /*
1452  * The diags use the LED to indicate diag info, so we leave
1453  * the external LED alone when the diags are running.
1454  */
1455  if (dd->diag_client)
1456  return;
1457 
1458  /* Allow override of LED display for, e.g. Locating system in rack */
1459  if (ppd->led_override) {
1460  ltst = (ppd->led_override & QIB_LED_PHYS) ?
1462  lst = (ppd->led_override & QIB_LED_LOG) ?
1464  } else if (on) {
1465  val = qib_read_kreg64(dd, kr_ibcstatus);
1466  ltst = qib_6120_phys_portstate(val);
1467  lst = qib_6120_iblink_state(val);
1468  } else {
1469  ltst = 0;
1470  lst = 0;
1471  }
1472 
1473  spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
1474  extctl = dd->cspec->extctrl & ~(SYM_MASK(EXTCtrl, LEDPriPortGreenOn) |
1475  SYM_MASK(EXTCtrl, LEDPriPortYellowOn));
1476 
1477  if (ltst == IB_PHYSPORTSTATE_LINKUP)
1478  extctl |= SYM_MASK(EXTCtrl, LEDPriPortYellowOn);
1479  if (lst == IB_PORT_ACTIVE)
1480  extctl |= SYM_MASK(EXTCtrl, LEDPriPortGreenOn);
1481  dd->cspec->extctrl = extctl;
1482  qib_write_kreg(dd, kr_extctrl, extctl);
1483  spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
1484 }
1485 
1486 static void qib_6120_free_irq(struct qib_devdata *dd)
1487 {
1488  if (dd->cspec->irq) {
1489  free_irq(dd->cspec->irq, dd);
1490  dd->cspec->irq = 0;
1491  }
1492  qib_nomsi(dd);
1493 }
1494 
1501 static void qib_6120_setup_cleanup(struct qib_devdata *dd)
1502 {
1503  qib_6120_free_irq(dd);
1504  kfree(dd->cspec->cntrs);
1505  kfree(dd->cspec->portcntrs);
1506  if (dd->cspec->dummy_hdrq) {
1507  dma_free_coherent(&dd->pcidev->dev,
1508  ALIGN(dd->rcvhdrcnt *
1509  dd->rcvhdrentsize *
1510  sizeof(u32), PAGE_SIZE),
1511  dd->cspec->dummy_hdrq,
1512  dd->cspec->dummy_hdrq_phys);
1513  dd->cspec->dummy_hdrq = NULL;
1514  }
1515 }
1516 
1517 static void qib_wantpiobuf_6120_intr(struct qib_devdata *dd, u32 needint)
1518 {
1519  unsigned long flags;
1520 
1521  spin_lock_irqsave(&dd->sendctrl_lock, flags);
1522  if (needint)
1523  dd->sendctrl |= SYM_MASK(SendCtrl, PIOIntBufAvail);
1524  else
1525  dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOIntBufAvail);
1526  qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
1527  qib_write_kreg(dd, kr_scratch, 0ULL);
1528  spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
1529 }
1530 
1531 /*
1532  * handle errors and unusual events first, separate function
1533  * to improve cache hits for fast path interrupt handling
1534  */
1535 static noinline void unlikely_6120_intr(struct qib_devdata *dd, u64 istat)
1536 {
1537  if (unlikely(istat & ~QLOGIC_IB_I_BITSEXTANT))
1538  qib_dev_err(dd, "interrupt with unknown interrupts %Lx set\n",
1539  istat & ~QLOGIC_IB_I_BITSEXTANT);
1540 
1541  if (istat & QLOGIC_IB_I_ERROR) {
1542  u64 estat = 0;
1543 
1544  qib_stats.sps_errints++;
1545  estat = qib_read_kreg64(dd, kr_errstatus);
1546  if (!estat)
1547  qib_devinfo(dd->pcidev,
1548  "error interrupt (%Lx), but no error bits set!\n",
1549  istat);
1550  handle_6120_errors(dd, estat);
1551  }
1552 
1553  if (istat & QLOGIC_IB_I_GPIO) {
1554  u32 gpiostatus;
1555  u32 to_clear = 0;
1556 
1557  /*
1558  * GPIO_3..5 on IBA6120 Rev2 chips indicate
1559  * errors that we need to count.
1560  */
1561  gpiostatus = qib_read_kreg32(dd, kr_gpio_status);
1562  /* First the error-counter case. */
1563  if (gpiostatus & GPIO_ERRINTR_MASK) {
1564  /* want to clear the bits we see asserted. */
1565  to_clear |= (gpiostatus & GPIO_ERRINTR_MASK);
1566 
1567  /*
1568  * Count appropriately, clear bits out of our copy,
1569  * as they have been "handled".
1570  */
1571  if (gpiostatus & (1 << GPIO_RXUVL_BIT))
1572  dd->cspec->rxfc_unsupvl_errs++;
1573  if (gpiostatus & (1 << GPIO_OVRUN_BIT))
1574  dd->cspec->overrun_thresh_errs++;
1575  if (gpiostatus & (1 << GPIO_LLI_BIT))
1576  dd->cspec->lli_errs++;
1577  gpiostatus &= ~GPIO_ERRINTR_MASK;
1578  }
1579  if (gpiostatus) {
1580  /*
1581  * Some unexpected bits remain. If they could have
1582  * caused the interrupt, complain and clear.
1583  * To avoid repetition of this condition, also clear
1584  * the mask. It is almost certainly due to error.
1585  */
1586  const u32 mask = qib_read_kreg32(dd, kr_gpio_mask);
1587 
1588  /*
1589  * Also check that the chip reflects our shadow,
1590  * and report issues, If they caused the interrupt.
1591  * we will suppress by refreshing from the shadow.
1592  */
1593  if (mask & gpiostatus) {
1594  to_clear |= (gpiostatus & mask);
1595  dd->cspec->gpio_mask &= ~(gpiostatus & mask);
1596  qib_write_kreg(dd, kr_gpio_mask,
1597  dd->cspec->gpio_mask);
1598  }
1599  }
1600  if (to_clear)
1601  qib_write_kreg(dd, kr_gpio_clear, (u64) to_clear);
1602  }
1603 }
1604 
1605 static irqreturn_t qib_6120intr(int irq, void *data)
1606 {
1607  struct qib_devdata *dd = data;
1608  irqreturn_t ret;
1609  u32 istat, ctxtrbits, rmask, crcs = 0;
1610  unsigned i;
1611 
1612  if ((dd->flags & (QIB_PRESENT | QIB_BADINTR)) != QIB_PRESENT) {
1613  /*
1614  * This return value is not great, but we do not want the
1615  * interrupt core code to remove our interrupt handler
1616  * because we don't appear to be handling an interrupt
1617  * during a chip reset.
1618  */
1619  ret = IRQ_HANDLED;
1620  goto bail;
1621  }
1622 
1623  istat = qib_read_kreg32(dd, kr_intstatus);
1624 
1625  if (unlikely(!istat)) {
1626  ret = IRQ_NONE; /* not our interrupt, or already handled */
1627  goto bail;
1628  }
1629  if (unlikely(istat == -1)) {
1630  qib_bad_intrstatus(dd);
1631  /* don't know if it was our interrupt or not */
1632  ret = IRQ_NONE;
1633  goto bail;
1634  }
1635 
1636  qib_stats.sps_ints++;
1637  if (dd->int_counter != (u32) -1)
1638  dd->int_counter++;
1639 
1640  if (unlikely(istat & (~QLOGIC_IB_I_BITSEXTANT |
1641  QLOGIC_IB_I_GPIO | QLOGIC_IB_I_ERROR)))
1642  unlikely_6120_intr(dd, istat);
1643 
1644  /*
1645  * Clear the interrupt bits we found set, relatively early, so we
1646  * "know" know the chip will have seen this by the time we process
1647  * the queue, and will re-interrupt if necessary. The processor
1648  * itself won't take the interrupt again until we return.
1649  */
1650  qib_write_kreg(dd, kr_intclear, istat);
1651 
1652  /*
1653  * Handle kernel receive queues before checking for pio buffers
1654  * available since receives can overflow; piobuf waiters can afford
1655  * a few extra cycles, since they were waiting anyway.
1656  */
1657  ctxtrbits = istat &
1660  if (ctxtrbits) {
1661  rmask = (1U << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1663  for (i = 0; i < dd->first_user_ctxt; i++) {
1664  if (ctxtrbits & rmask) {
1665  ctxtrbits &= ~rmask;
1666  crcs += qib_kreceive(dd->rcd[i],
1667  &dd->cspec->lli_counter,
1668  NULL);
1669  }
1670  rmask <<= 1;
1671  }
1672  if (crcs) {
1673  u32 cntr = dd->cspec->lli_counter;
1674  cntr += crcs;
1675  if (cntr) {
1676  if (cntr > dd->cspec->lli_thresh) {
1677  dd->cspec->lli_counter = 0;
1678  dd->cspec->lli_errs++;
1679  } else
1680  dd->cspec->lli_counter += cntr;
1681  }
1682  }
1683 
1684 
1685  if (ctxtrbits) {
1686  ctxtrbits =
1687  (ctxtrbits >> QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1688  (ctxtrbits >> QLOGIC_IB_I_RCVURG_SHIFT);
1689  qib_handle_urcv(dd, ctxtrbits);
1690  }
1691  }
1692 
1693  if ((istat & QLOGIC_IB_I_SPIOBUFAVAIL) && (dd->flags & QIB_INITTED))
1694  qib_ib_piobufavail(dd);
1695 
1696  ret = IRQ_HANDLED;
1697 bail:
1698  return ret;
1699 }
1700 
1701 /*
1702  * Set up our chip-specific interrupt handler
1703  * The interrupt type has already been setup, so
1704  * we just need to do the registration and error checking.
1705  */
1706 static void qib_setup_6120_interrupt(struct qib_devdata *dd)
1707 {
1708  /*
1709  * If the chip supports added error indication via GPIO pins,
1710  * enable interrupts on those bits so the interrupt routine
1711  * can count the events. Also set flag so interrupt routine
1712  * can know they are expected.
1713  */
1714  if (SYM_FIELD(dd->revision, Revision_R,
1715  ChipRevMinor) > 1) {
1716  /* Rev2+ reports extra errors via internal GPIO pins */
1717  dd->cspec->gpio_mask |= GPIO_ERRINTR_MASK;
1718  qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1719  }
1720 
1721  if (!dd->cspec->irq)
1722  qib_dev_err(dd,
1723  "irq is 0, BIOS error? Interrupts won't work\n");
1724  else {
1725  int ret;
1726  ret = request_irq(dd->cspec->irq, qib_6120intr, 0,
1727  QIB_DRV_NAME, dd);
1728  if (ret)
1729  qib_dev_err(dd,
1730  "Couldn't setup interrupt (irq=%d): %d\n",
1731  dd->cspec->irq, ret);
1732  }
1733 }
1734 
1741 static void pe_boardname(struct qib_devdata *dd)
1742 {
1743  char *n;
1744  u32 boardid, namelen;
1745 
1746  boardid = SYM_FIELD(dd->revision, Revision,
1747  BoardID);
1748 
1749  switch (boardid) {
1750  case 2:
1751  n = "InfiniPath_QLE7140";
1752  break;
1753  default:
1754  qib_dev_err(dd, "Unknown 6120 board with ID %u\n", boardid);
1755  n = "Unknown_InfiniPath_6120";
1756  break;
1757  }
1758  namelen = strlen(n) + 1;
1759  dd->boardname = kmalloc(namelen, GFP_KERNEL);
1760  if (!dd->boardname)
1761  qib_dev_err(dd, "Failed allocation for board name: %s\n", n);
1762  else
1763  snprintf(dd->boardname, namelen, "%s", n);
1764 
1765  if (dd->majrev != 4 || !dd->minrev || dd->minrev > 2)
1766  qib_dev_err(dd,
1767  "Unsupported InfiniPath hardware revision %u.%u!\n",
1768  dd->majrev, dd->minrev);
1769 
1770  snprintf(dd->boardversion, sizeof(dd->boardversion),
1771  "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
1773  (unsigned)SYM_FIELD(dd->revision, Revision_R, Arch),
1774  dd->majrev, dd->minrev,
1775  (unsigned)SYM_FIELD(dd->revision, Revision_R, SW));
1776 
1777 }
1778 
1779 /*
1780  * This routine sleeps, so it can only be called from user context, not
1781  * from interrupt context. If we need interrupt context, we can split
1782  * it into two routines.
1783  */
1784 static int qib_6120_setup_reset(struct qib_devdata *dd)
1785 {
1786  u64 val;
1787  int i;
1788  int ret;
1789  u16 cmdval;
1790  u8 int_line, clinesz;
1791 
1792  qib_pcie_getcmd(dd, &cmdval, &int_line, &clinesz);
1793 
1794  /* Use ERROR so it shows up in logs, etc. */
1795  qib_dev_err(dd, "Resetting InfiniPath unit %u\n", dd->unit);
1796 
1797  /* no interrupts till re-initted */
1798  qib_6120_set_intr_state(dd, 0);
1799 
1800  dd->cspec->ibdeltainprog = 0;
1801  dd->cspec->ibsymdelta = 0;
1802  dd->cspec->iblnkerrdelta = 0;
1803 
1804  /*
1805  * Keep chip from being accessed until we are ready. Use
1806  * writeq() directly, to allow the write even though QIB_PRESENT
1807  * isn't set.
1808  */
1809  dd->flags &= ~(QIB_INITTED | QIB_PRESENT);
1810  dd->int_counter = 0; /* so we check interrupts work again */
1811  val = dd->control | QLOGIC_IB_C_RESET;
1812  writeq(val, &dd->kregbase[kr_control]);
1813  mb(); /* prevent compiler re-ordering around actual reset */
1814 
1815  for (i = 1; i <= 5; i++) {
1816  /*
1817  * Allow MBIST, etc. to complete; longer on each retry.
1818  * We sometimes get machine checks from bus timeout if no
1819  * response, so for now, make it *really* long.
1820  */
1821  msleep(1000 + (1 + i) * 2000);
1822 
1823  qib_pcie_reenable(dd, cmdval, int_line, clinesz);
1824 
1825  /*
1826  * Use readq directly, so we don't need to mark it as PRESENT
1827  * until we get a successful indication that all is well.
1828  */
1829  val = readq(&dd->kregbase[kr_revision]);
1830  if (val == dd->revision) {
1831  dd->flags |= QIB_PRESENT; /* it's back */
1832  ret = qib_reinit_intr(dd);
1833  goto bail;
1834  }
1835  }
1836  ret = 0; /* failed */
1837 
1838 bail:
1839  if (ret) {
1840  if (qib_pcie_params(dd, dd->lbus_width, NULL, NULL))
1841  qib_dev_err(dd,
1842  "Reset failed to setup PCIe or interrupts; continuing anyway\n");
1843  /* clear the reset error, init error/hwerror mask */
1844  qib_6120_init_hwerrors(dd);
1845  /* for Rev2 error interrupts; nop for rev 1 */
1846  qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1847  /* clear the reset error, init error/hwerror mask */
1848  qib_6120_init_hwerrors(dd);
1849  }
1850  return ret;
1851 }
1852 
1865 static void qib_6120_put_tid(struct qib_devdata *dd, u64 __iomem *tidptr,
1866  u32 type, unsigned long pa)
1867 {
1868  u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1869  unsigned long flags;
1870  int tidx;
1871  spinlock_t *tidlockp; /* select appropriate spinlock */
1872 
1873  if (!dd->kregbase)
1874  return;
1875 
1876  if (pa != dd->tidinvalid) {
1877  if (pa & ((1U << 11) - 1)) {
1878  qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1879  pa);
1880  return;
1881  }
1882  pa >>= 11;
1883  if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1884  qib_dev_err(dd,
1885  "Physical page address 0x%lx larger than supported\n",
1886  pa);
1887  return;
1888  }
1889 
1890  if (type == RCVHQ_RCV_TYPE_EAGER)
1891  pa |= dd->tidtemplate;
1892  else /* for now, always full 4KB page */
1893  pa |= 2 << 29;
1894  }
1895 
1896  /*
1897  * Avoid chip issue by writing the scratch register
1898  * before and after the TID, and with an io write barrier.
1899  * We use a spinlock around the writes, so they can't intermix
1900  * with other TID (eager or expected) writes (the chip problem
1901  * is triggered by back to back TID writes). Unfortunately, this
1902  * call can be done from interrupt level for the ctxt 0 eager TIDs,
1903  * so we have to use irqsave locks.
1904  */
1905  /*
1906  * Assumes tidptr always > egrtidbase
1907  * if type == RCVHQ_RCV_TYPE_EAGER.
1908  */
1909  tidx = tidptr - dd->egrtidbase;
1910 
1911  tidlockp = (type == RCVHQ_RCV_TYPE_EAGER && tidx < dd->rcvhdrcnt)
1912  ? &dd->cspec->kernel_tid_lock : &dd->cspec->user_tid_lock;
1913  spin_lock_irqsave(tidlockp, flags);
1914  qib_write_kreg(dd, kr_scratch, 0xfeeddeaf);
1915  writel(pa, tidp32);
1916  qib_write_kreg(dd, kr_scratch, 0xdeadbeef);
1917  mmiowb();
1918  spin_unlock_irqrestore(tidlockp, flags);
1919 }
1920 
1933 static void qib_6120_put_tid_2(struct qib_devdata *dd, u64 __iomem *tidptr,
1934  u32 type, unsigned long pa)
1935 {
1936  u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1937  u32 tidx;
1938 
1939  if (!dd->kregbase)
1940  return;
1941 
1942  if (pa != dd->tidinvalid) {
1943  if (pa & ((1U << 11) - 1)) {
1944  qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1945  pa);
1946  return;
1947  }
1948  pa >>= 11;
1949  if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1950  qib_dev_err(dd,
1951  "Physical page address 0x%lx larger than supported\n",
1952  pa);
1953  return;
1954  }
1955 
1956  if (type == RCVHQ_RCV_TYPE_EAGER)
1957  pa |= dd->tidtemplate;
1958  else /* for now, always full 4KB page */
1959  pa |= 2 << 29;
1960  }
1961  tidx = tidptr - dd->egrtidbase;
1962  writel(pa, tidp32);
1963  mmiowb();
1964 }
1965 
1966 
1977 static void qib_6120_clear_tids(struct qib_devdata *dd,
1978  struct qib_ctxtdata *rcd)
1979 {
1980  u64 __iomem *tidbase;
1981  unsigned long tidinv;
1982  u32 ctxt;
1983  int i;
1984 
1985  if (!dd->kregbase || !rcd)
1986  return;
1987 
1988  ctxt = rcd->ctxt;
1989 
1990  tidinv = dd->tidinvalid;
1991  tidbase = (u64 __iomem *)
1992  ((char __iomem *)(dd->kregbase) +
1993  dd->rcvtidbase +
1994  ctxt * dd->rcvtidcnt * sizeof(*tidbase));
1995 
1996  for (i = 0; i < dd->rcvtidcnt; i++)
1997  /* use func pointer because could be one of two funcs */
1998  dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EXPECTED,
1999  tidinv);
2000 
2001  tidbase = (u64 __iomem *)
2002  ((char __iomem *)(dd->kregbase) +
2003  dd->rcvegrbase +
2004  rcd->rcvegr_tid_base * sizeof(*tidbase));
2005 
2006  for (i = 0; i < rcd->rcvegrcnt; i++)
2007  /* use func pointer because could be one of two funcs */
2008  dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EAGER,
2009  tidinv);
2010 }
2011 
2018 static void qib_6120_tidtemplate(struct qib_devdata *dd)
2019 {
2020  u32 egrsize = dd->rcvegrbufsize;
2021 
2022  /*
2023  * For now, we always allocate 4KB buffers (at init) so we can
2024  * receive max size packets. We may want a module parameter to
2025  * specify 2KB or 4KB and/or make be per ctxt instead of per device
2026  * for those who want to reduce memory footprint. Note that the
2027  * rcvhdrentsize size must be large enough to hold the largest
2028  * IB header (currently 96 bytes) that we expect to handle (plus of
2029  * course the 2 dwords of RHF).
2030  */
2031  if (egrsize == 2048)
2032  dd->tidtemplate = 1U << 29;
2033  else if (egrsize == 4096)
2034  dd->tidtemplate = 2U << 29;
2035  dd->tidinvalid = 0;
2036 }
2037 
2039 {
2040  return 0;
2041 }
2042 
2051 static int qib_6120_get_base_info(struct qib_ctxtdata *rcd,
2052  struct qib_base_info *kinfo)
2053 {
2054  if (qib_unordered_wc())
2056 
2059  return 0;
2060 }
2061 
2062 
2063 static struct qib_message_header *
2064 qib_6120_get_msgheader(struct qib_devdata *dd, __le32 *rhf_addr)
2065 {
2066  return (struct qib_message_header *)
2067  &rhf_addr[sizeof(u64) / sizeof(u32)];
2068 }
2069 
2070 static void qib_6120_config_ctxts(struct qib_devdata *dd)
2071 {
2072  dd->ctxtcnt = qib_read_kreg32(dd, kr_portcnt);
2073  if (qib_n_krcv_queues > 1) {
2075  if (dd->first_user_ctxt > dd->ctxtcnt)
2076  dd->first_user_ctxt = dd->ctxtcnt;
2077  dd->qpn_mask = dd->first_user_ctxt <= 2 ? 2 : 6;
2078  } else
2079  dd->first_user_ctxt = dd->num_pports;
2080  dd->n_krcv_queues = dd->first_user_ctxt;
2081 }
2082 
2083 static void qib_update_6120_usrhead(struct qib_ctxtdata *rcd, u64 hd,
2084  u32 updegr, u32 egrhd, u32 npkts)
2085 {
2086  if (updegr)
2087  qib_write_ureg(rcd->dd, ur_rcvegrindexhead, egrhd, rcd->ctxt);
2088  mmiowb();
2089  qib_write_ureg(rcd->dd, ur_rcvhdrhead, hd, rcd->ctxt);
2090  mmiowb();
2091 }
2092 
2093 static u32 qib_6120_hdrqempty(struct qib_ctxtdata *rcd)
2094 {
2095  u32 head, tail;
2096 
2097  head = qib_read_ureg32(rcd->dd, ur_rcvhdrhead, rcd->ctxt);
2098  if (rcd->rcvhdrtail_kvaddr)
2099  tail = qib_get_rcvhdrtail(rcd);
2100  else
2101  tail = qib_read_ureg32(rcd->dd, ur_rcvhdrtail, rcd->ctxt);
2102  return head == tail;
2103 }
2104 
2105 /*
2106  * Used when we close any ctxt, for DMA already in flight
2107  * at close. Can't be done until we know hdrq size, so not
2108  * early in chip init.
2109  */
2110 static void alloc_dummy_hdrq(struct qib_devdata *dd)
2111 {
2112  dd->cspec->dummy_hdrq = dma_alloc_coherent(&dd->pcidev->dev,
2113  dd->rcd[0]->rcvhdrq_size,
2114  &dd->cspec->dummy_hdrq_phys,
2116  if (!dd->cspec->dummy_hdrq) {
2117  qib_devinfo(dd->pcidev, "Couldn't allocate dummy hdrq\n");
2118  /* fallback to just 0'ing */
2119  dd->cspec->dummy_hdrq_phys = 0UL;
2120  }
2121 }
2122 
2123 /*
2124  * Modify the RCVCTRL register in chip-specific way. This
2125  * is a function because bit positions and (future) register
2126  * location is chip-specific, but the needed operations are
2127  * generic. <op> is a bit-mask because we often want to
2128  * do multiple modifications.
2129  */
2130 static void rcvctrl_6120_mod(struct qib_pportdata *ppd, unsigned int op,
2131  int ctxt)
2132 {
2133  struct qib_devdata *dd = ppd->dd;
2134  u64 mask, val;
2135  unsigned long flags;
2136 
2137  spin_lock_irqsave(&dd->cspec->rcvmod_lock, flags);
2138 
2139  if (op & QIB_RCVCTRL_TAILUPD_ENB)
2140  dd->rcvctrl |= (1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2141  if (op & QIB_RCVCTRL_TAILUPD_DIS)
2142  dd->rcvctrl &= ~(1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2143  if (op & QIB_RCVCTRL_PKEY_ENB)
2144  dd->rcvctrl &= ~(1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2145  if (op & QIB_RCVCTRL_PKEY_DIS)
2146  dd->rcvctrl |= (1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2147  if (ctxt < 0)
2148  mask = (1ULL << dd->ctxtcnt) - 1;
2149  else
2150  mask = (1ULL << ctxt);
2151  if (op & QIB_RCVCTRL_CTXT_ENB) {
2152  /* always done for specific ctxt */
2153  dd->rcvctrl |= (mask << SYM_LSB(RcvCtrl, PortEnable));
2154  if (!(dd->flags & QIB_NODMA_RTAIL))
2155  dd->rcvctrl |= 1ULL << QLOGIC_IB_R_TAILUPD_SHIFT;
2156  /* Write these registers before the context is enabled. */
2157  qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2158  dd->rcd[ctxt]->rcvhdrqtailaddr_phys);
2159  qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2160  dd->rcd[ctxt]->rcvhdrq_phys);
2161 
2162  if (ctxt == 0 && !dd->cspec->dummy_hdrq)
2163  alloc_dummy_hdrq(dd);
2164  }
2165  if (op & QIB_RCVCTRL_CTXT_DIS)
2166  dd->rcvctrl &= ~(mask << SYM_LSB(RcvCtrl, PortEnable));
2167  if (op & QIB_RCVCTRL_INTRAVAIL_ENB)
2168  dd->rcvctrl |= (mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2169  if (op & QIB_RCVCTRL_INTRAVAIL_DIS)
2170  dd->rcvctrl &= ~(mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2171  qib_write_kreg(dd, kr_rcvctrl, dd->rcvctrl);
2172  if ((op & QIB_RCVCTRL_INTRAVAIL_ENB) && dd->rhdrhead_intr_off) {
2173  /* arm rcv interrupt */
2174  val = qib_read_ureg32(dd, ur_rcvhdrhead, ctxt) |
2175  dd->rhdrhead_intr_off;
2176  qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2177  }
2178  if (op & QIB_RCVCTRL_CTXT_ENB) {
2179  /*
2180  * Init the context registers also; if we were
2181  * disabled, tail and head should both be zero
2182  * already from the enable, but since we don't
2183  * know, we have to do it explicitly.
2184  */
2185  val = qib_read_ureg32(dd, ur_rcvegrindextail, ctxt);
2186  qib_write_ureg(dd, ur_rcvegrindexhead, val, ctxt);
2187 
2188  val = qib_read_ureg32(dd, ur_rcvhdrtail, ctxt);
2189  dd->rcd[ctxt]->head = val;
2190  /* If kctxt, interrupt on next receive. */
2191  if (ctxt < dd->first_user_ctxt)
2192  val |= dd->rhdrhead_intr_off;
2193  qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2194  }
2195  if (op & QIB_RCVCTRL_CTXT_DIS) {
2196  /*
2197  * Be paranoid, and never write 0's to these, just use an
2198  * unused page. Of course,
2199  * rcvhdraddr points to a large chunk of memory, so this
2200  * could still trash things, but at least it won't trash
2201  * page 0, and by disabling the ctxt, it should stop "soon",
2202  * even if a packet or two is in already in flight after we
2203  * disabled the ctxt. Only 6120 has this issue.
2204  */
2205  if (ctxt >= 0) {
2206  qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2207  dd->cspec->dummy_hdrq_phys);
2208  qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2209  dd->cspec->dummy_hdrq_phys);
2210  } else {
2211  unsigned i;
2212 
2213  for (i = 0; i < dd->cfgctxts; i++) {
2214  qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr,
2215  i, dd->cspec->dummy_hdrq_phys);
2216  qib_write_kreg_ctxt(dd, kr_rcvhdraddr,
2217  i, dd->cspec->dummy_hdrq_phys);
2218  }
2219  }
2220  }
2221  spin_unlock_irqrestore(&dd->cspec->rcvmod_lock, flags);
2222 }
2223 
2224 /*
2225  * Modify the SENDCTRL register in chip-specific way. This
2226  * is a function there may be multiple such registers with
2227  * slightly different layouts. Only operations actually used
2228  * are implemented yet.
2229  * Chip requires no back-back sendctrl writes, so write
2230  * scratch register after writing sendctrl
2231  */
2232 static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op)
2233 {
2234  struct qib_devdata *dd = ppd->dd;
2235  u64 tmp_dd_sendctrl;
2236  unsigned long flags;
2237 
2238  spin_lock_irqsave(&dd->sendctrl_lock, flags);
2239 
2240  /* First the ones that are "sticky", saved in shadow */
2241  if (op & QIB_SENDCTRL_CLEAR)
2242  dd->sendctrl = 0;
2243  if (op & QIB_SENDCTRL_SEND_DIS)
2244  dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOEnable);
2245  else if (op & QIB_SENDCTRL_SEND_ENB)
2246  dd->sendctrl |= SYM_MASK(SendCtrl, PIOEnable);
2247  if (op & QIB_SENDCTRL_AVAIL_DIS)
2248  dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2249  else if (op & QIB_SENDCTRL_AVAIL_ENB)
2250  dd->sendctrl |= SYM_MASK(SendCtrl, PIOBufAvailUpd);
2251 
2252  if (op & QIB_SENDCTRL_DISARM_ALL) {
2253  u32 i, last;
2254 
2255  tmp_dd_sendctrl = dd->sendctrl;
2256  /*
2257  * disarm any that are not yet launched, disabling sends
2258  * and updates until done.
2259  */
2260  last = dd->piobcnt2k + dd->piobcnt4k;
2261  tmp_dd_sendctrl &=
2262  ~(SYM_MASK(SendCtrl, PIOEnable) |
2263  SYM_MASK(SendCtrl, PIOBufAvailUpd));
2264  for (i = 0; i < last; i++) {
2265  qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl |
2266  SYM_MASK(SendCtrl, Disarm) | i);
2267  qib_write_kreg(dd, kr_scratch, 0);
2268  }
2269  }
2270 
2271  tmp_dd_sendctrl = dd->sendctrl;
2272 
2273  if (op & QIB_SENDCTRL_FLUSH)
2274  tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Abort);
2275  if (op & QIB_SENDCTRL_DISARM)
2276  tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Disarm) |
2278  SYM_LSB(SendCtrl, DisarmPIOBuf));
2279  if (op & QIB_SENDCTRL_AVAIL_BLIP)
2280  tmp_dd_sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2281 
2282  qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl);
2283  qib_write_kreg(dd, kr_scratch, 0);
2284 
2285  if (op & QIB_SENDCTRL_AVAIL_BLIP) {
2286  qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
2287  qib_write_kreg(dd, kr_scratch, 0);
2288  }
2289 
2290  spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
2291 
2292  if (op & QIB_SENDCTRL_FLUSH) {
2293  u32 v;
2294  /*
2295  * ensure writes have hit chip, then do a few
2296  * more reads, to allow DMA of pioavail registers
2297  * to occur, so in-memory copy is in sync with
2298  * the chip. Not always safe to sleep.
2299  */
2300  v = qib_read_kreg32(dd, kr_scratch);
2301  qib_write_kreg(dd, kr_scratch, v);
2302  v = qib_read_kreg32(dd, kr_scratch);
2303  qib_write_kreg(dd, kr_scratch, v);
2304  qib_read_kreg32(dd, kr_scratch);
2305  }
2306 }
2307 
2313 static u64 qib_portcntr_6120(struct qib_pportdata *ppd, u32 reg)
2314 {
2315  u64 ret = 0ULL;
2316  struct qib_devdata *dd = ppd->dd;
2317  u16 creg;
2318  /* 0xffff for unimplemented or synthesized counters */
2319  static const u16 xlator[] = {
2322  [QIBPORTCNTR_PSXMITDATA] = 0xffff,
2323  [QIBPORTCNTR_PSXMITPKTS] = 0xffff,
2324  [QIBPORTCNTR_PSXMITWAIT] = 0xffff,
2327  [QIBPORTCNTR_PSRCVDATA] = 0xffff,
2328  [QIBPORTCNTR_PSRCVPKTS] = 0xffff,
2333  [QIBPORTCNTR_RXLOCALPHYERR] = 0xffff,
2334  [QIBPORTCNTR_RXVLERR] = 0xffff,
2343  [QIBPORTCNTR_EXCESSBUFOVFL] = 0xffff,
2347  [QIBPORTCNTR_LLI] = 0xffff,
2348  [QIBPORTCNTR_PSINTERVAL] = 0xffff,
2349  [QIBPORTCNTR_PSSTART] = 0xffff,
2350  [QIBPORTCNTR_PSSTAT] = 0xffff,
2351  [QIBPORTCNTR_VL15PKTDROP] = 0xffff,
2353  [QIBPORTCNTR_KHDROVFL] = 0xffff,
2354  };
2355 
2356  if (reg >= ARRAY_SIZE(xlator)) {
2357  qib_devinfo(ppd->dd->pcidev,
2358  "Unimplemented portcounter %u\n", reg);
2359  goto done;
2360  }
2361  creg = xlator[reg];
2362 
2363  /* handle counters requests not implemented as chip counters */
2364  if (reg == QIBPORTCNTR_LLI)
2365  ret = dd->cspec->lli_errs;
2366  else if (reg == QIBPORTCNTR_EXCESSBUFOVFL)
2367  ret = dd->cspec->overrun_thresh_errs;
2368  else if (reg == QIBPORTCNTR_KHDROVFL) {
2369  int i;
2370 
2371  /* sum over all kernel contexts */
2372  for (i = 0; i < dd->first_user_ctxt; i++)
2373  ret += read_6120_creg32(dd, cr_portovfl + i);
2374  } else if (reg == QIBPORTCNTR_PSSTAT)
2375  ret = dd->cspec->pma_sample_status;
2376  if (creg == 0xffff)
2377  goto done;
2378 
2379  /*
2380  * only fast incrementing counters are 64bit; use 32 bit reads to
2381  * avoid two independent reads when on opteron
2382  */
2383  if (creg == cr_wordsend || creg == cr_wordrcv ||
2384  creg == cr_pktsend || creg == cr_pktrcv)
2385  ret = read_6120_creg(dd, creg);
2386  else
2387  ret = read_6120_creg32(dd, creg);
2388  if (creg == cr_ibsymbolerr) {
2389  if (dd->cspec->ibdeltainprog)
2390  ret -= ret - dd->cspec->ibsymsnap;
2391  ret -= dd->cspec->ibsymdelta;
2392  } else if (creg == cr_iblinkerrrecov) {
2393  if (dd->cspec->ibdeltainprog)
2394  ret -= ret - dd->cspec->iblnkerrsnap;
2395  ret -= dd->cspec->iblnkerrdelta;
2396  }
2397  if (reg == QIBPORTCNTR_RXDROPPKT) /* add special cased count */
2398  ret += dd->cspec->rxfc_unsupvl_errs;
2399 
2400 done:
2401  return ret;
2402 }
2403 
2404 /*
2405  * Device counter names (not port-specific), one line per stat,
2406  * single string. Used by utilities like ipathstats to print the stats
2407  * in a way which works for different versions of drivers, without changing
2408  * the utility. Names need to be 12 chars or less (w/o newline), for proper
2409  * display by utility.
2410  * Non-error counters are first.
2411  * Start of "error" conters is indicated by a leading "E " on the first
2412  * "error" counter, and doesn't count in label length.
2413  * The EgrOvfl list needs to be last so we truncate them at the configured
2414  * context count for the device.
2415  * cntr6120indices contains the corresponding register indices.
2416  */
2417 static const char cntr6120names[] =
2418  "Interrupts\n"
2419  "HostBusStall\n"
2420  "E RxTIDFull\n"
2421  "RxTIDInvalid\n"
2422  "Ctxt0EgrOvfl\n"
2423  "Ctxt1EgrOvfl\n"
2424  "Ctxt2EgrOvfl\n"
2425  "Ctxt3EgrOvfl\n"
2426  "Ctxt4EgrOvfl\n";
2427 
2428 static const size_t cntr6120indices[] = {
2429  cr_lbint,
2431  cr_errtidfull,
2433  cr_portovfl + 0,
2434  cr_portovfl + 1,
2435  cr_portovfl + 2,
2436  cr_portovfl + 3,
2437  cr_portovfl + 4,
2438 };
2439 
2440 /*
2441  * same as cntr6120names and cntr6120indices, but for port-specific counters.
2442  * portcntr6120indices is somewhat complicated by some registers needing
2443  * adjustments of various kinds, and those are ORed with _PORT_VIRT_FLAG
2444  */
2445 static const char portcntr6120names[] =
2446  "TxPkt\n"
2447  "TxFlowPkt\n"
2448  "TxWords\n"
2449  "RxPkt\n"
2450  "RxFlowPkt\n"
2451  "RxWords\n"
2452  "TxFlowStall\n"
2453  "E IBStatusChng\n"
2454  "IBLinkDown\n"
2455  "IBLnkRecov\n"
2456  "IBRxLinkErr\n"
2457  "IBSymbolErr\n"
2458  "RxLLIErr\n"
2459  "RxBadFormat\n"
2460  "RxBadLen\n"
2461  "RxBufOvrfl\n"
2462  "RxEBP\n"
2463  "RxFlowCtlErr\n"
2464  "RxICRCerr\n"
2465  "RxLPCRCerr\n"
2466  "RxVCRCerr\n"
2467  "RxInvalLen\n"
2468  "RxInvalPKey\n"
2469  "RxPktDropped\n"
2470  "TxBadLength\n"
2471  "TxDropped\n"
2472  "TxInvalLen\n"
2473  "TxUnderrun\n"
2474  "TxUnsupVL\n"
2475  ;
2476 
2477 #define _PORT_VIRT_FLAG 0x8000 /* "virtual", need adjustments */
2478 static const size_t portcntr6120indices[] = {
2505  cr_errslen,
2507  cr_txunsupvl,
2508 };
2509 
2510 /* do all the setup to make the counter reads efficient later */
2511 static void init_6120_cntrnames(struct qib_devdata *dd)
2512 {
2513  int i, j = 0;
2514  char *s;
2515 
2516  for (i = 0, s = (char *)cntr6120names; s && j <= dd->cfgctxts;
2517  i++) {
2518  /* we always have at least one counter before the egrovfl */
2519  if (!j && !strncmp("Ctxt0EgrOvfl", s + 1, 12))
2520  j = 1;
2521  s = strchr(s + 1, '\n');
2522  if (s && j)
2523  j++;
2524  }
2525  dd->cspec->ncntrs = i;
2526  if (!s)
2527  /* full list; size is without terminating null */
2528  dd->cspec->cntrnamelen = sizeof(cntr6120names) - 1;
2529  else
2530  dd->cspec->cntrnamelen = 1 + s - cntr6120names;
2531  dd->cspec->cntrs = kmalloc(dd->cspec->ncntrs
2532  * sizeof(u64), GFP_KERNEL);
2533  if (!dd->cspec->cntrs)
2534  qib_dev_err(dd, "Failed allocation for counters\n");
2535 
2536  for (i = 0, s = (char *)portcntr6120names; s; i++)
2537  s = strchr(s + 1, '\n');
2538  dd->cspec->nportcntrs = i - 1;
2539  dd->cspec->portcntrnamelen = sizeof(portcntr6120names) - 1;
2540  dd->cspec->portcntrs = kmalloc(dd->cspec->nportcntrs
2541  * sizeof(u64), GFP_KERNEL);
2542  if (!dd->cspec->portcntrs)
2543  qib_dev_err(dd, "Failed allocation for portcounters\n");
2544 }
2545 
2546 static u32 qib_read_6120cntrs(struct qib_devdata *dd, loff_t pos, char **namep,
2547  u64 **cntrp)
2548 {
2549  u32 ret;
2550 
2551  if (namep) {
2552  ret = dd->cspec->cntrnamelen;
2553  if (pos >= ret)
2554  ret = 0; /* final read after getting everything */
2555  else
2556  *namep = (char *)cntr6120names;
2557  } else {
2558  u64 *cntr = dd->cspec->cntrs;
2559  int i;
2560 
2561  ret = dd->cspec->ncntrs * sizeof(u64);
2562  if (!cntr || pos >= ret) {
2563  /* everything read, or couldn't get memory */
2564  ret = 0;
2565  goto done;
2566  }
2567  if (pos >= ret) {
2568  ret = 0; /* final read after getting everything */
2569  goto done;
2570  }
2571  *cntrp = cntr;
2572  for (i = 0; i < dd->cspec->ncntrs; i++)
2573  *cntr++ = read_6120_creg32(dd, cntr6120indices[i]);
2574  }
2575 done:
2576  return ret;
2577 }
2578 
2579 static u32 qib_read_6120portcntrs(struct qib_devdata *dd, loff_t pos, u32 port,
2580  char **namep, u64 **cntrp)
2581 {
2582  u32 ret;
2583 
2584  if (namep) {
2585  ret = dd->cspec->portcntrnamelen;
2586  if (pos >= ret)
2587  ret = 0; /* final read after getting everything */
2588  else
2589  *namep = (char *)portcntr6120names;
2590  } else {
2591  u64 *cntr = dd->cspec->portcntrs;
2592  struct qib_pportdata *ppd = &dd->pport[port];
2593  int i;
2594 
2595  ret = dd->cspec->nportcntrs * sizeof(u64);
2596  if (!cntr || pos >= ret) {
2597  /* everything read, or couldn't get memory */
2598  ret = 0;
2599  goto done;
2600  }
2601  *cntrp = cntr;
2602  for (i = 0; i < dd->cspec->nportcntrs; i++) {
2603  if (portcntr6120indices[i] & _PORT_VIRT_FLAG)
2604  *cntr++ = qib_portcntr_6120(ppd,
2605  portcntr6120indices[i] &
2606  ~_PORT_VIRT_FLAG);
2607  else
2608  *cntr++ = read_6120_creg32(dd,
2609  portcntr6120indices[i]);
2610  }
2611  }
2612 done:
2613  return ret;
2614 }
2615 
2616 static void qib_chk_6120_errormask(struct qib_devdata *dd)
2617 {
2618  static u32 fixed;
2619  u32 ctrl;
2620  unsigned long errormask;
2621  unsigned long hwerrs;
2622 
2623  if (!dd->cspec->errormask || !(dd->flags & QIB_INITTED))
2624  return;
2625 
2626  errormask = qib_read_kreg64(dd, kr_errmask);
2627 
2628  if (errormask == dd->cspec->errormask)
2629  return;
2630  fixed++;
2631 
2632  hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
2633  ctrl = qib_read_kreg32(dd, kr_control);
2634 
2635  qib_write_kreg(dd, kr_errmask,
2636  dd->cspec->errormask);
2637 
2638  if ((hwerrs & dd->cspec->hwerrmask) ||
2639  (ctrl & QLOGIC_IB_C_FREEZEMODE)) {
2640  qib_write_kreg(dd, kr_hwerrclear, 0ULL);
2641  qib_write_kreg(dd, kr_errclear, 0ULL);
2642  /* force re-interrupt of pending events, just in case */
2643  qib_write_kreg(dd, kr_intclear, 0ULL);
2644  qib_devinfo(dd->pcidev,
2645  "errormask fixed(%u) %lx->%lx, ctrl %x hwerr %lx\n",
2646  fixed, errormask, (unsigned long)dd->cspec->errormask,
2647  ctrl, hwerrs);
2648  }
2649 }
2650 
2659 static void qib_get_6120_faststats(unsigned long opaque)
2660 {
2661  struct qib_devdata *dd = (struct qib_devdata *) opaque;
2662  struct qib_pportdata *ppd = dd->pport;
2663  unsigned long flags;
2664  u64 traffic_wds;
2665 
2666  /*
2667  * don't access the chip while running diags, or memory diags can
2668  * fail
2669  */
2670  if (!(dd->flags & QIB_INITTED) || dd->diag_client)
2671  /* but re-arm the timer, for diags case; won't hurt other */
2672  goto done;
2673 
2674  /*
2675  * We now try to maintain an activity timer, based on traffic
2676  * exceeding a threshold, so we need to check the word-counts
2677  * even if they are 64-bit.
2678  */
2679  traffic_wds = qib_portcntr_6120(ppd, cr_wordsend) +
2680  qib_portcntr_6120(ppd, cr_wordrcv);
2681  spin_lock_irqsave(&dd->eep_st_lock, flags);
2682  traffic_wds -= dd->traffic_wds;
2683  dd->traffic_wds += traffic_wds;
2684  if (traffic_wds >= QIB_TRAFFIC_ACTIVE_THRESHOLD)
2685  atomic_add(5, &dd->active_time); /* S/B #define */
2686  spin_unlock_irqrestore(&dd->eep_st_lock, flags);
2687 
2688  qib_chk_6120_errormask(dd);
2689 done:
2690  mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
2691 }
2692 
2693 /* no interrupt fallback for these chips */
2694 static int qib_6120_nointr_fallback(struct qib_devdata *dd)
2695 {
2696  return 0;
2697 }
2698 
2699 /*
2700  * reset the XGXS (between serdes and IBC). Slightly less intrusive
2701  * than resetting the IBC or external link state, and useful in some
2702  * cases to cause some retraining. To do this right, we reset IBC
2703  * as well.
2704  */
2705 static void qib_6120_xgxs_reset(struct qib_pportdata *ppd)
2706 {
2707  u64 val, prev_val;
2708  struct qib_devdata *dd = ppd->dd;
2709 
2710  prev_val = qib_read_kreg64(dd, kr_xgxs_cfg);
2711  val = prev_val | QLOGIC_IB_XGXS_RESET;
2712  prev_val &= ~QLOGIC_IB_XGXS_RESET; /* be sure */
2713  qib_write_kreg(dd, kr_control,
2715  qib_write_kreg(dd, kr_xgxs_cfg, val);
2716  qib_read_kreg32(dd, kr_scratch);
2717  qib_write_kreg(dd, kr_xgxs_cfg, prev_val);
2718  qib_write_kreg(dd, kr_control, dd->control);
2719 }
2720 
2721 static int qib_6120_get_ib_cfg(struct qib_pportdata *ppd, int which)
2722 {
2723  int ret;
2724 
2725  switch (which) {
2726  case QIB_IB_CFG_LWID:
2727  ret = ppd->link_width_active;
2728  break;
2729 
2730  case QIB_IB_CFG_SPD:
2731  ret = ppd->link_speed_active;
2732  break;
2733 
2734  case QIB_IB_CFG_LWID_ENB:
2735  ret = ppd->link_width_enabled;
2736  break;
2737 
2738  case QIB_IB_CFG_SPD_ENB:
2739  ret = ppd->link_speed_enabled;
2740  break;
2741 
2742  case QIB_IB_CFG_OP_VLS:
2743  ret = ppd->vls_operational;
2744  break;
2745 
2747  ret = 0;
2748  break;
2749 
2750  case QIB_IB_CFG_VL_LOW_CAP:
2751  ret = 0;
2752  break;
2753 
2754  case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2755  ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2756  OverrunThreshold);
2757  break;
2758 
2759  case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2760  ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2761  PhyerrThreshold);
2762  break;
2763 
2764  case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2765  /* will only take effect when the link state changes */
2766  ret = (ppd->dd->cspec->ibcctrl &
2767  SYM_MASK(IBCCtrl, LinkDownDefaultState)) ?
2769  break;
2770 
2771  case QIB_IB_CFG_HRTBT: /* Get Heartbeat off/enable/auto */
2772  ret = 0; /* no heartbeat on this chip */
2773  break;
2774 
2775  case QIB_IB_CFG_PMA_TICKS:
2776  ret = 250; /* 1 usec. */
2777  break;
2778 
2779  default:
2780  ret = -EINVAL;
2781  break;
2782  }
2783  return ret;
2784 }
2785 
2786 /*
2787  * We assume range checking is already done, if needed.
2788  */
2789 static int qib_6120_set_ib_cfg(struct qib_pportdata *ppd, int which, u32 val)
2790 {
2791  struct qib_devdata *dd = ppd->dd;
2792  int ret = 0;
2793  u64 val64;
2794  u16 lcmd, licmd;
2795 
2796  switch (which) {
2797  case QIB_IB_CFG_LWID_ENB:
2798  ppd->link_width_enabled = val;
2799  break;
2800 
2801  case QIB_IB_CFG_SPD_ENB:
2802  ppd->link_speed_enabled = val;
2803  break;
2804 
2805  case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2806  val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2807  OverrunThreshold);
2808  if (val64 != val) {
2809  dd->cspec->ibcctrl &=
2810  ~SYM_MASK(IBCCtrl, OverrunThreshold);
2811  dd->cspec->ibcctrl |= (u64) val <<
2812  SYM_LSB(IBCCtrl, OverrunThreshold);
2813  qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2814  qib_write_kreg(dd, kr_scratch, 0);
2815  }
2816  break;
2817 
2818  case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2819  val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2820  PhyerrThreshold);
2821  if (val64 != val) {
2822  dd->cspec->ibcctrl &=
2823  ~SYM_MASK(IBCCtrl, PhyerrThreshold);
2824  dd->cspec->ibcctrl |= (u64) val <<
2825  SYM_LSB(IBCCtrl, PhyerrThreshold);
2826  qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2827  qib_write_kreg(dd, kr_scratch, 0);
2828  }
2829  break;
2830 
2831  case QIB_IB_CFG_PKEYS: /* update pkeys */
2832  val64 = (u64) ppd->pkeys[0] | ((u64) ppd->pkeys[1] << 16) |
2833  ((u64) ppd->pkeys[2] << 32) |
2834  ((u64) ppd->pkeys[3] << 48);
2835  qib_write_kreg(dd, kr_partitionkey, val64);
2836  break;
2837 
2838  case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2839  /* will only take effect when the link state changes */
2840  if (val == IB_LINKINITCMD_POLL)
2841  dd->cspec->ibcctrl &=
2842  ~SYM_MASK(IBCCtrl, LinkDownDefaultState);
2843  else /* SLEEP */
2844  dd->cspec->ibcctrl |=
2845  SYM_MASK(IBCCtrl, LinkDownDefaultState);
2846  qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2847  qib_write_kreg(dd, kr_scratch, 0);
2848  break;
2849 
2850  case QIB_IB_CFG_MTU: /* update the MTU in IBC */
2851  /*
2852  * Update our housekeeping variables, and set IBC max
2853  * size, same as init code; max IBC is max we allow in
2854  * buffer, less the qword pbc, plus 1 for ICRC, in dwords
2855  * Set even if it's unchanged, print debug message only
2856  * on changes.
2857  */
2858  val = (ppd->ibmaxlen >> 2) + 1;
2859  dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, MaxPktLen);
2860  dd->cspec->ibcctrl |= (u64)val <<
2861  SYM_LSB(IBCCtrl, MaxPktLen);
2862  qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2863  qib_write_kreg(dd, kr_scratch, 0);
2864  break;
2865 
2866  case QIB_IB_CFG_LSTATE: /* set the IB link state */
2867  switch (val & 0xffff0000) {
2868  case IB_LINKCMD_DOWN:
2870  if (!dd->cspec->ibdeltainprog) {
2871  dd->cspec->ibdeltainprog = 1;
2872  dd->cspec->ibsymsnap =
2873  read_6120_creg32(dd, cr_ibsymbolerr);
2874  dd->cspec->iblnkerrsnap =
2875  read_6120_creg32(dd, cr_iblinkerrrecov);
2876  }
2877  break;
2878 
2879  case IB_LINKCMD_ARMED:
2881  break;
2882 
2883  case IB_LINKCMD_ACTIVE:
2885  break;
2886 
2887  default:
2888  ret = -EINVAL;
2889  qib_dev_err(dd, "bad linkcmd req 0x%x\n", val >> 16);
2890  goto bail;
2891  }
2892  switch (val & 0xffff) {
2893  case IB_LINKINITCMD_NOP:
2894  licmd = 0;
2895  break;
2896 
2897  case IB_LINKINITCMD_POLL:
2899  break;
2900 
2901  case IB_LINKINITCMD_SLEEP:
2903  break;
2904 
2907  break;
2908 
2909  default:
2910  ret = -EINVAL;
2911  qib_dev_err(dd, "bad linkinitcmd req 0x%x\n",
2912  val & 0xffff);
2913  goto bail;
2914  }
2915  qib_set_ib_6120_lstate(ppd, lcmd, licmd);
2916  goto bail;
2917 
2918  case QIB_IB_CFG_HRTBT:
2919  ret = -EINVAL;
2920  break;
2921 
2922  default:
2923  ret = -EINVAL;
2924  }
2925 bail:
2926  return ret;
2927 }
2928 
2929 static int qib_6120_set_loopback(struct qib_pportdata *ppd, const char *what)
2930 {
2931  int ret = 0;
2932  if (!strncmp(what, "ibc", 3)) {
2933  ppd->dd->cspec->ibcctrl |= SYM_MASK(IBCCtrl, Loopback);
2934  qib_devinfo(ppd->dd->pcidev, "Enabling IB%u:%u IBC loopback\n",
2935  ppd->dd->unit, ppd->port);
2936  } else if (!strncmp(what, "off", 3)) {
2937  ppd->dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, Loopback);
2938  qib_devinfo(ppd->dd->pcidev,
2939  "Disabling IB%u:%u IBC loopback (normal)\n",
2940  ppd->dd->unit, ppd->port);
2941  } else
2942  ret = -EINVAL;
2943  if (!ret) {
2944  qib_write_kreg(ppd->dd, kr_ibcctrl, ppd->dd->cspec->ibcctrl);
2945  qib_write_kreg(ppd->dd, kr_scratch, 0);
2946  }
2947  return ret;
2948 }
2949 
2950 static void pma_6120_timer(unsigned long data)
2951 {
2952  struct qib_pportdata *ppd = (struct qib_pportdata *)data;
2953  struct qib_chip_specific *cs = ppd->dd->cspec;
2954  struct qib_ibport *ibp = &ppd->ibport_data;
2955  unsigned long flags;
2956 
2957  spin_lock_irqsave(&ibp->lock, flags);
2960  qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2961  &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2962  mod_timer(&cs->pma_timer,
2963  jiffies + usecs_to_jiffies(ibp->pma_sample_interval));
2964  } else if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
2965  u64 ta, tb, tc, td, te;
2966 
2968  qib_snapshot_counters(ppd, &ta, &tb, &tc, &td, &te);
2969 
2970  cs->sword = ta - cs->sword;
2971  cs->rword = tb - cs->rword;
2972  cs->spkts = tc - cs->spkts;
2973  cs->rpkts = td - cs->rpkts;
2974  cs->xmit_wait = te - cs->xmit_wait;
2975  }
2976  spin_unlock_irqrestore(&ibp->lock, flags);
2977 }
2978 
2979 /*
2980  * Note that the caller has the ibp->lock held.
2981  */
2982 static void qib_set_cntr_6120_sample(struct qib_pportdata *ppd, u32 intv,
2983  u32 start)
2984 {
2985  struct qib_chip_specific *cs = ppd->dd->cspec;
2986 
2987  if (start && intv) {
2989  mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(start));
2990  } else if (intv) {
2992  qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2993  &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2994  mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(intv));
2995  } else {
2997  cs->sword = 0;
2998  cs->rword = 0;
2999  cs->spkts = 0;
3000  cs->rpkts = 0;
3001  cs->xmit_wait = 0;
3002  }
3003 }
3004 
3005 static u32 qib_6120_iblink_state(u64 ibcs)
3006 {
3007  u32 state = (u32)SYM_FIELD(ibcs, IBCStatus, LinkState);
3008 
3009  switch (state) {
3010  case IB_6120_L_STATE_INIT:
3011  state = IB_PORT_INIT;
3012  break;
3013  case IB_6120_L_STATE_ARM:
3014  state = IB_PORT_ARMED;
3015  break;
3017  /* fall through */
3019  state = IB_PORT_ACTIVE;
3020  break;
3021  default: /* fall through */
3022  case IB_6120_L_STATE_DOWN:
3023  state = IB_PORT_DOWN;
3024  break;
3025  }
3026  return state;
3027 }
3028 
3029 /* returns the IBTA port state, rather than the IBC link training state */
3030 static u8 qib_6120_phys_portstate(u64 ibcs)
3031 {
3032  u8 state = (u8)SYM_FIELD(ibcs, IBCStatus, LinkTrainingState);
3033  return qib_6120_physportstate[state];
3034 }
3035 
3036 static int qib_6120_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs)
3037 {
3038  unsigned long flags;
3039 
3040  spin_lock_irqsave(&ppd->lflags_lock, flags);
3041  ppd->lflags &= ~QIBL_IB_FORCE_NOTIFY;
3042  spin_unlock_irqrestore(&ppd->lflags_lock, flags);
3043 
3044  if (ibup) {
3045  if (ppd->dd->cspec->ibdeltainprog) {
3046  ppd->dd->cspec->ibdeltainprog = 0;
3047  ppd->dd->cspec->ibsymdelta +=
3048  read_6120_creg32(ppd->dd, cr_ibsymbolerr) -
3049  ppd->dd->cspec->ibsymsnap;
3050  ppd->dd->cspec->iblnkerrdelta +=
3051  read_6120_creg32(ppd->dd, cr_iblinkerrrecov) -
3052  ppd->dd->cspec->iblnkerrsnap;
3053  }
3054  qib_hol_init(ppd);
3055  } else {
3056  ppd->dd->cspec->lli_counter = 0;
3057  if (!ppd->dd->cspec->ibdeltainprog) {
3058  ppd->dd->cspec->ibdeltainprog = 1;
3059  ppd->dd->cspec->ibsymsnap =
3060  read_6120_creg32(ppd->dd, cr_ibsymbolerr);
3061  ppd->dd->cspec->iblnkerrsnap =
3062  read_6120_creg32(ppd->dd, cr_iblinkerrrecov);
3063  }
3064  qib_hol_down(ppd);
3065  }
3066 
3067  qib_6120_setup_setextled(ppd, ibup);
3068 
3069  return 0;
3070 }
3071 
3072 /* Does read/modify/write to appropriate registers to
3073  * set output and direction bits selected by mask.
3074  * these are in their canonical postions (e.g. lsb of
3075  * dir will end up in D48 of extctrl on existing chips).
3076  * returns contents of GP Inputs.
3077  */
3078 static int gpio_6120_mod(struct qib_devdata *dd, u32 out, u32 dir, u32 mask)
3079 {
3080  u64 read_val, new_out;
3081  unsigned long flags;
3082 
3083  if (mask) {
3084  /* some bits being written, lock access to GPIO */
3085  dir &= mask;
3086  out &= mask;
3087  spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
3088  dd->cspec->extctrl &= ~((u64)mask << SYM_LSB(EXTCtrl, GPIOOe));
3089  dd->cspec->extctrl |= ((u64) dir << SYM_LSB(EXTCtrl, GPIOOe));
3090  new_out = (dd->cspec->gpio_out & ~mask) | out;
3091 
3092  qib_write_kreg(dd, kr_extctrl, dd->cspec->extctrl);
3093  qib_write_kreg(dd, kr_gpio_out, new_out);
3094  dd->cspec->gpio_out = new_out;
3095  spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
3096  }
3097  /*
3098  * It is unlikely that a read at this time would get valid
3099  * data on a pin whose direction line was set in the same
3100  * call to this function. We include the read here because
3101  * that allows us to potentially combine a change on one pin with
3102  * a read on another, and because the old code did something like
3103  * this.
3104  */
3105  read_val = qib_read_kreg64(dd, kr_extstatus);
3106  return SYM_FIELD(read_val, EXTStatus, GPIOIn);
3107 }
3108 
3109 /*
3110  * Read fundamental info we need to use the chip. These are
3111  * the registers that describe chip capabilities, and are
3112  * saved in shadow registers.
3113  */
3114 static void get_6120_chip_params(struct qib_devdata *dd)
3115 {
3116  u64 val;
3117  u32 piobufs;
3118  int mtu;
3119 
3120  dd->uregbase = qib_read_kreg32(dd, kr_userregbase);
3121 
3122  dd->rcvtidcnt = qib_read_kreg32(dd, kr_rcvtidcnt);
3123  dd->rcvtidbase = qib_read_kreg32(dd, kr_rcvtidbase);
3124  dd->rcvegrbase = qib_read_kreg32(dd, kr_rcvegrbase);
3125  dd->palign = qib_read_kreg32(dd, kr_palign);
3126  dd->piobufbase = qib_read_kreg64(dd, kr_sendpiobufbase);
3127  dd->pio2k_bufbase = dd->piobufbase & 0xffffffff;
3128 
3129  dd->rcvhdrcnt = qib_read_kreg32(dd, kr_rcvegrcnt);
3130 
3131  val = qib_read_kreg64(dd, kr_sendpiosize);
3132  dd->piosize2k = val & ~0U;
3133  dd->piosize4k = val >> 32;
3134 
3135  mtu = ib_mtu_enum_to_int(qib_ibmtu);
3136  if (mtu == -1)
3137  mtu = QIB_DEFAULT_MTU;
3138  dd->pport->ibmtu = (u32)mtu;
3139 
3140  val = qib_read_kreg64(dd, kr_sendpiobufcnt);
3141  dd->piobcnt2k = val & ~0U;
3142  dd->piobcnt4k = val >> 32;
3143  dd->last_pio = dd->piobcnt4k + dd->piobcnt2k - 1;
3144  /* these may be adjusted in init_chip_wc_pat() */
3145  dd->pio2kbase = (u32 __iomem *)
3146  (((char __iomem *)dd->kregbase) + dd->pio2k_bufbase);
3147  if (dd->piobcnt4k) {
3148  dd->pio4kbase = (u32 __iomem *)
3149  (((char __iomem *) dd->kregbase) +
3150  (dd->piobufbase >> 32));
3151  /*
3152  * 4K buffers take 2 pages; we use roundup just to be
3153  * paranoid; we calculate it once here, rather than on
3154  * ever buf allocate
3155  */
3156  dd->align4k = ALIGN(dd->piosize4k, dd->palign);
3157  }
3158 
3159  piobufs = dd->piobcnt4k + dd->piobcnt2k;
3160 
3161  dd->pioavregs = ALIGN(piobufs, sizeof(u64) * BITS_PER_BYTE / 2) /
3162  (sizeof(u64) * BITS_PER_BYTE / 2);
3163 }
3164 
3165 /*
3166  * The chip base addresses in cspec and cpspec have to be set
3167  * after possible init_chip_wc_pat(), rather than in
3168  * get_6120_chip_params(), so split out as separate function
3169  */
3170 static void set_6120_baseaddrs(struct qib_devdata *dd)
3171 {
3172  u32 cregbase;
3173  cregbase = qib_read_kreg32(dd, kr_counterregbase);
3174  dd->cspec->cregbase = (u64 __iomem *)
3175  ((char __iomem *) dd->kregbase + cregbase);
3176 
3177  dd->egrtidbase = (u64 __iomem *)
3178  ((char __iomem *) dd->kregbase + dd->rcvegrbase);
3179 }
3180 
3181 /*
3182  * Write the final few registers that depend on some of the
3183  * init setup. Done late in init, just before bringing up
3184  * the serdes.
3185  */
3186 static int qib_late_6120_initreg(struct qib_devdata *dd)
3187 {
3188  int ret = 0;
3189  u64 val;
3190 
3191  qib_write_kreg(dd, kr_rcvhdrentsize, dd->rcvhdrentsize);
3192  qib_write_kreg(dd, kr_rcvhdrsize, dd->rcvhdrsize);
3193  qib_write_kreg(dd, kr_rcvhdrcnt, dd->rcvhdrcnt);
3194  qib_write_kreg(dd, kr_sendpioavailaddr, dd->pioavailregs_phys);
3195  val = qib_read_kreg64(dd, kr_sendpioavailaddr);
3196  if (val != dd->pioavailregs_phys) {
3197  qib_dev_err(dd,
3198  "Catastrophic software error, SendPIOAvailAddr written as %lx, read back as %llx\n",
3199  (unsigned long) dd->pioavailregs_phys,
3200  (unsigned long long) val);
3201  ret = -EINVAL;
3202  }
3203  return ret;
3204 }
3205 
3206 static int init_6120_variables(struct qib_devdata *dd)
3207 {
3208  int ret = 0;
3209  struct qib_pportdata *ppd;
3210  u32 sbufs;
3211 
3212  ppd = (struct qib_pportdata *)(dd + 1);
3213  dd->pport = ppd;
3214  dd->num_pports = 1;
3215 
3216  dd->cspec = (struct qib_chip_specific *)(ppd + dd->num_pports);
3217  ppd->cpspec = NULL; /* not used in this chip */
3218 
3219  spin_lock_init(&dd->cspec->kernel_tid_lock);
3220  spin_lock_init(&dd->cspec->user_tid_lock);
3221  spin_lock_init(&dd->cspec->rcvmod_lock);
3222  spin_lock_init(&dd->cspec->gpio_lock);
3223 
3224  /* we haven't yet set QIB_PRESENT, so use read directly */
3225  dd->revision = readq(&dd->kregbase[kr_revision]);
3226 
3227  if ((dd->revision & 0xffffffffU) == 0xffffffffU) {
3228  qib_dev_err(dd,
3229  "Revision register read failure, giving up initialization\n");
3230  ret = -ENODEV;
3231  goto bail;
3232  }
3233  dd->flags |= QIB_PRESENT; /* now register routines work */
3234 
3235  dd->majrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3236  ChipRevMajor);
3237  dd->minrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3238  ChipRevMinor);
3239 
3240  get_6120_chip_params(dd);
3241  pe_boardname(dd); /* fill in boardname */
3242 
3243  /*
3244  * GPIO bits for TWSI data and clock,
3245  * used for serial EEPROM.
3246  */
3250 
3251  if (qib_unordered_wc())
3252  dd->flags |= QIB_PIO_FLUSH_WC;
3253 
3254  /*
3255  * EEPROM error log 0 is TXE Parity errors. 1 is RXE Parity.
3256  * 2 is Some Misc, 3 is reserved for future.
3257  */
3258  dd->eep_st_masks[0].hwerrs_to_log = HWE_MASK(TXEMemParityErr);
3259 
3260  /* Ignore errors in PIO/PBC on systems with unordered write-combining */
3261  if (qib_unordered_wc())
3262  dd->eep_st_masks[0].hwerrs_to_log &= ~TXE_PIO_PARITY;
3263 
3264  dd->eep_st_masks[1].hwerrs_to_log = HWE_MASK(RXEMemParityErr);
3265 
3266  dd->eep_st_masks[2].errs_to_log = ERR_MASK(ResetNegated);
3267 
3268  qib_init_pportdata(ppd, dd, 0, 1);
3273  /* these can't change for this chip, so set once */
3276  ppd->vls_supported = IB_VL_VL0;
3277  ppd->vls_operational = ppd->vls_supported;
3278 
3281  dd->rhf_offset = 0;
3282 
3283  /* we always allocate at least 2048 bytes for eager buffers */
3284  ret = ib_mtu_enum_to_int(qib_ibmtu);
3285  dd->rcvegrbufsize = ret != -1 ? max(ret, 2048) : QIB_DEFAULT_MTU;
3288 
3289  qib_6120_tidtemplate(dd);
3290 
3291  /*
3292  * We can request a receive interrupt for 1 or
3293  * more packets from current offset. For now, we set this
3294  * up for a single packet.
3295  */
3296  dd->rhdrhead_intr_off = 1ULL << 32;
3297 
3298  /* setup the stats timer; the add_timer is done at end of init */
3299  init_timer(&dd->stats_timer);
3300  dd->stats_timer.function = qib_get_6120_faststats;
3301  dd->stats_timer.data = (unsigned long) dd;
3302 
3303  init_timer(&dd->cspec->pma_timer);
3304  dd->cspec->pma_timer.function = pma_6120_timer;
3305  dd->cspec->pma_timer.data = (unsigned long) ppd;
3306 
3307  dd->ureg_align = qib_read_kreg32(dd, kr_palign);
3308 
3309  dd->piosize2kmax_dwords = dd->piosize2k >> 2;
3310  qib_6120_config_ctxts(dd);
3311  qib_set_ctxtcnt(dd);
3312 
3313  if (qib_wc_pat) {
3314  ret = init_chip_wc_pat(dd, 0);
3315  if (ret)
3316  goto bail;
3317  }
3318  set_6120_baseaddrs(dd); /* set chip access pointers now */
3319 
3320  ret = 0;
3321  if (qib_mini_init)
3322  goto bail;
3323 
3324  qib_num_cfg_vls = 1; /* if any 6120's, only one VL */
3325 
3326  ret = qib_create_ctxts(dd);
3327  init_6120_cntrnames(dd);
3328 
3329  /* use all of 4KB buffers for the kernel, otherwise 16 */
3330  sbufs = dd->piobcnt4k ? dd->piobcnt4k : 16;
3331 
3332  dd->lastctxt_piobuf = dd->piobcnt2k + dd->piobcnt4k - sbufs;
3333  dd->pbufsctxt = dd->lastctxt_piobuf /
3334  (dd->cfgctxts - dd->first_user_ctxt);
3335 
3336  if (ret)
3337  goto bail;
3338 bail:
3339  return ret;
3340 }
3341 
3342 /*
3343  * For this chip, we want to use the same buffer every time
3344  * when we are trying to bring the link up (they are always VL15
3345  * packets). At that link state the packet should always go out immediately
3346  * (or at least be discarded at the tx interface if the link is down).
3347  * If it doesn't, and the buffer isn't available, that means some other
3348  * sender has gotten ahead of us, and is preventing our packet from going
3349  * out. In that case, we flush all packets, and try again. If that still
3350  * fails, we fail the request, and hope things work the next time around.
3351  *
3352  * We don't need very complicated heuristics on whether the packet had
3353  * time to go out or not, since even at SDR 1X, it goes out in very short
3354  * time periods, covered by the chip reads done here and as part of the
3355  * flush.
3356  */
3357 static u32 __iomem *get_6120_link_buf(struct qib_pportdata *ppd, u32 *bnum)
3358 {
3359  u32 __iomem *buf;
3360  u32 lbuf = ppd->dd->piobcnt2k + ppd->dd->piobcnt4k - 1;
3361 
3362  /*
3363  * always blip to get avail list updated, since it's almost
3364  * always needed, and is fairly cheap.
3365  */
3366  sendctrl_6120_mod(ppd->dd->pport, QIB_SENDCTRL_AVAIL_BLIP);
3367  qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3368  buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3369  if (buf)
3370  goto done;
3371 
3372  sendctrl_6120_mod(ppd, QIB_SENDCTRL_DISARM_ALL | QIB_SENDCTRL_FLUSH |
3373  QIB_SENDCTRL_AVAIL_BLIP);
3374  ppd->dd->upd_pio_shadow = 1; /* update our idea of what's busy */
3375  qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3376  buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3377 done:
3378  return buf;
3379 }
3380 
3381 static u32 __iomem *qib_6120_getsendbuf(struct qib_pportdata *ppd, u64 pbc,
3382  u32 *pbufnum)
3383 {
3384  u32 first, last, plen = pbc & QIB_PBC_LENGTH_MASK;
3385  struct qib_devdata *dd = ppd->dd;
3386  u32 __iomem *buf;
3387 
3388  if (((pbc >> 32) & PBC_6120_VL15_SEND_CTRL) &&
3390  buf = get_6120_link_buf(ppd, pbufnum);
3391  else {
3392 
3393  if ((plen + 1) > dd->piosize2kmax_dwords)
3394  first = dd->piobcnt2k;
3395  else
3396  first = 0;
3397  /* try 4k if all 2k busy, so same last for both sizes */
3398  last = dd->piobcnt2k + dd->piobcnt4k - 1;
3399  buf = qib_getsendbuf_range(dd, pbufnum, first, last);
3400  }
3401  return buf;
3402 }
3403 
3404 static int init_sdma_6120_regs(struct qib_pportdata *ppd)
3405 {
3406  return -ENODEV;
3407 }
3408 
3409 static u16 qib_sdma_6120_gethead(struct qib_pportdata *ppd)
3410 {
3411  return 0;
3412 }
3413 
3414 static int qib_sdma_6120_busy(struct qib_pportdata *ppd)
3415 {
3416  return 0;
3417 }
3418 
3419 static void qib_sdma_update_6120_tail(struct qib_pportdata *ppd, u16 tail)
3420 {
3421 }
3422 
3423 static void qib_6120_sdma_sendctrl(struct qib_pportdata *ppd, unsigned op)
3424 {
3425 }
3426 
3427 static void qib_sdma_set_6120_desc_cnt(struct qib_pportdata *ppd, unsigned cnt)
3428 {
3429 }
3430 
3431 /*
3432  * the pbc doesn't need a VL15 indicator, but we need it for link_buf.
3433  * The chip ignores the bit if set.
3434  */
3435 static u32 qib_6120_setpbc_control(struct qib_pportdata *ppd, u32 plen,
3436  u8 srate, u8 vl)
3437 {
3438  return vl == 15 ? PBC_6120_VL15_SEND_CTRL : 0;
3439 }
3440 
3441 static void qib_6120_initvl15_bufs(struct qib_devdata *dd)
3442 {
3443 }
3444 
3445 static void qib_6120_init_ctxt(struct qib_ctxtdata *rcd)
3446 {
3447  rcd->rcvegrcnt = rcd->dd->rcvhdrcnt;
3448  rcd->rcvegr_tid_base = rcd->ctxt * rcd->rcvegrcnt;
3449 }
3450 
3451 static void qib_6120_txchk_change(struct qib_devdata *dd, u32 start,
3452  u32 len, u32 avail, struct qib_ctxtdata *rcd)
3453 {
3454 }
3455 
3456 static void writescratch(struct qib_devdata *dd, u32 val)
3457 {
3458  (void) qib_write_kreg(dd, kr_scratch, val);
3459 }
3460 
3461 static int qib_6120_tempsense_rd(struct qib_devdata *dd, int regnum)
3462 {
3463  return -ENXIO;
3464 }
3465 
3466 /* Dummy function, as 6120 boards never disable EEPROM Write */
3467 static int qib_6120_eeprom_wen(struct qib_devdata *dd, int wen)
3468 {
3469  return 1;
3470 }
3471 
3484  const struct pci_device_id *ent)
3485 {
3486  struct qib_devdata *dd;
3487  int ret;
3488 
3489  dd = qib_alloc_devdata(pdev, sizeof(struct qib_pportdata) +
3490  sizeof(struct qib_chip_specific));
3491  if (IS_ERR(dd))
3492  goto bail;
3493 
3494  dd->f_bringup_serdes = qib_6120_bringup_serdes;
3495  dd->f_cleanup = qib_6120_setup_cleanup;
3496  dd->f_clear_tids = qib_6120_clear_tids;
3497  dd->f_free_irq = qib_6120_free_irq;
3498  dd->f_get_base_info = qib_6120_get_base_info;
3499  dd->f_get_msgheader = qib_6120_get_msgheader;
3500  dd->f_getsendbuf = qib_6120_getsendbuf;
3501  dd->f_gpio_mod = gpio_6120_mod;
3502  dd->f_eeprom_wen = qib_6120_eeprom_wen;
3503  dd->f_hdrqempty = qib_6120_hdrqempty;
3504  dd->f_ib_updown = qib_6120_ib_updown;
3505  dd->f_init_ctxt = qib_6120_init_ctxt;
3506  dd->f_initvl15_bufs = qib_6120_initvl15_bufs;
3507  dd->f_intr_fallback = qib_6120_nointr_fallback;
3508  dd->f_late_initreg = qib_late_6120_initreg;
3509  dd->f_setpbc_control = qib_6120_setpbc_control;
3510  dd->f_portcntr = qib_portcntr_6120;
3511  dd->f_put_tid = (dd->minrev >= 2) ?
3512  qib_6120_put_tid_2 :
3513  qib_6120_put_tid;
3514  dd->f_quiet_serdes = qib_6120_quiet_serdes;
3515  dd->f_rcvctrl = rcvctrl_6120_mod;
3516  dd->f_read_cntrs = qib_read_6120cntrs;
3517  dd->f_read_portcntrs = qib_read_6120portcntrs;
3518  dd->f_reset = qib_6120_setup_reset;
3519  dd->f_init_sdma_regs = init_sdma_6120_regs;
3520  dd->f_sdma_busy = qib_sdma_6120_busy;
3521  dd->f_sdma_gethead = qib_sdma_6120_gethead;
3522  dd->f_sdma_sendctrl = qib_6120_sdma_sendctrl;
3523  dd->f_sdma_set_desc_cnt = qib_sdma_set_6120_desc_cnt;
3524  dd->f_sdma_update_tail = qib_sdma_update_6120_tail;
3525  dd->f_sendctrl = sendctrl_6120_mod;
3526  dd->f_set_armlaunch = qib_set_6120_armlaunch;
3527  dd->f_set_cntr_sample = qib_set_cntr_6120_sample;
3528  dd->f_iblink_state = qib_6120_iblink_state;
3529  dd->f_ibphys_portstate = qib_6120_phys_portstate;
3530  dd->f_get_ib_cfg = qib_6120_get_ib_cfg;
3531  dd->f_set_ib_cfg = qib_6120_set_ib_cfg;
3532  dd->f_set_ib_loopback = qib_6120_set_loopback;
3533  dd->f_set_intr_state = qib_6120_set_intr_state;
3534  dd->f_setextled = qib_6120_setup_setextled;
3535  dd->f_txchk_change = qib_6120_txchk_change;
3536  dd->f_update_usrhead = qib_update_6120_usrhead;
3537  dd->f_wantpiobuf_intr = qib_wantpiobuf_6120_intr;
3538  dd->f_xgxs_reset = qib_6120_xgxs_reset;
3539  dd->f_writescratch = writescratch;
3540  dd->f_tempsense_rd = qib_6120_tempsense_rd;
3541  /*
3542  * Do remaining pcie setup and save pcie values in dd.
3543  * Any error printing is already done by the init code.
3544  * On return, we have the chip mapped and accessible,
3545  * but chip registers are not set up until start of
3546  * init_6120_variables.
3547  */
3548  ret = qib_pcie_ddinit(dd, pdev, ent);
3549  if (ret < 0)
3550  goto bail_free;
3551 
3552  /* initialize chip-specific variables */
3553  ret = init_6120_variables(dd);
3554  if (ret)
3555  goto bail_cleanup;
3556 
3557  if (qib_mini_init)
3558  goto bail;
3559 
3560  if (qib_pcie_params(dd, 8, NULL, NULL))
3561  qib_dev_err(dd,
3562  "Failed to setup PCIe or interrupts; continuing anyway\n");
3563  dd->cspec->irq = pdev->irq; /* save IRQ */
3564 
3565  /* clear diagctrl register, in case diags were running and crashed */
3566  qib_write_kreg(dd, kr_hwdiagctrl, 0);
3567 
3568  if (qib_read_kreg64(dd, kr_hwerrstatus) &
3569  QLOGIC_IB_HWE_SERDESPLLFAILED)
3570  qib_write_kreg(dd, kr_hwerrclear,
3571  QLOGIC_IB_HWE_SERDESPLLFAILED);
3572 
3573  /* setup interrupt handler (interrupt type handled above) */
3574  qib_setup_6120_interrupt(dd);
3575  /* Note that qpn_mask is set by qib_6120_config_ctxts() first */
3576  qib_6120_init_hwerrors(dd);
3577 
3578  goto bail;
3579 
3580 bail_cleanup:
3581  qib_pcie_ddcleanup(dd);
3582 bail_free:
3583  qib_free_devdata(dd);
3584  dd = ERR_PTR(ret);
3585 bail:
3586  return dd;
3587 }