1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright 2019 Joyent, Inc.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
33 */
34
35 #include "ixgbe_sw.h"
36
37 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
38
39 /*
40 * Local function protoypes
41 */
42 static int ixgbe_register_mac(ixgbe_t *);
43 static int ixgbe_identify_hardware(ixgbe_t *);
44 static int ixgbe_regs_map(ixgbe_t *);
45 static void ixgbe_init_properties(ixgbe_t *);
46 static int ixgbe_init_driver_settings(ixgbe_t *);
47 static void ixgbe_init_locks(ixgbe_t *);
48 static void ixgbe_destroy_locks(ixgbe_t *);
49 static int ixgbe_init(ixgbe_t *);
50 static int ixgbe_chip_start(ixgbe_t *);
51 static void ixgbe_chip_stop(ixgbe_t *);
52 static int ixgbe_reset(ixgbe_t *);
53 static void ixgbe_tx_clean(ixgbe_t *);
54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
56 static int ixgbe_alloc_rings(ixgbe_t *);
57 static void ixgbe_free_rings(ixgbe_t *);
58 static int ixgbe_alloc_rx_data(ixgbe_t *);
59 static void ixgbe_free_rx_data(ixgbe_t *);
60 static void ixgbe_setup_rings(ixgbe_t *);
61 static void ixgbe_setup_rx(ixgbe_t *);
62 static void ixgbe_setup_tx(ixgbe_t *);
63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
65 static void ixgbe_setup_rss(ixgbe_t *);
66 static void ixgbe_setup_vmdq(ixgbe_t *);
67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
68 static void ixgbe_setup_rss_table(ixgbe_t *);
69 static void ixgbe_init_unicst(ixgbe_t *);
70 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
71 static void ixgbe_setup_multicst(ixgbe_t *);
72 static void ixgbe_get_hw_state(ixgbe_t *);
73 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
74 static void ixgbe_get_conf(ixgbe_t *);
75 static void ixgbe_init_params(ixgbe_t *);
76 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
77 static void ixgbe_driver_link_check(ixgbe_t *);
78 static void ixgbe_sfp_check(void *);
79 static void ixgbe_overtemp_check(void *);
80 static void ixgbe_phy_check(void *);
81 static void ixgbe_link_timer(void *);
82 static void ixgbe_local_timer(void *);
83 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
84 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
87 static boolean_t is_valid_mac_addr(uint8_t *);
88 static boolean_t ixgbe_stall_check(ixgbe_t *);
89 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
91 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
92 static int ixgbe_alloc_intrs(ixgbe_t *);
93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
94 static int ixgbe_add_intr_handlers(ixgbe_t *);
95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
101 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
102 static void ixgbe_setup_adapter_vector(ixgbe_t *);
103 static void ixgbe_rem_intr_handlers(ixgbe_t *);
104 static void ixgbe_rem_intrs(ixgbe_t *);
105 static int ixgbe_enable_intrs(ixgbe_t *);
106 static int ixgbe_disable_intrs(ixgbe_t *);
107 static uint_t ixgbe_intr_legacy(void *, void *);
108 static uint_t ixgbe_intr_msi(void *, void *);
109 static uint_t ixgbe_intr_msix(void *, void *);
110 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
111 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
112 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
113 static void ixgbe_get_driver_control(struct ixgbe_hw *);
114 static int ixgbe_addmac(void *, const uint8_t *);
115 static int ixgbe_remmac(void *, const uint8_t *);
116 static void ixgbe_release_driver_control(struct ixgbe_hw *);
117
118 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
119 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
120 static int ixgbe_resume(dev_info_t *);
121 static int ixgbe_suspend(dev_info_t *);
122 static int ixgbe_quiesce(dev_info_t *);
123 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
124 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
125 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
126 static int ixgbe_intr_cb_register(ixgbe_t *);
127 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
128
129 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
130 const void *impl_data);
131 static void ixgbe_fm_init(ixgbe_t *);
132 static void ixgbe_fm_fini(ixgbe_t *);
133
134 char *ixgbe_priv_props[] = {
135 "_tx_copy_thresh",
136 "_tx_recycle_thresh",
137 "_tx_overload_thresh",
138 "_tx_resched_thresh",
139 "_rx_copy_thresh",
140 "_rx_limit_per_intr",
141 "_intr_throttling",
142 "_adv_pause_cap",
143 "_adv_asym_pause_cap",
144 NULL
145 };
146
147 #define IXGBE_MAX_PRIV_PROPS \
148 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
149
150 static struct cb_ops ixgbe_cb_ops = {
151 nulldev, /* cb_open */
152 nulldev, /* cb_close */
153 nodev, /* cb_strategy */
154 nodev, /* cb_print */
155 nodev, /* cb_dump */
156 nodev, /* cb_read */
157 nodev, /* cb_write */
158 nodev, /* cb_ioctl */
159 nodev, /* cb_devmap */
160 nodev, /* cb_mmap */
161 nodev, /* cb_segmap */
162 nochpoll, /* cb_chpoll */
163 ddi_prop_op, /* cb_prop_op */
164 NULL, /* cb_stream */
165 D_MP | D_HOTPLUG, /* cb_flag */
166 CB_REV, /* cb_rev */
167 nodev, /* cb_aread */
168 nodev /* cb_awrite */
169 };
170
171 static struct dev_ops ixgbe_dev_ops = {
172 DEVO_REV, /* devo_rev */
173 0, /* devo_refcnt */
174 NULL, /* devo_getinfo */
175 nulldev, /* devo_identify */
176 nulldev, /* devo_probe */
177 ixgbe_attach, /* devo_attach */
178 ixgbe_detach, /* devo_detach */
179 nodev, /* devo_reset */
180 &ixgbe_cb_ops, /* devo_cb_ops */
181 NULL, /* devo_bus_ops */
182 ddi_power, /* devo_power */
183 ixgbe_quiesce, /* devo_quiesce */
184 };
185
186 static struct modldrv ixgbe_modldrv = {
187 &mod_driverops, /* Type of module. This one is a driver */
188 ixgbe_ident, /* Discription string */
189 &ixgbe_dev_ops /* driver ops */
190 };
191
192 static struct modlinkage ixgbe_modlinkage = {
193 MODREV_1, &ixgbe_modldrv, NULL
194 };
195
196 /*
197 * Access attributes for register mapping
198 */
199 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
200 DDI_DEVICE_ATTR_V1,
201 DDI_STRUCTURE_LE_ACC,
202 DDI_STRICTORDER_ACC,
203 DDI_FLAGERR_ACC
204 };
205
206 /*
207 * Loopback property
208 */
209 static lb_property_t lb_normal = {
210 normal, "normal", IXGBE_LB_NONE
211 };
212
213 static lb_property_t lb_mac = {
214 internal, "MAC", IXGBE_LB_INTERNAL_MAC
215 };
216
217 static lb_property_t lb_external = {
218 external, "External", IXGBE_LB_EXTERNAL
219 };
220
221 #define IXGBE_M_CALLBACK_FLAGS \
222 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
223
224 static mac_callbacks_t ixgbe_m_callbacks = {
225 IXGBE_M_CALLBACK_FLAGS,
226 ixgbe_m_stat,
227 ixgbe_m_start,
228 ixgbe_m_stop,
229 ixgbe_m_promisc,
230 ixgbe_m_multicst,
231 NULL,
232 NULL,
233 NULL,
234 ixgbe_m_ioctl,
235 ixgbe_m_getcapab,
236 NULL,
237 NULL,
238 ixgbe_m_setprop,
239 ixgbe_m_getprop,
240 ixgbe_m_propinfo
241 };
242
243 /*
244 * Initialize capabilities of each supported adapter type
245 */
246 static adapter_info_t ixgbe_82598eb_cap = {
247 64, /* maximum number of rx queues */
248 1, /* minimum number of rx queues */
249 64, /* default number of rx queues */
250 16, /* maximum number of rx groups */
251 1, /* minimum number of rx groups */
252 1, /* default number of rx groups */
253 32, /* maximum number of tx queues */
254 1, /* minimum number of tx queues */
255 8, /* default number of tx queues */
256 16366, /* maximum MTU size */
257 0xFFFF, /* maximum interrupt throttle rate */
258 0, /* minimum interrupt throttle rate */
259 200, /* default interrupt throttle rate */
260 18, /* maximum total msix vectors */
261 16, /* maximum number of ring vectors */
262 2, /* maximum number of other vectors */
263 IXGBE_EICR_LSC, /* "other" interrupt types handled */
264 0, /* "other" interrupt types enable mask */
265 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
266 | IXGBE_FLAG_RSS_CAPABLE
267 | IXGBE_FLAG_VMDQ_CAPABLE)
268 };
269
270 static adapter_info_t ixgbe_82599eb_cap = {
271 128, /* maximum number of rx queues */
272 1, /* minimum number of rx queues */
273 128, /* default number of rx queues */
274 64, /* maximum number of rx groups */
275 1, /* minimum number of rx groups */
276 1, /* default number of rx groups */
277 128, /* maximum number of tx queues */
278 1, /* minimum number of tx queues */
279 8, /* default number of tx queues */
280 15500, /* maximum MTU size */
281 0xFF8, /* maximum interrupt throttle rate */
282 0, /* minimum interrupt throttle rate */
283 200, /* default interrupt throttle rate */
284 64, /* maximum total msix vectors */
285 16, /* maximum number of ring vectors */
286 2, /* maximum number of other vectors */
287 (IXGBE_EICR_LSC
288 | IXGBE_EICR_GPI_SDP1
289 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
290
291 (IXGBE_SDP1_GPIEN
292 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
293
294 (IXGBE_FLAG_DCA_CAPABLE
295 | IXGBE_FLAG_RSS_CAPABLE
296 | IXGBE_FLAG_VMDQ_CAPABLE
297 | IXGBE_FLAG_RSC_CAPABLE
298 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
299 };
300
301 static adapter_info_t ixgbe_X540_cap = {
302 128, /* maximum number of rx queues */
303 1, /* minimum number of rx queues */
304 128, /* default number of rx queues */
305 64, /* maximum number of rx groups */
306 1, /* minimum number of rx groups */
307 1, /* default number of rx groups */
308 128, /* maximum number of tx queues */
309 1, /* minimum number of tx queues */
310 8, /* default number of tx queues */
311 15500, /* maximum MTU size */
312 0xFF8, /* maximum interrupt throttle rate */
313 0, /* minimum interrupt throttle rate */
314 200, /* default interrupt throttle rate */
315 64, /* maximum total msix vectors */
316 16, /* maximum number of ring vectors */
317 2, /* maximum number of other vectors */
318 (IXGBE_EICR_LSC
319 | IXGBE_EICR_GPI_SDP1_X540
320 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
321
322 (IXGBE_SDP1_GPIEN_X540
323 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
324
325 (IXGBE_FLAG_DCA_CAPABLE
326 | IXGBE_FLAG_RSS_CAPABLE
327 | IXGBE_FLAG_VMDQ_CAPABLE
328 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
329 };
330
331 static adapter_info_t ixgbe_X550_cap = {
332 128, /* maximum number of rx queues */
333 1, /* minimum number of rx queues */
334 128, /* default number of rx queues */
335 64, /* maximum number of rx groups */
336 1, /* minimum number of rx groups */
337 1, /* default number of rx groups */
338 128, /* maximum number of tx queues */
339 1, /* minimum number of tx queues */
340 8, /* default number of tx queues */
341 15500, /* maximum MTU size */
342 0xFF8, /* maximum interrupt throttle rate */
343 0, /* minimum interrupt throttle rate */
344 0x200, /* default interrupt throttle rate */
345 64, /* maximum total msix vectors */
346 16, /* maximum number of ring vectors */
347 2, /* maximum number of other vectors */
348 IXGBE_EICR_LSC, /* "other" interrupt types handled */
349 0, /* "other" interrupt types enable mask */
350 (IXGBE_FLAG_RSS_CAPABLE
351 | IXGBE_FLAG_VMDQ_CAPABLE
352 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
353 };
354
355 /*
356 * Module Initialization Functions.
357 */
358
359 int
360 _init(void)
361 {
362 int status;
363
364 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
365
366 status = mod_install(&ixgbe_modlinkage);
367
368 if (status != DDI_SUCCESS) {
369 mac_fini_ops(&ixgbe_dev_ops);
370 }
371
372 return (status);
373 }
374
375 int
376 _fini(void)
377 {
378 int status;
379
380 status = mod_remove(&ixgbe_modlinkage);
381
382 if (status == DDI_SUCCESS) {
383 mac_fini_ops(&ixgbe_dev_ops);
384 }
385
386 return (status);
387 }
388
389 int
390 _info(struct modinfo *modinfop)
391 {
392 int status;
393
394 status = mod_info(&ixgbe_modlinkage, modinfop);
395
396 return (status);
397 }
398
399 /*
400 * ixgbe_attach - Driver attach.
401 *
402 * This function is the device specific initialization entry
403 * point. This entry point is required and must be written.
404 * The DDI_ATTACH command must be provided in the attach entry
405 * point. When attach() is called with cmd set to DDI_ATTACH,
406 * all normal kernel services (such as kmem_alloc(9F)) are
407 * available for use by the driver.
408 *
409 * The attach() function will be called once for each instance
410 * of the device on the system with cmd set to DDI_ATTACH.
411 * Until attach() succeeds, the only driver entry points which
412 * may be called are open(9E) and getinfo(9E).
413 */
414 static int
415 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
416 {
417 ixgbe_t *ixgbe;
418 struct ixgbe_osdep *osdep;
419 struct ixgbe_hw *hw;
420 int instance;
421 char taskqname[32];
422
423 /*
424 * Check the command and perform corresponding operations
425 */
426 switch (cmd) {
427 default:
428 return (DDI_FAILURE);
429
430 case DDI_RESUME:
431 return (ixgbe_resume(devinfo));
432
433 case DDI_ATTACH:
434 break;
435 }
436
437 /* Get the device instance */
438 instance = ddi_get_instance(devinfo);
439
440 /* Allocate memory for the instance data structure */
441 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
442
443 ixgbe->dip = devinfo;
444 ixgbe->instance = instance;
445
446 hw = &ixgbe->hw;
447 osdep = &ixgbe->osdep;
448 hw->back = osdep;
449 osdep->ixgbe = ixgbe;
450
451 /* Attach the instance pointer to the dev_info data structure */
452 ddi_set_driver_private(devinfo, ixgbe);
453
454 /*
455 * Initialize for FMA support
456 */
457 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
458 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
459 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
460 ixgbe_fm_init(ixgbe);
461 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
462
463 /*
464 * Map PCI config space registers
465 */
466 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
467 ixgbe_error(ixgbe, "Failed to map PCI configurations");
468 goto attach_fail;
469 }
470 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
471
472 /*
473 * Identify the chipset family
474 */
475 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
476 ixgbe_error(ixgbe, "Failed to identify hardware");
477 goto attach_fail;
478 }
479
480 /*
481 * Map device registers
482 */
483 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
484 ixgbe_error(ixgbe, "Failed to map device registers");
485 goto attach_fail;
486 }
487 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
488
489 /*
490 * Initialize driver parameters
491 */
492 ixgbe_init_properties(ixgbe);
493 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
494
495 /*
496 * Register interrupt callback
497 */
498 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
499 ixgbe_error(ixgbe, "Failed to register interrupt callback");
500 goto attach_fail;
501 }
502
503 /*
504 * Allocate interrupts
505 */
506 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
507 ixgbe_error(ixgbe, "Failed to allocate interrupts");
508 goto attach_fail;
509 }
510 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
511
512 /*
513 * Allocate rx/tx rings based on the ring numbers.
514 * The actual numbers of rx/tx rings are decided by the number of
515 * allocated interrupt vectors, so we should allocate the rings after
516 * interrupts are allocated.
517 */
518 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
519 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
520 goto attach_fail;
521 }
522 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
523
524 /*
525 * Map rings to interrupt vectors
526 */
527 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
528 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
529 goto attach_fail;
530 }
531
532 /*
533 * Add interrupt handlers
534 */
535 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
536 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
537 goto attach_fail;
538 }
539 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
540
541 /*
542 * Create a taskq for sfp-change
543 */
544 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
545 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
546 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
547 ixgbe_error(ixgbe, "sfp_taskq create failed");
548 goto attach_fail;
549 }
550 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
551
552 /*
553 * Create a taskq for over-temp
554 */
555 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
556 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
557 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
558 ixgbe_error(ixgbe, "overtemp_taskq create failed");
559 goto attach_fail;
560 }
561 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
562
563 /*
564 * Create a taskq for processing external PHY interrupts
565 */
566 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance);
567 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname,
568 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
569 ixgbe_error(ixgbe, "phy_taskq create failed");
570 goto attach_fail;
571 }
572 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ;
573
574 /*
575 * Initialize driver parameters
576 */
577 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
578 ixgbe_error(ixgbe, "Failed to initialize driver settings");
579 goto attach_fail;
580 }
581
582 /*
583 * Initialize mutexes for this device.
584 * Do this before enabling the interrupt handler and
585 * register the softint to avoid the condition where
586 * interrupt handler can try using uninitialized mutex.
587 */
588 ixgbe_init_locks(ixgbe);
589 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
590
591 /*
592 * Initialize chipset hardware
593 */
594 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
595 ixgbe_error(ixgbe, "Failed to initialize adapter");
596 goto attach_fail;
597 }
598 ixgbe->link_check_complete = B_FALSE;
599 ixgbe->link_check_hrtime = gethrtime() +
600 (IXGBE_LINK_UP_TIME * 100000000ULL);
601 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
602
603 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
604 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
605 goto attach_fail;
606 }
607
608 /*
609 * Initialize adapter capabilities
610 */
611 ixgbe_init_params(ixgbe);
612
613 /*
614 * Initialize statistics
615 */
616 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
617 ixgbe_error(ixgbe, "Failed to initialize statistics");
618 goto attach_fail;
619 }
620 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
621
622 /*
623 * Register the driver to the MAC
624 */
625 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
626 ixgbe_error(ixgbe, "Failed to register MAC");
627 goto attach_fail;
628 }
629 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
630 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
631
632 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
633 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
634 if (ixgbe->periodic_id == 0) {
635 ixgbe_error(ixgbe, "Failed to add the link check timer");
636 goto attach_fail;
637 }
638 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
639
640 /*
641 * Now that mutex locks are initialized, and the chip is also
642 * initialized, enable interrupts.
643 */
644 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
645 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
646 goto attach_fail;
647 }
648 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
649
650 ixgbe_log(ixgbe, "%s", ixgbe_ident);
651 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
652
653 return (DDI_SUCCESS);
654
655 attach_fail:
656 ixgbe_unconfigure(devinfo, ixgbe);
657 return (DDI_FAILURE);
658 }
659
660 /*
661 * ixgbe_detach - Driver detach.
662 *
663 * The detach() function is the complement of the attach routine.
664 * If cmd is set to DDI_DETACH, detach() is used to remove the
665 * state associated with a given instance of a device node
666 * prior to the removal of that instance from the system.
667 *
668 * The detach() function will be called once for each instance
669 * of the device for which there has been a successful attach()
670 * once there are no longer any opens on the device.
671 *
672 * Interrupts routine are disabled, All memory allocated by this
673 * driver are freed.
674 */
675 static int
676 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
677 {
678 ixgbe_t *ixgbe;
679
680 /*
681 * Check detach command
682 */
683 switch (cmd) {
684 default:
685 return (DDI_FAILURE);
686
687 case DDI_SUSPEND:
688 return (ixgbe_suspend(devinfo));
689
690 case DDI_DETACH:
691 break;
692 }
693
694 /*
695 * Get the pointer to the driver private data structure
696 */
697 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
698 if (ixgbe == NULL)
699 return (DDI_FAILURE);
700
701 /*
702 * If the device is still running, it needs to be stopped first.
703 * This check is necessary because under some specific circumstances,
704 * the detach routine can be called without stopping the interface
705 * first.
706 */
707 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
708 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
709 mutex_enter(&ixgbe->gen_lock);
710 ixgbe_stop(ixgbe, B_TRUE);
711 mutex_exit(&ixgbe->gen_lock);
712 /* Disable and stop the watchdog timer */
713 ixgbe_disable_watchdog_timer(ixgbe);
714 }
715
716 /*
717 * Check if there are still rx buffers held by the upper layer.
718 * If so, fail the detach.
719 */
720 if (!ixgbe_rx_drain(ixgbe))
721 return (DDI_FAILURE);
722
723 /*
724 * Do the remaining unconfigure routines
725 */
726 ixgbe_unconfigure(devinfo, ixgbe);
727
728 return (DDI_SUCCESS);
729 }
730
731 /*
732 * quiesce(9E) entry point.
733 *
734 * This function is called when the system is single-threaded at high
735 * PIL with preemption disabled. Therefore, this function must not be
736 * blocked.
737 *
738 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
739 * DDI_FAILURE indicates an error condition and should almost never happen.
740 */
741 static int
742 ixgbe_quiesce(dev_info_t *devinfo)
743 {
744 ixgbe_t *ixgbe;
745 struct ixgbe_hw *hw;
746
747 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
748
749 if (ixgbe == NULL)
750 return (DDI_FAILURE);
751
752 hw = &ixgbe->hw;
753
754 /*
755 * Disable the adapter interrupts
756 */
757 ixgbe_disable_adapter_interrupts(ixgbe);
758
759 /*
760 * Tell firmware driver is no longer in control
761 */
762 ixgbe_release_driver_control(hw);
763
764 /*
765 * Reset the chipset
766 */
767 (void) ixgbe_reset_hw(hw);
768
769 /*
770 * Reset PHY
771 */
772 (void) ixgbe_reset_phy(hw);
773
774 return (DDI_SUCCESS);
775 }
776
777 static void
778 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
779 {
780 /*
781 * Disable interrupt
782 */
783 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
784 (void) ixgbe_disable_intrs(ixgbe);
785 }
786
787 /*
788 * remove the link check timer
789 */
790 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
791 if (ixgbe->periodic_id != NULL) {
792 ddi_periodic_delete(ixgbe->periodic_id);
793 ixgbe->periodic_id = NULL;
794 }
795 }
796
797 /*
798 * Unregister MAC
799 */
800 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
801 (void) mac_unregister(ixgbe->mac_hdl);
802 }
803
804 /*
805 * Free statistics
806 */
807 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
808 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
809 }
810
811 /*
812 * Remove interrupt handlers
813 */
814 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
815 ixgbe_rem_intr_handlers(ixgbe);
816 }
817
818 /*
819 * Remove taskq for sfp-status-change
820 */
821 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
822 ddi_taskq_destroy(ixgbe->sfp_taskq);
823 }
824
825 /*
826 * Remove taskq for over-temp
827 */
828 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
829 ddi_taskq_destroy(ixgbe->overtemp_taskq);
830 }
831
832 /*
833 * Remove taskq for external PHYs
834 */
835 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) {
836 ddi_taskq_destroy(ixgbe->phy_taskq);
837 }
838
839 /*
840 * Remove interrupts
841 */
842 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
843 ixgbe_rem_intrs(ixgbe);
844 }
845
846 /*
847 * Unregister interrupt callback handler
848 */
849 if (ixgbe->cb_hdl != NULL) {
850 (void) ddi_cb_unregister(ixgbe->cb_hdl);
851 }
852
853 /*
854 * Remove driver properties
855 */
856 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
857 (void) ddi_prop_remove_all(devinfo);
858 }
859
860 /*
861 * Stop the chipset
862 */
863 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
864 mutex_enter(&ixgbe->gen_lock);
865 ixgbe_chip_stop(ixgbe);
866 mutex_exit(&ixgbe->gen_lock);
867 }
868
869 /*
870 * Free register handle
871 */
872 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
873 if (ixgbe->osdep.reg_handle != NULL)
874 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
875 }
876
877 /*
878 * Free PCI config handle
879 */
880 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
881 if (ixgbe->osdep.cfg_handle != NULL)
882 pci_config_teardown(&ixgbe->osdep.cfg_handle);
883 }
884
885 /*
886 * Free locks
887 */
888 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
889 ixgbe_destroy_locks(ixgbe);
890 }
891
892 /*
893 * Free the rx/tx rings
894 */
895 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
896 ixgbe_free_rings(ixgbe);
897 }
898
899 /*
900 * Unregister FMA capabilities
901 */
902 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
903 ixgbe_fm_fini(ixgbe);
904 }
905
906 /*
907 * Free the driver data structure
908 */
909 kmem_free(ixgbe, sizeof (ixgbe_t));
910
911 ddi_set_driver_private(devinfo, NULL);
912 }
913
914 /*
915 * ixgbe_register_mac - Register the driver and its function pointers with
916 * the GLD interface.
917 */
918 static int
919 ixgbe_register_mac(ixgbe_t *ixgbe)
920 {
921 struct ixgbe_hw *hw = &ixgbe->hw;
922 mac_register_t *mac;
923 int status;
924
925 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
926 return (IXGBE_FAILURE);
927
928 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
929 mac->m_driver = ixgbe;
930 mac->m_dip = ixgbe->dip;
931 mac->m_src_addr = hw->mac.addr;
932 mac->m_callbacks = &ixgbe_m_callbacks;
933 mac->m_min_sdu = 0;
934 mac->m_max_sdu = ixgbe->default_mtu;
935 mac->m_margin = VLAN_TAGSZ;
936 mac->m_priv_props = ixgbe_priv_props;
937 mac->m_v12n = MAC_VIRT_LEVEL1;
938
939 status = mac_register(mac, &ixgbe->mac_hdl);
940
941 mac_free(mac);
942
943 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
944 }
945
946 /*
947 * ixgbe_identify_hardware - Identify the type of the chipset.
948 */
949 static int
950 ixgbe_identify_hardware(ixgbe_t *ixgbe)
951 {
952 struct ixgbe_hw *hw = &ixgbe->hw;
953 struct ixgbe_osdep *osdep = &ixgbe->osdep;
954
955 /*
956 * Get the device id
957 */
958 hw->vendor_id =
959 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
960 hw->device_id =
961 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
962 hw->revision_id =
963 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
964 hw->subsystem_device_id =
965 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
966 hw->subsystem_vendor_id =
967 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
968
969 /*
970 * Set the mac type of the adapter based on the device id
971 */
972 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
973 return (IXGBE_FAILURE);
974 }
975
976 /*
977 * Install adapter capabilities
978 */
979 switch (hw->mac.type) {
980 case ixgbe_mac_82598EB:
981 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
982 ixgbe->capab = &ixgbe_82598eb_cap;
983
984 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
985 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
986 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
987 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
988 }
989 break;
990
991 case ixgbe_mac_82599EB:
992 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
993 ixgbe->capab = &ixgbe_82599eb_cap;
994
995 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
996 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
997 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
998 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
999 }
1000 break;
1001
1002 case ixgbe_mac_X540:
1003 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
1004 ixgbe->capab = &ixgbe_X540_cap;
1005 /*
1006 * For now, X540 is all set in its capab structure.
1007 * As other X540 variants show up, things can change here.
1008 */
1009 break;
1010
1011 case ixgbe_mac_X550:
1012 case ixgbe_mac_X550EM_x:
1013 case ixgbe_mac_X550EM_a:
1014 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
1015 ixgbe->capab = &ixgbe_X550_cap;
1016
1017 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1018 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
1019 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
1020 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP ||
1021 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP_N) {
1022 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
1023 }
1024
1025 /*
1026 * Link detection on X552 SFP+ and X552/X557-AT
1027 */
1028 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1029 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
1030 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
1031 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
1032 ixgbe->capab->other_intr |=
1033 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
1034 }
1035 if (hw->phy.type == ixgbe_phy_x550em_ext_t) {
1036 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540;
1037 }
1038 break;
1039
1040 default:
1041 IXGBE_DEBUGLOG_1(ixgbe,
1042 "adapter not supported in ixgbe_identify_hardware(): %d\n",
1043 hw->mac.type);
1044 return (IXGBE_FAILURE);
1045 }
1046
1047 return (IXGBE_SUCCESS);
1048 }
1049
1050 /*
1051 * ixgbe_regs_map - Map the device registers.
1052 *
1053 */
1054 static int
1055 ixgbe_regs_map(ixgbe_t *ixgbe)
1056 {
1057 dev_info_t *devinfo = ixgbe->dip;
1058 struct ixgbe_hw *hw = &ixgbe->hw;
1059 struct ixgbe_osdep *osdep = &ixgbe->osdep;
1060 off_t mem_size;
1061
1062 /*
1063 * First get the size of device registers to be mapped.
1064 */
1065 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
1066 != DDI_SUCCESS) {
1067 return (IXGBE_FAILURE);
1068 }
1069
1070 /*
1071 * Call ddi_regs_map_setup() to map registers
1072 */
1073 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
1074 (caddr_t *)&hw->hw_addr, 0,
1075 mem_size, &ixgbe_regs_acc_attr,
1076 &osdep->reg_handle)) != DDI_SUCCESS) {
1077 return (IXGBE_FAILURE);
1078 }
1079
1080 return (IXGBE_SUCCESS);
1081 }
1082
1083 /*
1084 * ixgbe_init_properties - Initialize driver properties.
1085 */
1086 static void
1087 ixgbe_init_properties(ixgbe_t *ixgbe)
1088 {
1089 /*
1090 * Get conf file properties, including link settings
1091 * jumbo frames, ring number, descriptor number, etc.
1092 */
1093 ixgbe_get_conf(ixgbe);
1094 }
1095
1096 /*
1097 * ixgbe_init_driver_settings - Initialize driver settings.
1098 *
1099 * The settings include hardware function pointers, bus information,
1100 * rx/tx rings settings, link state, and any other parameters that
1101 * need to be setup during driver initialization.
1102 */
1103 static int
1104 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1105 {
1106 struct ixgbe_hw *hw = &ixgbe->hw;
1107 dev_info_t *devinfo = ixgbe->dip;
1108 ixgbe_rx_ring_t *rx_ring;
1109 ixgbe_rx_group_t *rx_group;
1110 ixgbe_tx_ring_t *tx_ring;
1111 uint32_t rx_size;
1112 uint32_t tx_size;
1113 uint32_t ring_per_group;
1114 int i;
1115
1116 /*
1117 * Initialize chipset specific hardware function pointers
1118 */
1119 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1120 return (IXGBE_FAILURE);
1121 }
1122
1123 /*
1124 * Get the system page size
1125 */
1126 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1127
1128 /*
1129 * Set rx buffer size
1130 *
1131 * The IP header alignment room is counted in the calculation.
1132 * The rx buffer size is in unit of 1K that is required by the
1133 * chipset hardware.
1134 */
1135 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1136 ixgbe->rx_buf_size = ((rx_size >> 10) +
1137 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1138
1139 /*
1140 * Set tx buffer size
1141 */
1142 tx_size = ixgbe->max_frame_size;
1143 ixgbe->tx_buf_size = ((tx_size >> 10) +
1144 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1145
1146 /*
1147 * Initialize rx/tx rings/groups parameters
1148 */
1149 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1150 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1151 rx_ring = &ixgbe->rx_rings[i];
1152 rx_ring->index = i;
1153 rx_ring->ixgbe = ixgbe;
1154 rx_ring->group_index = i / ring_per_group;
1155 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1156 }
1157
1158 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1159 rx_group = &ixgbe->rx_groups[i];
1160 rx_group->index = i;
1161 rx_group->ixgbe = ixgbe;
1162 }
1163
1164 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1165 tx_ring = &ixgbe->tx_rings[i];
1166 tx_ring->index = i;
1167 tx_ring->ixgbe = ixgbe;
1168 if (ixgbe->tx_head_wb_enable)
1169 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1170 else
1171 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1172
1173 tx_ring->ring_size = ixgbe->tx_ring_size;
1174 tx_ring->free_list_size = ixgbe->tx_ring_size +
1175 (ixgbe->tx_ring_size >> 1);
1176 }
1177
1178 /*
1179 * Initialize values of interrupt throttling rate
1180 */
1181 for (i = 1; i < MAX_INTR_VECTOR; i++)
1182 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1183
1184 /*
1185 * The initial link state should be "unknown"
1186 */
1187 ixgbe->link_state = LINK_STATE_UNKNOWN;
1188
1189 return (IXGBE_SUCCESS);
1190 }
1191
1192 /*
1193 * ixgbe_init_locks - Initialize locks.
1194 */
1195 static void
1196 ixgbe_init_locks(ixgbe_t *ixgbe)
1197 {
1198 ixgbe_rx_ring_t *rx_ring;
1199 ixgbe_tx_ring_t *tx_ring;
1200 int i;
1201
1202 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1203 rx_ring = &ixgbe->rx_rings[i];
1204 mutex_init(&rx_ring->rx_lock, NULL,
1205 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1206 }
1207
1208 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1209 tx_ring = &ixgbe->tx_rings[i];
1210 mutex_init(&tx_ring->tx_lock, NULL,
1211 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1212 mutex_init(&tx_ring->recycle_lock, NULL,
1213 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1214 mutex_init(&tx_ring->tcb_head_lock, NULL,
1215 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1216 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1217 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1218 }
1219
1220 mutex_init(&ixgbe->gen_lock, NULL,
1221 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1222
1223 mutex_init(&ixgbe->watchdog_lock, NULL,
1224 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1225 }
1226
1227 /*
1228 * ixgbe_destroy_locks - Destroy locks.
1229 */
1230 static void
1231 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1232 {
1233 ixgbe_rx_ring_t *rx_ring;
1234 ixgbe_tx_ring_t *tx_ring;
1235 int i;
1236
1237 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1238 rx_ring = &ixgbe->rx_rings[i];
1239 mutex_destroy(&rx_ring->rx_lock);
1240 }
1241
1242 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1243 tx_ring = &ixgbe->tx_rings[i];
1244 mutex_destroy(&tx_ring->tx_lock);
1245 mutex_destroy(&tx_ring->recycle_lock);
1246 mutex_destroy(&tx_ring->tcb_head_lock);
1247 mutex_destroy(&tx_ring->tcb_tail_lock);
1248 }
1249
1250 mutex_destroy(&ixgbe->gen_lock);
1251 mutex_destroy(&ixgbe->watchdog_lock);
1252 }
1253
1254 /*
1255 * We need to try and determine which LED index in hardware corresponds to the
1256 * link/activity LED. This is the one that'll be overwritten when we perform
1257 * GLDv3 LED activity.
1258 */
1259 static void
1260 ixgbe_led_init(ixgbe_t *ixgbe)
1261 {
1262 uint32_t reg, i;
1263 struct ixgbe_hw *hw = &ixgbe->hw;
1264
1265 reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL);
1266 for (i = 0; i < 4; i++) {
1267 if (((reg >> IXGBE_LED_MODE_SHIFT(i)) &
1268 IXGBE_LED_MODE_MASK_BASE) == IXGBE_LED_LINK_ACTIVE) {
1269 ixgbe->ixgbe_led_index = i;
1270 return;
1271 }
1272 }
1273
1274 /*
1275 * If we couldn't determine this, we use the default for various MACs
1276 * based on information Intel has inserted into other drivers over the
1277 * years.
1278 */
1279 switch (hw->mac.type) {
1280 case ixgbe_mac_X550EM_a:
1281 ixgbe->ixgbe_led_index = 0;
1282 break;
1283 case ixgbe_mac_X550EM_x:
1284 ixgbe->ixgbe_led_index = 1;
1285 break;
1286 default:
1287 ixgbe->ixgbe_led_index = 2;
1288 break;
1289 }
1290 }
1291
1292 static int
1293 ixgbe_resume(dev_info_t *devinfo)
1294 {
1295 ixgbe_t *ixgbe;
1296 int i;
1297
1298 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1299 if (ixgbe == NULL)
1300 return (DDI_FAILURE);
1301
1302 mutex_enter(&ixgbe->gen_lock);
1303
1304 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1305 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1306 mutex_exit(&ixgbe->gen_lock);
1307 return (DDI_FAILURE);
1308 }
1309
1310 /*
1311 * Enable and start the watchdog timer
1312 */
1313 ixgbe_enable_watchdog_timer(ixgbe);
1314 }
1315
1316 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1317
1318 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1319 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1320 mac_tx_ring_update(ixgbe->mac_hdl,
1321 ixgbe->tx_rings[i].ring_handle);
1322 }
1323 }
1324
1325 mutex_exit(&ixgbe->gen_lock);
1326
1327 return (DDI_SUCCESS);
1328 }
1329
1330 static int
1331 ixgbe_suspend(dev_info_t *devinfo)
1332 {
1333 ixgbe_t *ixgbe;
1334
1335 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1336 if (ixgbe == NULL)
1337 return (DDI_FAILURE);
1338
1339 mutex_enter(&ixgbe->gen_lock);
1340
1341 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1342 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1343 mutex_exit(&ixgbe->gen_lock);
1344 return (DDI_SUCCESS);
1345 }
1346 ixgbe_stop(ixgbe, B_FALSE);
1347
1348 mutex_exit(&ixgbe->gen_lock);
1349
1350 /*
1351 * Disable and stop the watchdog timer
1352 */
1353 ixgbe_disable_watchdog_timer(ixgbe);
1354
1355 return (DDI_SUCCESS);
1356 }
1357
1358 /*
1359 * ixgbe_init - Initialize the device.
1360 */
1361 static int
1362 ixgbe_init(ixgbe_t *ixgbe)
1363 {
1364 struct ixgbe_hw *hw = &ixgbe->hw;
1365 u8 pbanum[IXGBE_PBANUM_LENGTH];
1366 int rv;
1367
1368 mutex_enter(&ixgbe->gen_lock);
1369
1370 /*
1371 * Configure/Initialize hardware
1372 */
1373 rv = ixgbe_init_hw(hw);
1374 if (rv != IXGBE_SUCCESS) {
1375 switch (rv) {
1376
1377 /*
1378 * The first three errors are not prohibitive to us progressing
1379 * further, and are maily advisory in nature. In the case of a
1380 * SFP module not being present or not deemed supported by the
1381 * common code, we adivse the operator of this fact but carry on
1382 * instead of failing hard, as SFPs can be inserted or replaced
1383 * while the driver is running. In the case of a unknown error,
1384 * we fail-hard, logging the reason and emitting a FMA event.
1385 */
1386 case IXGBE_ERR_EEPROM_VERSION:
1387 ixgbe_error(ixgbe,
1388 "This Intel 10Gb Ethernet device is pre-release and"
1389 " contains outdated firmware. Please contact your"
1390 " hardware vendor for a replacement.");
1391 break;
1392 case IXGBE_ERR_SFP_NOT_PRESENT:
1393 ixgbe_error(ixgbe,
1394 "No SFP+ module detected on this interface. Please "
1395 "install a supported SFP+ module for this "
1396 "interface to become operational.");
1397 break;
1398 case IXGBE_ERR_SFP_NOT_SUPPORTED:
1399 ixgbe_error(ixgbe,
1400 "Unsupported SFP+ module detected. Please replace "
1401 "it with a supported SFP+ module per Intel "
1402 "documentation, or bypass this check with "
1403 "allow_unsupported_sfp=1 in ixgbe.conf.");
1404 break;
1405 default:
1406 ixgbe_error(ixgbe,
1407 "Failed to initialize hardware. ixgbe_init_hw "
1408 "returned %d", rv);
1409 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1410 goto init_fail;
1411 }
1412 }
1413
1414 /*
1415 * Need to init eeprom before validating the checksum.
1416 */
1417 if (ixgbe_init_eeprom_params(hw) < 0) {
1418 ixgbe_error(ixgbe,
1419 "Unable to intitialize the eeprom interface.");
1420 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1421 goto init_fail;
1422 }
1423
1424 /*
1425 * NVM validation
1426 */
1427 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1428 /*
1429 * Some PCI-E parts fail the first check due to
1430 * the link being in sleep state. Call it again,
1431 * if it fails a second time it's a real issue.
1432 */
1433 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1434 ixgbe_error(ixgbe,
1435 "Invalid NVM checksum. Please contact "
1436 "the vendor to update the NVM.");
1437 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1438 goto init_fail;
1439 }
1440 }
1441
1442 /*
1443 * Setup default flow control thresholds - enable/disable
1444 * & flow control type is controlled by ixgbe.conf
1445 */
1446 hw->fc.high_water[0] = DEFAULT_FCRTH;
1447 hw->fc.low_water[0] = DEFAULT_FCRTL;
1448 hw->fc.pause_time = DEFAULT_FCPAUSE;
1449 hw->fc.send_xon = B_TRUE;
1450
1451 /*
1452 * Initialize flow control
1453 */
1454 (void) ixgbe_start_hw(hw);
1455
1456 /*
1457 * Initialize link settings
1458 */
1459 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1460
1461 /*
1462 * Initialize the chipset hardware
1463 */
1464 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1465 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1466 goto init_fail;
1467 }
1468
1469 /*
1470 * Read identifying information and place in devinfo.
1471 */
1472 pbanum[0] = '\0';
1473 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1474 if (*pbanum != '\0') {
1475 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1476 "printed-board-assembly", (char *)pbanum);
1477 }
1478
1479 /*
1480 * Determine LED index.
1481 */
1482 ixgbe_led_init(ixgbe);
1483
1484 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1485 goto init_fail;
1486 }
1487
1488 mutex_exit(&ixgbe->gen_lock);
1489 return (IXGBE_SUCCESS);
1490
1491 init_fail:
1492 /*
1493 * Reset PHY
1494 */
1495 (void) ixgbe_reset_phy(hw);
1496
1497 mutex_exit(&ixgbe->gen_lock);
1498 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1499 return (IXGBE_FAILURE);
1500 }
1501
1502 /*
1503 * ixgbe_chip_start - Initialize and start the chipset hardware.
1504 */
1505 static int
1506 ixgbe_chip_start(ixgbe_t *ixgbe)
1507 {
1508 struct ixgbe_hw *hw = &ixgbe->hw;
1509 int i;
1510
1511 ASSERT(mutex_owned(&ixgbe->gen_lock));
1512
1513 /*
1514 * Get the mac address
1515 * This function should handle SPARC case correctly.
1516 */
1517 if (!ixgbe_find_mac_address(ixgbe)) {
1518 ixgbe_error(ixgbe, "Failed to get the mac address");
1519 return (IXGBE_FAILURE);
1520 }
1521
1522 /*
1523 * Validate the mac address
1524 */
1525 (void) ixgbe_init_rx_addrs(hw);
1526 if (!is_valid_mac_addr(hw->mac.addr)) {
1527 ixgbe_error(ixgbe, "Invalid mac address");
1528 return (IXGBE_FAILURE);
1529 }
1530
1531 /*
1532 * Re-enable relaxed ordering for performance. It is disabled
1533 * by default in the hardware init.
1534 */
1535 if (ixgbe->relax_order_enable == B_TRUE)
1536 ixgbe_enable_relaxed_ordering(hw);
1537
1538 /*
1539 * Setup adapter interrupt vectors
1540 */
1541 ixgbe_setup_adapter_vector(ixgbe);
1542
1543 /*
1544 * Initialize unicast addresses.
1545 */
1546 ixgbe_init_unicst(ixgbe);
1547
1548 /*
1549 * Setup and initialize the mctable structures.
1550 */
1551 ixgbe_setup_multicst(ixgbe);
1552
1553 /*
1554 * Set interrupt throttling rate
1555 */
1556 for (i = 0; i < ixgbe->intr_cnt; i++) {
1557 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1558 }
1559
1560 /*
1561 * Disable Wake-on-LAN
1562 */
1563 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
1564
1565 /*
1566 * Some adapters offer Energy Efficient Ethernet (EEE) support.
1567 * Due to issues with EEE in e1000g/igb, we disable this by default
1568 * as a precautionary measure.
1569 *
1570 * Currently, this is present on a number of the X550 family parts.
1571 */
1572 (void) ixgbe_setup_eee(hw, B_FALSE);
1573
1574 /*
1575 * Turn on any present SFP Tx laser
1576 */
1577 ixgbe_enable_tx_laser(hw);
1578
1579 /*
1580 * Power on the PHY
1581 */
1582 (void) ixgbe_set_phy_power(hw, B_TRUE);
1583
1584 /*
1585 * Save the state of the PHY
1586 */
1587 ixgbe_get_hw_state(ixgbe);
1588
1589 /*
1590 * Make sure driver has control
1591 */
1592 ixgbe_get_driver_control(hw);
1593
1594 return (IXGBE_SUCCESS);
1595 }
1596
1597 /*
1598 * ixgbe_chip_stop - Stop the chipset hardware
1599 */
1600 static void
1601 ixgbe_chip_stop(ixgbe_t *ixgbe)
1602 {
1603 struct ixgbe_hw *hw = &ixgbe->hw;
1604 int rv;
1605
1606 ASSERT(mutex_owned(&ixgbe->gen_lock));
1607
1608 /*
1609 * Stop interupt generation and disable Tx unit
1610 */
1611 hw->adapter_stopped = B_FALSE;
1612 (void) ixgbe_stop_adapter(hw);
1613
1614 /*
1615 * Reset the chipset
1616 */
1617 (void) ixgbe_reset_hw(hw);
1618
1619 /*
1620 * Reset PHY
1621 */
1622 (void) ixgbe_reset_phy(hw);
1623
1624 /*
1625 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting
1626 * the PHY while doing so. Else, just power down the PHY.
1627 */
1628 if (hw->phy.ops.enter_lplu != NULL) {
1629 hw->phy.reset_disable = B_TRUE;
1630 rv = hw->phy.ops.enter_lplu(hw);
1631 if (rv != IXGBE_SUCCESS)
1632 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv);
1633 hw->phy.reset_disable = B_FALSE;
1634 } else {
1635 (void) ixgbe_set_phy_power(hw, B_FALSE);
1636 }
1637
1638 /*
1639 * Turn off any present SFP Tx laser
1640 * Expected for health and safety reasons
1641 */
1642 ixgbe_disable_tx_laser(hw);
1643
1644 /*
1645 * Tell firmware driver is no longer in control
1646 */
1647 ixgbe_release_driver_control(hw);
1648
1649 }
1650
1651 /*
1652 * ixgbe_reset - Reset the chipset and re-start the driver.
1653 *
1654 * It involves stopping and re-starting the chipset,
1655 * and re-configuring the rx/tx rings.
1656 */
1657 static int
1658 ixgbe_reset(ixgbe_t *ixgbe)
1659 {
1660 int i;
1661
1662 /*
1663 * Disable and stop the watchdog timer
1664 */
1665 ixgbe_disable_watchdog_timer(ixgbe);
1666
1667 mutex_enter(&ixgbe->gen_lock);
1668
1669 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1670 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1671
1672 ixgbe_stop(ixgbe, B_FALSE);
1673
1674 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1675 mutex_exit(&ixgbe->gen_lock);
1676 return (IXGBE_FAILURE);
1677 }
1678
1679 /*
1680 * After resetting, need to recheck the link status.
1681 */
1682 ixgbe->link_check_complete = B_FALSE;
1683 ixgbe->link_check_hrtime = gethrtime() +
1684 (IXGBE_LINK_UP_TIME * 100000000ULL);
1685
1686 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1687
1688 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1689 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1690 mac_tx_ring_update(ixgbe->mac_hdl,
1691 ixgbe->tx_rings[i].ring_handle);
1692 }
1693 }
1694
1695 mutex_exit(&ixgbe->gen_lock);
1696
1697 /*
1698 * Enable and start the watchdog timer
1699 */
1700 ixgbe_enable_watchdog_timer(ixgbe);
1701
1702 return (IXGBE_SUCCESS);
1703 }
1704
1705 /*
1706 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1707 */
1708 static void
1709 ixgbe_tx_clean(ixgbe_t *ixgbe)
1710 {
1711 ixgbe_tx_ring_t *tx_ring;
1712 tx_control_block_t *tcb;
1713 link_list_t pending_list;
1714 uint32_t desc_num;
1715 int i, j;
1716
1717 LINK_LIST_INIT(&pending_list);
1718
1719 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1720 tx_ring = &ixgbe->tx_rings[i];
1721
1722 mutex_enter(&tx_ring->recycle_lock);
1723
1724 /*
1725 * Clean the pending tx data - the pending packets in the
1726 * work_list that have no chances to be transmitted again.
1727 *
1728 * We must ensure the chipset is stopped or the link is down
1729 * before cleaning the transmit packets.
1730 */
1731 desc_num = 0;
1732 for (j = 0; j < tx_ring->ring_size; j++) {
1733 tcb = tx_ring->work_list[j];
1734 if (tcb != NULL) {
1735 desc_num += tcb->desc_num;
1736
1737 tx_ring->work_list[j] = NULL;
1738
1739 ixgbe_free_tcb(tcb);
1740
1741 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1742 }
1743 }
1744
1745 if (desc_num > 0) {
1746 atomic_add_32(&tx_ring->tbd_free, desc_num);
1747 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1748
1749 /*
1750 * Reset the head and tail pointers of the tbd ring;
1751 * Reset the writeback head if it's enable.
1752 */
1753 tx_ring->tbd_head = 0;
1754 tx_ring->tbd_tail = 0;
1755 if (ixgbe->tx_head_wb_enable)
1756 *tx_ring->tbd_head_wb = 0;
1757
1758 IXGBE_WRITE_REG(&ixgbe->hw,
1759 IXGBE_TDH(tx_ring->index), 0);
1760 IXGBE_WRITE_REG(&ixgbe->hw,
1761 IXGBE_TDT(tx_ring->index), 0);
1762 }
1763
1764 mutex_exit(&tx_ring->recycle_lock);
1765
1766 /*
1767 * Add the tx control blocks in the pending list to
1768 * the free list.
1769 */
1770 ixgbe_put_free_list(tx_ring, &pending_list);
1771 }
1772 }
1773
1774 /*
1775 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1776 * transmitted.
1777 */
1778 static boolean_t
1779 ixgbe_tx_drain(ixgbe_t *ixgbe)
1780 {
1781 ixgbe_tx_ring_t *tx_ring;
1782 boolean_t done;
1783 int i, j;
1784
1785 /*
1786 * Wait for a specific time to allow pending tx packets
1787 * to be transmitted.
1788 *
1789 * Check the counter tbd_free to see if transmission is done.
1790 * No lock protection is needed here.
1791 *
1792 * Return B_TRUE if all pending packets have been transmitted;
1793 * Otherwise return B_FALSE;
1794 */
1795 for (i = 0; i < TX_DRAIN_TIME; i++) {
1796
1797 done = B_TRUE;
1798 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1799 tx_ring = &ixgbe->tx_rings[j];
1800 done = done &&
1801 (tx_ring->tbd_free == tx_ring->ring_size);
1802 }
1803
1804 if (done)
1805 break;
1806
1807 msec_delay(1);
1808 }
1809
1810 return (done);
1811 }
1812
1813 /*
1814 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1815 */
1816 static boolean_t
1817 ixgbe_rx_drain(ixgbe_t *ixgbe)
1818 {
1819 boolean_t done = B_TRUE;
1820 int i;
1821
1822 /*
1823 * Polling the rx free list to check if those rx buffers held by
1824 * the upper layer are released.
1825 *
1826 * Check the counter rcb_free to see if all pending buffers are
1827 * released. No lock protection is needed here.
1828 *
1829 * Return B_TRUE if all pending buffers have been released;
1830 * Otherwise return B_FALSE;
1831 */
1832 for (i = 0; i < RX_DRAIN_TIME; i++) {
1833 done = (ixgbe->rcb_pending == 0);
1834
1835 if (done)
1836 break;
1837
1838 msec_delay(1);
1839 }
1840
1841 return (done);
1842 }
1843
1844 /*
1845 * ixgbe_start - Start the driver/chipset.
1846 */
1847 int
1848 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1849 {
1850 struct ixgbe_hw *hw = &ixgbe->hw;
1851 int i;
1852
1853 ASSERT(mutex_owned(&ixgbe->gen_lock));
1854
1855 if (alloc_buffer) {
1856 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1857 ixgbe_error(ixgbe,
1858 "Failed to allocate software receive rings");
1859 return (IXGBE_FAILURE);
1860 }
1861
1862 /* Allocate buffers for all the rx/tx rings */
1863 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1864 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1865 return (IXGBE_FAILURE);
1866 }
1867
1868 ixgbe->tx_ring_init = B_TRUE;
1869 } else {
1870 ixgbe->tx_ring_init = B_FALSE;
1871 }
1872
1873 for (i = 0; i < ixgbe->num_rx_rings; i++)
1874 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1875 for (i = 0; i < ixgbe->num_tx_rings; i++)
1876 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1877
1878 /*
1879 * Start the chipset hardware
1880 */
1881 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1882 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1883 goto start_failure;
1884 }
1885
1886 /*
1887 * Configure link now for X550
1888 *
1889 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the
1890 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550,
1891 * the resting state of the link would be the maximum speed that
1892 * autonegotiation will allow (usually 10Gb, infrastructure allowing)
1893 * so we never bothered with explicitly setting the link to 10Gb as it
1894 * would already be at that state on driver attach. With X550, we must
1895 * trigger a re-negotiation of the link in order to switch from a LPLU
1896 * 1Gb link to 10Gb (cable and link partner permitting.)
1897 */
1898 if (hw->mac.type == ixgbe_mac_X550 ||
1899 hw->mac.type == ixgbe_mac_X550EM_a ||
1900 hw->mac.type == ixgbe_mac_X550EM_x) {
1901 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE);
1902 ixgbe_get_hw_state(ixgbe);
1903 }
1904
1905 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1906 goto start_failure;
1907 }
1908
1909 /*
1910 * Setup the rx/tx rings
1911 */
1912 ixgbe_setup_rings(ixgbe);
1913
1914 /*
1915 * ixgbe_start() will be called when resetting, however if reset
1916 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1917 * before enabling the interrupts.
1918 */
1919 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1920 | IXGBE_STALL| IXGBE_OVERTEMP));
1921
1922 /*
1923 * Enable adapter interrupts
1924 * The interrupts must be enabled after the driver state is START
1925 */
1926 ixgbe_enable_adapter_interrupts(ixgbe);
1927
1928 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1929 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1930 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1931 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1932
1933 return (IXGBE_SUCCESS);
1934
1935 start_failure:
1936 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1937 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1938 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1939 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1940
1941 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1942
1943 return (IXGBE_FAILURE);
1944 }
1945
1946 /*
1947 * ixgbe_stop - Stop the driver/chipset.
1948 */
1949 void
1950 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1951 {
1952 int i;
1953
1954 ASSERT(mutex_owned(&ixgbe->gen_lock));
1955
1956 /*
1957 * Disable the adapter interrupts
1958 */
1959 ixgbe_disable_adapter_interrupts(ixgbe);
1960
1961 /*
1962 * Drain the pending tx packets
1963 */
1964 (void) ixgbe_tx_drain(ixgbe);
1965
1966 for (i = 0; i < ixgbe->num_rx_rings; i++)
1967 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1968 for (i = 0; i < ixgbe->num_tx_rings; i++)
1969 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1970
1971 /*
1972 * Stop the chipset hardware
1973 */
1974 ixgbe_chip_stop(ixgbe);
1975
1976 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1977 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1978 }
1979
1980 /*
1981 * Clean the pending tx data/resources
1982 */
1983 ixgbe_tx_clean(ixgbe);
1984
1985 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1986 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1987 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1988 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1989
1990 if (ixgbe->link_state == LINK_STATE_UP) {
1991 ixgbe->link_state = LINK_STATE_UNKNOWN;
1992 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1993 }
1994
1995 if (free_buffer) {
1996 /*
1997 * Release the DMA/memory resources of rx/tx rings
1998 */
1999 ixgbe_free_dma(ixgbe);
2000 ixgbe_free_rx_data(ixgbe);
2001 }
2002 }
2003
2004 /*
2005 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
2006 */
2007 /* ARGSUSED */
2008 static int
2009 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
2010 void *arg1, void *arg2)
2011 {
2012 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
2013
2014 switch (cbaction) {
2015 /* IRM callback */
2016 int count;
2017 case DDI_CB_INTR_ADD:
2018 case DDI_CB_INTR_REMOVE:
2019 count = (int)(uintptr_t)cbarg;
2020 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
2021 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
2022 int, ixgbe->intr_cnt);
2023 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
2024 DDI_SUCCESS) {
2025 ixgbe_error(ixgbe,
2026 "IRM CB: Failed to adjust interrupts");
2027 goto cb_fail;
2028 }
2029 break;
2030 default:
2031 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
2032 cbaction);
2033 return (DDI_ENOTSUP);
2034 }
2035 return (DDI_SUCCESS);
2036 cb_fail:
2037 return (DDI_FAILURE);
2038 }
2039
2040 /*
2041 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
2042 */
2043 static int
2044 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
2045 {
2046 int i, rc, actual;
2047
2048 if (count == 0)
2049 return (DDI_SUCCESS);
2050
2051 if ((cbaction == DDI_CB_INTR_ADD &&
2052 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
2053 (cbaction == DDI_CB_INTR_REMOVE &&
2054 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
2055 return (DDI_FAILURE);
2056
2057 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
2058 return (DDI_FAILURE);
2059 }
2060
2061 for (i = 0; i < ixgbe->num_rx_rings; i++)
2062 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
2063 for (i = 0; i < ixgbe->num_tx_rings; i++)
2064 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
2065
2066 mutex_enter(&ixgbe->gen_lock);
2067 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
2068 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
2069 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
2070 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
2071
2072 ixgbe_stop(ixgbe, B_FALSE);
2073 /*
2074 * Disable interrupts
2075 */
2076 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
2077 rc = ixgbe_disable_intrs(ixgbe);
2078 ASSERT(rc == IXGBE_SUCCESS);
2079 }
2080 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
2081
2082 /*
2083 * Remove interrupt handlers
2084 */
2085 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
2086 ixgbe_rem_intr_handlers(ixgbe);
2087 }
2088 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
2089
2090 /*
2091 * Clear vect_map
2092 */
2093 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
2094 switch (cbaction) {
2095 case DDI_CB_INTR_ADD:
2096 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
2097 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
2098 DDI_INTR_ALLOC_NORMAL);
2099 if (rc != DDI_SUCCESS || actual != count) {
2100 ixgbe_log(ixgbe, "Adjust interrupts failed."
2101 "return: %d, irm cb size: %d, actual: %d",
2102 rc, count, actual);
2103 goto intr_adjust_fail;
2104 }
2105 ixgbe->intr_cnt += count;
2106 break;
2107
2108 case DDI_CB_INTR_REMOVE:
2109 for (i = ixgbe->intr_cnt - count;
2110 i < ixgbe->intr_cnt; i ++) {
2111 rc = ddi_intr_free(ixgbe->htable[i]);
2112 ixgbe->htable[i] = NULL;
2113 if (rc != DDI_SUCCESS) {
2114 ixgbe_log(ixgbe, "Adjust interrupts failed."
2115 "return: %d, irm cb size: %d, actual: %d",
2116 rc, count, actual);
2117 goto intr_adjust_fail;
2118 }
2119 }
2120 ixgbe->intr_cnt -= count;
2121 break;
2122 }
2123
2124 /*
2125 * Get priority for first vector, assume remaining are all the same
2126 */
2127 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
2128 if (rc != DDI_SUCCESS) {
2129 ixgbe_log(ixgbe,
2130 "Get interrupt priority failed: %d", rc);
2131 goto intr_adjust_fail;
2132 }
2133 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
2134 if (rc != DDI_SUCCESS) {
2135 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
2136 goto intr_adjust_fail;
2137 }
2138 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
2139
2140 /*
2141 * Map rings to interrupt vectors
2142 */
2143 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
2144 ixgbe_error(ixgbe,
2145 "IRM CB: Failed to map interrupts to vectors");
2146 goto intr_adjust_fail;
2147 }
2148
2149 /*
2150 * Add interrupt handlers
2151 */
2152 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
2153 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
2154 goto intr_adjust_fail;
2155 }
2156 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
2157
2158 /*
2159 * Now that mutex locks are initialized, and the chip is also
2160 * initialized, enable interrupts.
2161 */
2162 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
2163 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
2164 goto intr_adjust_fail;
2165 }
2166 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
2167 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
2168 ixgbe_error(ixgbe, "IRM CB: Failed to start");
2169 goto intr_adjust_fail;
2170 }
2171 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
2172 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
2173 ixgbe->ixgbe_state |= IXGBE_STARTED;
2174 mutex_exit(&ixgbe->gen_lock);
2175
2176 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2177 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
2178 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
2179 }
2180 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2181 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
2182 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
2183 }
2184
2185 /* Wakeup all Tx rings */
2186 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2187 mac_tx_ring_update(ixgbe->mac_hdl,
2188 ixgbe->tx_rings[i].ring_handle);
2189 }
2190
2191 IXGBE_DEBUGLOG_3(ixgbe,
2192 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
2193 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
2194 return (DDI_SUCCESS);
2195
2196 intr_adjust_fail:
2197 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2198 mutex_exit(&ixgbe->gen_lock);
2199 return (DDI_FAILURE);
2200 }
2201
2202 /*
2203 * ixgbe_intr_cb_register - Register interrupt callback function.
2204 */
2205 static int
2206 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
2207 {
2208 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
2209 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
2210 return (IXGBE_FAILURE);
2211 }
2212 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
2213 return (IXGBE_SUCCESS);
2214 }
2215
2216 /*
2217 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2218 */
2219 static int
2220 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2221 {
2222 /*
2223 * Allocate memory space for rx rings
2224 */
2225 ixgbe->rx_rings = kmem_zalloc(
2226 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2227 KM_NOSLEEP);
2228
2229 if (ixgbe->rx_rings == NULL) {
2230 return (IXGBE_FAILURE);
2231 }
2232
2233 /*
2234 * Allocate memory space for tx rings
2235 */
2236 ixgbe->tx_rings = kmem_zalloc(
2237 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2238 KM_NOSLEEP);
2239
2240 if (ixgbe->tx_rings == NULL) {
2241 kmem_free(ixgbe->rx_rings,
2242 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2243 ixgbe->rx_rings = NULL;
2244 return (IXGBE_FAILURE);
2245 }
2246
2247 /*
2248 * Allocate memory space for rx ring groups
2249 */
2250 ixgbe->rx_groups = kmem_zalloc(
2251 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2252 KM_NOSLEEP);
2253
2254 if (ixgbe->rx_groups == NULL) {
2255 kmem_free(ixgbe->rx_rings,
2256 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2257 kmem_free(ixgbe->tx_rings,
2258 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2259 ixgbe->rx_rings = NULL;
2260 ixgbe->tx_rings = NULL;
2261 return (IXGBE_FAILURE);
2262 }
2263
2264 return (IXGBE_SUCCESS);
2265 }
2266
2267 /*
2268 * ixgbe_free_rings - Free the memory space of rx/tx rings.
2269 */
2270 static void
2271 ixgbe_free_rings(ixgbe_t *ixgbe)
2272 {
2273 if (ixgbe->rx_rings != NULL) {
2274 kmem_free(ixgbe->rx_rings,
2275 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2276 ixgbe->rx_rings = NULL;
2277 }
2278
2279 if (ixgbe->tx_rings != NULL) {
2280 kmem_free(ixgbe->tx_rings,
2281 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2282 ixgbe->tx_rings = NULL;
2283 }
2284
2285 if (ixgbe->rx_groups != NULL) {
2286 kmem_free(ixgbe->rx_groups,
2287 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2288 ixgbe->rx_groups = NULL;
2289 }
2290 }
2291
2292 static int
2293 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2294 {
2295 ixgbe_rx_ring_t *rx_ring;
2296 int i;
2297
2298 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2299 rx_ring = &ixgbe->rx_rings[i];
2300 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2301 goto alloc_rx_rings_failure;
2302 }
2303 return (IXGBE_SUCCESS);
2304
2305 alloc_rx_rings_failure:
2306 ixgbe_free_rx_data(ixgbe);
2307 return (IXGBE_FAILURE);
2308 }
2309
2310 static void
2311 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2312 {
2313 ixgbe_rx_ring_t *rx_ring;
2314 ixgbe_rx_data_t *rx_data;
2315 int i;
2316
2317 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2318 rx_ring = &ixgbe->rx_rings[i];
2319
2320 mutex_enter(&ixgbe->rx_pending_lock);
2321 rx_data = rx_ring->rx_data;
2322
2323 if (rx_data != NULL) {
2324 rx_data->flag |= IXGBE_RX_STOPPED;
2325
2326 if (rx_data->rcb_pending == 0) {
2327 ixgbe_free_rx_ring_data(rx_data);
2328 rx_ring->rx_data = NULL;
2329 }
2330 }
2331
2332 mutex_exit(&ixgbe->rx_pending_lock);
2333 }
2334 }
2335
2336 /*
2337 * ixgbe_setup_rings - Setup rx/tx rings.
2338 */
2339 static void
2340 ixgbe_setup_rings(ixgbe_t *ixgbe)
2341 {
2342 /*
2343 * Setup the rx/tx rings, including the following:
2344 *
2345 * 1. Setup the descriptor ring and the control block buffers;
2346 * 2. Initialize necessary registers for receive/transmit;
2347 * 3. Initialize software pointers/parameters for receive/transmit;
2348 */
2349 ixgbe_setup_rx(ixgbe);
2350
2351 ixgbe_setup_tx(ixgbe);
2352 }
2353
2354 static void
2355 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2356 {
2357 ixgbe_t *ixgbe = rx_ring->ixgbe;
2358 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2359 struct ixgbe_hw *hw = &ixgbe->hw;
2360 rx_control_block_t *rcb;
2361 union ixgbe_adv_rx_desc *rbd;
2362 uint32_t size;
2363 uint32_t buf_low;
2364 uint32_t buf_high;
2365 uint32_t reg_val;
2366 int i;
2367
2368 ASSERT(mutex_owned(&rx_ring->rx_lock));
2369 ASSERT(mutex_owned(&ixgbe->gen_lock));
2370
2371 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2372 rcb = rx_data->work_list[i];
2373 rbd = &rx_data->rbd_ring[i];
2374
2375 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2376 rbd->read.hdr_addr = 0;
2377 }
2378
2379 /*
2380 * Initialize the length register
2381 */
2382 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2383 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2384
2385 /*
2386 * Initialize the base address registers
2387 */
2388 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2389 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2390 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2391 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2392
2393 /*
2394 * Setup head & tail pointers
2395 */
2396 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2397 rx_data->ring_size - 1);
2398 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2399
2400 rx_data->rbd_next = 0;
2401 rx_data->lro_first = 0;
2402
2403 /*
2404 * Setup the Receive Descriptor Control Register (RXDCTL)
2405 * PTHRESH=32 descriptors (half the internal cache)
2406 * HTHRESH=0 descriptors (to minimize latency on fetch)
2407 * WTHRESH defaults to 1 (writeback each descriptor)
2408 */
2409 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2410 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2411
2412 /* Not a valid value for 82599, X540 or X550 */
2413 if (hw->mac.type == ixgbe_mac_82598EB) {
2414 reg_val |= 0x0020; /* pthresh */
2415 }
2416 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2417
2418 if (hw->mac.type == ixgbe_mac_82599EB ||
2419 hw->mac.type == ixgbe_mac_X540 ||
2420 hw->mac.type == ixgbe_mac_X550 ||
2421 hw->mac.type == ixgbe_mac_X550EM_x ||
2422 hw->mac.type == ixgbe_mac_X550EM_a) {
2423 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2424 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2425 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2426 }
2427
2428 /*
2429 * Setup the Split and Replication Receive Control Register.
2430 * Set the rx buffer size and the advanced descriptor type.
2431 */
2432 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2433 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2434 reg_val |= IXGBE_SRRCTL_DROP_EN;
2435 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2436 }
2437
2438 static void
2439 ixgbe_setup_rx(ixgbe_t *ixgbe)
2440 {
2441 ixgbe_rx_ring_t *rx_ring;
2442 struct ixgbe_hw *hw = &ixgbe->hw;
2443 uint32_t reg_val;
2444 uint32_t i;
2445 uint32_t psrtype_rss_bit;
2446
2447 /*
2448 * Ensure that Rx is disabled while setting up
2449 * the Rx unit and Rx descriptor ring(s)
2450 */
2451 ixgbe_disable_rx(hw);
2452
2453 /* PSRTYPE must be configured for 82599 */
2454 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2455 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2456 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2457 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2458 reg_val |= IXGBE_PSRTYPE_L2HDR;
2459 reg_val |= 0x80000000;
2460 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2461 } else {
2462 if (ixgbe->num_rx_groups > 32) {
2463 psrtype_rss_bit = 0x20000000;
2464 } else {
2465 psrtype_rss_bit = 0x40000000;
2466 }
2467 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2468 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2469 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2470 reg_val |= IXGBE_PSRTYPE_L2HDR;
2471 reg_val |= psrtype_rss_bit;
2472 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2473 }
2474 }
2475
2476 /*
2477 * Set filter control in FCTRL to determine types of packets are passed
2478 * up to the driver.
2479 * - Pass broadcast packets.
2480 * - Do not pass flow control pause frames (82598-specific)
2481 */
2482 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2483 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */
2484 if (hw->mac.type == ixgbe_mac_82598EB) {
2485 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */
2486 }
2487 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2488
2489 /*
2490 * Hardware checksum settings
2491 */
2492 if (ixgbe->rx_hcksum_enable) {
2493 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2494 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2495 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2496 }
2497
2498 /*
2499 * Setup VMDq and RSS for multiple receive queues
2500 */
2501 switch (ixgbe->classify_mode) {
2502 case IXGBE_CLASSIFY_RSS:
2503 /*
2504 * One group, only RSS is needed when more than
2505 * one ring enabled.
2506 */
2507 ixgbe_setup_rss(ixgbe);
2508 break;
2509
2510 case IXGBE_CLASSIFY_VMDQ:
2511 /*
2512 * Multiple groups, each group has one ring,
2513 * only VMDq is needed.
2514 */
2515 ixgbe_setup_vmdq(ixgbe);
2516 break;
2517
2518 case IXGBE_CLASSIFY_VMDQ_RSS:
2519 /*
2520 * Multiple groups and multiple rings, both
2521 * VMDq and RSS are needed.
2522 */
2523 ixgbe_setup_vmdq_rss(ixgbe);
2524 break;
2525
2526 default:
2527 break;
2528 }
2529
2530 /*
2531 * Enable the receive unit. This must be done after filter
2532 * control is set in FCTRL. On 82598, we disable the descriptor monitor.
2533 * 82598 is the only adapter which defines this RXCTRL option.
2534 */
2535 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
2536 if (hw->mac.type == ixgbe_mac_82598EB)
2537 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */
2538 reg_val |= IXGBE_RXCTRL_RXEN;
2539 (void) ixgbe_enable_rx_dma(hw, reg_val);
2540
2541 /*
2542 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2543 */
2544 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2545 rx_ring = &ixgbe->rx_rings[i];
2546 ixgbe_setup_rx_ring(rx_ring);
2547 }
2548
2549 /*
2550 * The 82598 controller gives us the RNBC (Receive No Buffer
2551 * Count) register to determine the number of frames dropped
2552 * due to no available descriptors on the destination queue.
2553 * However, this register was removed starting with 82599 and
2554 * it was replaced with the RQSMR/QPRDC registers. The nice
2555 * thing about the new registers is that they allow you to map
2556 * groups of queues to specific stat registers. The bad thing
2557 * is there are only 16 slots in the stat registers, so this
2558 * won't work when we have 32 Rx groups. Instead, we map all
2559 * queues to the zero slot of the stat registers, giving us a
2560 * global counter at QPRDC[0] (with the equivalent semantics
2561 * of RNBC). Perhaps future controllers will have more slots
2562 * and we can implement per-group counters.
2563 */
2564 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2565 uint32_t index = ixgbe->rx_rings[i].hw_index;
2566 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), 0);
2567 }
2568
2569 /*
2570 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2571 * by four bytes if the packet has a VLAN field, so includes MTU,
2572 * ethernet header and frame check sequence.
2573 * Register is MAXFRS in 82599.
2574 */
2575 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD);
2576 reg_val &= ~IXGBE_MHADD_MFS_MASK;
2577 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header)
2578 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2579 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2580
2581 /*
2582 * Setup Jumbo Frame enable bit
2583 */
2584 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2585 if (ixgbe->default_mtu > ETHERMTU)
2586 reg_val |= IXGBE_HLREG0_JUMBOEN;
2587 else
2588 reg_val &= ~IXGBE_HLREG0_JUMBOEN;
2589 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2590
2591 /*
2592 * Setup RSC for multiple receive queues.
2593 */
2594 if (ixgbe->lro_enable) {
2595 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2596 /*
2597 * Make sure rx_buf_size * MAXDESC not greater
2598 * than 65535.
2599 * Intel recommends 4 for MAXDESC field value.
2600 */
2601 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2602 reg_val |= IXGBE_RSCCTL_RSCEN;
2603 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2604 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2605 else
2606 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2607 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2608 }
2609
2610 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2611 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2612 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2613
2614 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2615 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2616 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2617 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2618
2619 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2620 }
2621 }
2622
2623 static void
2624 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2625 {
2626 ixgbe_t *ixgbe = tx_ring->ixgbe;
2627 struct ixgbe_hw *hw = &ixgbe->hw;
2628 uint32_t size;
2629 uint32_t buf_low;
2630 uint32_t buf_high;
2631 uint32_t reg_val;
2632
2633 ASSERT(mutex_owned(&tx_ring->tx_lock));
2634 ASSERT(mutex_owned(&ixgbe->gen_lock));
2635
2636 /*
2637 * Initialize the length register
2638 */
2639 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2640 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2641
2642 /*
2643 * Initialize the base address registers
2644 */
2645 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2646 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2647 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2648 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2649
2650 /*
2651 * Setup head & tail pointers
2652 */
2653 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2654 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2655
2656 /*
2657 * Setup head write-back
2658 */
2659 if (ixgbe->tx_head_wb_enable) {
2660 /*
2661 * The memory of the head write-back is allocated using
2662 * the extra tbd beyond the tail of the tbd ring.
2663 */
2664 tx_ring->tbd_head_wb = (uint32_t *)
2665 ((uintptr_t)tx_ring->tbd_area.address + size);
2666 *tx_ring->tbd_head_wb = 0;
2667
2668 buf_low = (uint32_t)
2669 (tx_ring->tbd_area.dma_address + size);
2670 buf_high = (uint32_t)
2671 ((tx_ring->tbd_area.dma_address + size) >> 32);
2672
2673 /* Set the head write-back enable bit */
2674 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2675
2676 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2677 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2678
2679 /*
2680 * Turn off relaxed ordering for head write back or it will
2681 * cause problems with the tx recycling
2682 */
2683
2684 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2685 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2686 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2687 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2688 if (hw->mac.type == ixgbe_mac_82598EB) {
2689 IXGBE_WRITE_REG(hw,
2690 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2691 } else {
2692 IXGBE_WRITE_REG(hw,
2693 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2694 }
2695 } else {
2696 tx_ring->tbd_head_wb = NULL;
2697 }
2698
2699 tx_ring->tbd_head = 0;
2700 tx_ring->tbd_tail = 0;
2701 tx_ring->tbd_free = tx_ring->ring_size;
2702
2703 if (ixgbe->tx_ring_init == B_TRUE) {
2704 tx_ring->tcb_head = 0;
2705 tx_ring->tcb_tail = 0;
2706 tx_ring->tcb_free = tx_ring->free_list_size;
2707 }
2708
2709 /*
2710 * Initialize the s/w context structure
2711 */
2712 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2713 }
2714
2715 static void
2716 ixgbe_setup_tx(ixgbe_t *ixgbe)
2717 {
2718 struct ixgbe_hw *hw = &ixgbe->hw;
2719 ixgbe_tx_ring_t *tx_ring;
2720 uint32_t reg_val;
2721 int i;
2722
2723 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2724 tx_ring = &ixgbe->tx_rings[i];
2725 ixgbe_setup_tx_ring(tx_ring);
2726 }
2727
2728 /*
2729 * Setup the per-ring statistics mapping. We map all Tx queues
2730 * to slot 0 to stay consistent with Rx.
2731 */
2732 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2733 switch (hw->mac.type) {
2734 case ixgbe_mac_82598EB:
2735 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 0);
2736 break;
2737
2738 default:
2739 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 0);
2740 break;
2741 }
2742 }
2743
2744 /*
2745 * Enable CRC appending and TX padding (for short tx frames)
2746 */
2747 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2748 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2749 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2750
2751 /*
2752 * enable DMA for 82599, X540 and X550 parts
2753 */
2754 if (hw->mac.type == ixgbe_mac_82599EB ||
2755 hw->mac.type == ixgbe_mac_X540 ||
2756 hw->mac.type == ixgbe_mac_X550 ||
2757 hw->mac.type == ixgbe_mac_X550EM_x ||
2758 hw->mac.type == ixgbe_mac_X550EM_a) {
2759 /* DMATXCTL.TE must be set after all Tx config is complete */
2760 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2761 reg_val |= IXGBE_DMATXCTL_TE;
2762 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2763
2764 /* Disable arbiter to set MTQC */
2765 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2766 reg_val |= IXGBE_RTTDCS_ARBDIS;
2767 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2768 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2769 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2770 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2771 }
2772
2773 /*
2774 * Enabling tx queues ..
2775 * For 82599 must be done after DMATXCTL.TE is set
2776 */
2777 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2778 tx_ring = &ixgbe->tx_rings[i];
2779 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2780 reg_val |= IXGBE_TXDCTL_ENABLE;
2781 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2782 }
2783 }
2784
2785 /*
2786 * ixgbe_setup_rss - Setup receive-side scaling feature.
2787 */
2788 static void
2789 ixgbe_setup_rss(ixgbe_t *ixgbe)
2790 {
2791 struct ixgbe_hw *hw = &ixgbe->hw;
2792 uint32_t mrqc;
2793
2794 /*
2795 * Initialize RETA/ERETA table
2796 */
2797 ixgbe_setup_rss_table(ixgbe);
2798
2799 /*
2800 * Enable RSS & perform hash on these packet types
2801 */
2802 mrqc = IXGBE_MRQC_RSSEN |
2803 IXGBE_MRQC_RSS_FIELD_IPV4 |
2804 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2805 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2806 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2807 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2808 IXGBE_MRQC_RSS_FIELD_IPV6 |
2809 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2810 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2811 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2812 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2813 }
2814
2815 /*
2816 * ixgbe_setup_vmdq - Setup MAC classification feature
2817 */
2818 static void
2819 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2820 {
2821 struct ixgbe_hw *hw = &ixgbe->hw;
2822 uint32_t vmdctl, i, vtctl;
2823
2824 /*
2825 * Setup the VMDq Control register, enable VMDq based on
2826 * packet destination MAC address:
2827 */
2828 switch (hw->mac.type) {
2829 case ixgbe_mac_82598EB:
2830 /*
2831 * VMDq Enable = 1;
2832 * VMDq Filter = 0; MAC filtering
2833 * Default VMDq output index = 0;
2834 */
2835 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2836 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2837 break;
2838
2839 case ixgbe_mac_82599EB:
2840 case ixgbe_mac_X540:
2841 case ixgbe_mac_X550:
2842 case ixgbe_mac_X550EM_x:
2843 case ixgbe_mac_X550EM_a:
2844 /*
2845 * Enable VMDq-only.
2846 */
2847 vmdctl = IXGBE_MRQC_VMDQEN;
2848 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2849
2850 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2851 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2852 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2853 }
2854
2855 /*
2856 * Enable Virtualization and Replication.
2857 */
2858 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2859 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2860
2861 /*
2862 * Enable receiving packets to all VFs
2863 */
2864 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2865 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2866 break;
2867
2868 default:
2869 break;
2870 }
2871 }
2872
2873 /*
2874 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2875 */
2876 static void
2877 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2878 {
2879 struct ixgbe_hw *hw = &ixgbe->hw;
2880 uint32_t i, mrqc;
2881 uint32_t vtctl, vmdctl;
2882
2883 /*
2884 * Initialize RETA/ERETA table
2885 */
2886 ixgbe_setup_rss_table(ixgbe);
2887
2888 /*
2889 * Enable and setup RSS and VMDq
2890 */
2891 switch (hw->mac.type) {
2892 case ixgbe_mac_82598EB:
2893 /*
2894 * Enable RSS & Setup RSS Hash functions
2895 */
2896 mrqc = IXGBE_MRQC_RSSEN |
2897 IXGBE_MRQC_RSS_FIELD_IPV4 |
2898 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2899 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2900 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2901 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2902 IXGBE_MRQC_RSS_FIELD_IPV6 |
2903 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2904 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2905 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2906 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2907
2908 /*
2909 * Enable and Setup VMDq
2910 * VMDq Filter = 0; MAC filtering
2911 * Default VMDq output index = 0;
2912 */
2913 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2914 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2915 break;
2916
2917 case ixgbe_mac_82599EB:
2918 case ixgbe_mac_X540:
2919 case ixgbe_mac_X550:
2920 case ixgbe_mac_X550EM_x:
2921 case ixgbe_mac_X550EM_a:
2922 /*
2923 * Enable RSS & Setup RSS Hash functions
2924 */
2925 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2926 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2927 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2928 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2929 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2930 IXGBE_MRQC_RSS_FIELD_IPV6 |
2931 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2932 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2933 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2934
2935 /*
2936 * Enable VMDq+RSS.
2937 */
2938 if (ixgbe->num_rx_groups > 32) {
2939 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2940 } else {
2941 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2942 }
2943
2944 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2945
2946 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2947 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2948 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2949 }
2950 break;
2951
2952 default:
2953 break;
2954
2955 }
2956
2957 if (hw->mac.type == ixgbe_mac_82599EB ||
2958 hw->mac.type == ixgbe_mac_X540 ||
2959 hw->mac.type == ixgbe_mac_X550 ||
2960 hw->mac.type == ixgbe_mac_X550EM_x ||
2961 hw->mac.type == ixgbe_mac_X550EM_a) {
2962 /*
2963 * Enable Virtualization and Replication.
2964 */
2965 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2966 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2967
2968 /*
2969 * Enable receiving packets to all VFs
2970 */
2971 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2972 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2973 }
2974 }
2975
2976 /*
2977 * ixgbe_setup_rss_table - Setup RSS table
2978 */
2979 static void
2980 ixgbe_setup_rss_table(ixgbe_t *ixgbe)
2981 {
2982 struct ixgbe_hw *hw = &ixgbe->hw;
2983 uint32_t i, j;
2984 uint32_t random;
2985 uint32_t reta;
2986 uint32_t ring_per_group;
2987 uint32_t ring;
2988 uint32_t table_size;
2989 uint32_t index_mult;
2990 uint32_t rxcsum;
2991
2992 /*
2993 * Set multiplier for RETA setup and table size based on MAC type.
2994 * RETA table sizes vary by model:
2995 *
2996 * 82598, 82599, X540: 128 table entries.
2997 * X550: 512 table entries.
2998 */
2999 index_mult = 0x1;
3000 table_size = 128;
3001 switch (ixgbe->hw.mac.type) {
3002 case ixgbe_mac_82598EB:
3003 index_mult = 0x11;
3004 break;
3005 case ixgbe_mac_X550:
3006 case ixgbe_mac_X550EM_x:
3007 case ixgbe_mac_X550EM_a:
3008 table_size = 512;
3009 break;
3010 default:
3011 break;
3012 }
3013
3014 /*
3015 * Fill out RSS redirection table. The configuation of the indices is
3016 * hardware-dependent.
3017 *
3018 * 82598: 8 bits wide containing two 4 bit RSS indices
3019 * 82599, X540: 8 bits wide containing one 4 bit RSS index
3020 * X550: 8 bits wide containing one 6 bit RSS index
3021 */
3022 reta = 0;
3023 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3024
3025 for (i = 0, j = 0; i < table_size; i++, j++) {
3026 if (j == ring_per_group) j = 0;
3027
3028 /*
3029 * The low 8 bits are for hash value (n+0);
3030 * The next 8 bits are for hash value (n+1), etc.
3031 */
3032 ring = (j * index_mult);
3033 reta = reta >> 8;
3034 reta = reta | (((uint32_t)ring) << 24);
3035
3036 if ((i & 3) == 3) {
3037 /*
3038 * The first 128 table entries are programmed into the
3039 * RETA register, with any beyond that (eg; on X550)
3040 * into ERETA.
3041 */
3042 if (i < 128)
3043 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3044 else
3045 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3046 reta);
3047 reta = 0;
3048 }
3049 }
3050
3051 /*
3052 * Fill out hash function seeds with a random constant
3053 */
3054 for (i = 0; i < 10; i++) {
3055 (void) random_get_pseudo_bytes((uint8_t *)&random,
3056 sizeof (uint32_t));
3057 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
3058 }
3059
3060 /*
3061 * Disable Packet Checksum to enable RSS for multiple receive queues.
3062 * It is an adapter hardware limitation that Packet Checksum is
3063 * mutually exclusive with RSS.
3064 */
3065 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3066 rxcsum |= IXGBE_RXCSUM_PCSD;
3067 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3068 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3069 }
3070
3071 /*
3072 * ixgbe_init_unicst - Initialize the unicast addresses.
3073 */
3074 static void
3075 ixgbe_init_unicst(ixgbe_t *ixgbe)
3076 {
3077 struct ixgbe_hw *hw = &ixgbe->hw;
3078 uint8_t *mac_addr;
3079 int slot;
3080 /*
3081 * Here we should consider two situations:
3082 *
3083 * 1. Chipset is initialized at the first time,
3084 * Clear all the multiple unicast addresses.
3085 *
3086 * 2. Chipset is reset
3087 * Recover the multiple unicast addresses from the
3088 * software data structure to the RAR registers.
3089 */
3090 if (!ixgbe->unicst_init) {
3091 /*
3092 * Initialize the multiple unicast addresses
3093 */
3094 ixgbe->unicst_total = hw->mac.num_rar_entries;
3095 ixgbe->unicst_avail = ixgbe->unicst_total;
3096 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3097 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3098 bzero(mac_addr, ETHERADDRL);
3099 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3100 ixgbe->unicst_addr[slot].mac.set = 0;
3101 }
3102 ixgbe->unicst_init = B_TRUE;
3103 } else {
3104 /* Re-configure the RAR registers */
3105 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3106 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3107 if (ixgbe->unicst_addr[slot].mac.set == 1) {
3108 (void) ixgbe_set_rar(hw, slot, mac_addr,
3109 ixgbe->unicst_addr[slot].mac.group_index,
3110 IXGBE_RAH_AV);
3111 } else {
3112 bzero(mac_addr, ETHERADDRL);
3113 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3114 }
3115 }
3116 }
3117 }
3118
3119 /*
3120 * ixgbe_unicst_find - Find the slot for the specified unicast address
3121 */
3122 int
3123 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
3124 {
3125 int slot;
3126
3127 ASSERT(mutex_owned(&ixgbe->gen_lock));
3128
3129 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3130 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
3131 mac_addr, ETHERADDRL) == 0)
3132 return (slot);
3133 }
3134
3135 return (-1);
3136 }
3137
3138 /*
3139 * ixgbe_multicst_add - Add a multicst address.
3140 */
3141 int
3142 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3143 {
3144 ASSERT(mutex_owned(&ixgbe->gen_lock));
3145
3146 if ((multiaddr[0] & 01) == 0) {
3147 return (EINVAL);
3148 }
3149
3150 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
3151 return (ENOENT);
3152 }
3153
3154 bcopy(multiaddr,
3155 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
3156 ixgbe->mcast_count++;
3157
3158 /*
3159 * Update the multicast table in the hardware
3160 */
3161 ixgbe_setup_multicst(ixgbe);
3162
3163 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3164 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3165 return (EIO);
3166 }
3167
3168 return (0);
3169 }
3170
3171 /*
3172 * ixgbe_multicst_remove - Remove a multicst address.
3173 */
3174 int
3175 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3176 {
3177 int i;
3178
3179 ASSERT(mutex_owned(&ixgbe->gen_lock));
3180
3181 for (i = 0; i < ixgbe->mcast_count; i++) {
3182 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
3183 ETHERADDRL) == 0) {
3184 for (i++; i < ixgbe->mcast_count; i++) {
3185 ixgbe->mcast_table[i - 1] =
3186 ixgbe->mcast_table[i];
3187 }
3188 ixgbe->mcast_count--;
3189 break;
3190 }
3191 }
3192
3193 /*
3194 * Update the multicast table in the hardware
3195 */
3196 ixgbe_setup_multicst(ixgbe);
3197
3198 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3199 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3200 return (EIO);
3201 }
3202
3203 return (0);
3204 }
3205
3206 /*
3207 * ixgbe_setup_multicast - Setup multicast data structures.
3208 *
3209 * This routine initializes all of the multicast related structures
3210 * and save them in the hardware registers.
3211 */
3212 static void
3213 ixgbe_setup_multicst(ixgbe_t *ixgbe)
3214 {
3215 uint8_t *mc_addr_list;
3216 uint32_t mc_addr_count;
3217 struct ixgbe_hw *hw = &ixgbe->hw;
3218
3219 ASSERT(mutex_owned(&ixgbe->gen_lock));
3220
3221 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
3222
3223 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
3224 mc_addr_count = ixgbe->mcast_count;
3225
3226 /*
3227 * Update the multicast addresses to the MTA registers
3228 */
3229 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
3230 ixgbe_mc_table_itr, TRUE);
3231 }
3232
3233 /*
3234 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
3235 *
3236 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
3237 * Different chipsets may have different allowed configuration of vmdq and rss.
3238 */
3239 static void
3240 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
3241 {
3242 struct ixgbe_hw *hw = &ixgbe->hw;
3243 uint32_t ring_per_group;
3244
3245 switch (hw->mac.type) {
3246 case ixgbe_mac_82598EB:
3247 /*
3248 * 82598 supports the following combination:
3249 * vmdq no. x rss no.
3250 * [5..16] x 1
3251 * [1..4] x [1..16]
3252 * However 8 rss queue per pool (vmdq) is sufficient for
3253 * most cases.
3254 */
3255 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3256 if (ixgbe->num_rx_groups > 4) {
3257 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3258 } else {
3259 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3260 min(8, ring_per_group);
3261 }
3262
3263 break;
3264
3265 case ixgbe_mac_82599EB:
3266 case ixgbe_mac_X540:
3267 case ixgbe_mac_X550:
3268 case ixgbe_mac_X550EM_x:
3269 case ixgbe_mac_X550EM_a:
3270 /*
3271 * 82599 supports the following combination:
3272 * vmdq no. x rss no.
3273 * [33..64] x [1..2]
3274 * [2..32] x [1..4]
3275 * 1 x [1..16]
3276 * However 8 rss queue per pool (vmdq) is sufficient for
3277 * most cases.
3278 *
3279 * For now, treat X540 and X550 like the 82599.
3280 */
3281 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3282 if (ixgbe->num_rx_groups == 1) {
3283 ixgbe->num_rx_rings = min(8, ring_per_group);
3284 } else if (ixgbe->num_rx_groups <= 32) {
3285 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3286 min(4, ring_per_group);
3287 } else if (ixgbe->num_rx_groups <= 64) {
3288 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3289 min(2, ring_per_group);
3290 }
3291 break;
3292
3293 default:
3294 break;
3295 }
3296
3297 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3298
3299 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3300 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3301 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3302 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3303 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3304 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3305 } else {
3306 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3307 }
3308
3309 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3310 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3311 }
3312
3313 /*
3314 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3315 *
3316 * This routine gets user-configured values out of the configuration
3317 * file ixgbe.conf.
3318 *
3319 * For each configurable value, there is a minimum, a maximum, and a
3320 * default.
3321 * If user does not configure a value, use the default.
3322 * If user configures below the minimum, use the minumum.
3323 * If user configures above the maximum, use the maxumum.
3324 */
3325 static void
3326 ixgbe_get_conf(ixgbe_t *ixgbe)
3327 {
3328 struct ixgbe_hw *hw = &ixgbe->hw;
3329 uint32_t flow_control;
3330
3331 /*
3332 * ixgbe driver supports the following user configurations:
3333 *
3334 * Jumbo frame configuration:
3335 * default_mtu
3336 *
3337 * Ethernet flow control configuration:
3338 * flow_control
3339 *
3340 * Multiple rings configurations:
3341 * tx_queue_number
3342 * tx_ring_size
3343 * rx_queue_number
3344 * rx_ring_size
3345 *
3346 * Call ixgbe_get_prop() to get the value for a specific
3347 * configuration parameter.
3348 */
3349
3350 /*
3351 * Jumbo frame configuration - max_frame_size controls host buffer
3352 * allocation, so includes MTU, ethernet header, vlan tag and
3353 * frame check sequence.
3354 */
3355 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3356 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3357
3358 ixgbe->max_frame_size = ixgbe->default_mtu +
3359 sizeof (struct ether_vlan_header) + ETHERFCSL;
3360
3361 /*
3362 * Ethernet flow control configuration
3363 */
3364 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3365 ixgbe_fc_none, 3, ixgbe_fc_none);
3366 if (flow_control == 3)
3367 flow_control = ixgbe_fc_default;
3368
3369 /*
3370 * fc.requested mode is what the user requests. After autoneg,
3371 * fc.current_mode will be the flow_control mode that was negotiated.
3372 */
3373 hw->fc.requested_mode = flow_control;
3374
3375 /*
3376 * Multiple rings configurations
3377 */
3378 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3379 ixgbe->capab->min_tx_que_num,
3380 ixgbe->capab->max_tx_que_num,
3381 ixgbe->capab->def_tx_que_num);
3382 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3383 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3384
3385 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3386 ixgbe->capab->min_rx_que_num,
3387 ixgbe->capab->max_rx_que_num,
3388 ixgbe->capab->def_rx_que_num);
3389 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3390 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3391
3392 /*
3393 * Multiple groups configuration
3394 */
3395 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3396 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3397 ixgbe->capab->def_rx_grp_num);
3398
3399 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3400 0, 1, DEFAULT_MR_ENABLE);
3401
3402 if (ixgbe->mr_enable == B_FALSE) {
3403 ixgbe->num_tx_rings = 1;
3404 ixgbe->num_rx_rings = 1;
3405 ixgbe->num_rx_groups = 1;
3406 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3407 } else {
3408 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3409 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3410 /*
3411 * The combination of num_rx_rings and num_rx_groups
3412 * may be not supported by h/w. We need to adjust
3413 * them to appropriate values.
3414 */
3415 ixgbe_setup_vmdq_rss_conf(ixgbe);
3416 }
3417
3418 /*
3419 * Tunable used to force an interrupt type. The only use is
3420 * for testing of the lesser interrupt types.
3421 * 0 = don't force interrupt type
3422 * 1 = force interrupt type MSI-X
3423 * 2 = force interrupt type MSI
3424 * 3 = force interrupt type Legacy
3425 */
3426 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3427 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3428
3429 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3430 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3431 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3432 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3433 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3434 0, 1, DEFAULT_LSO_ENABLE);
3435 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3436 0, 1, DEFAULT_LRO_ENABLE);
3437 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3438 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3439 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3440 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3441
3442 /* Head Write Back not recommended for 82599, X540 and X550 */
3443 if (hw->mac.type == ixgbe_mac_82599EB ||
3444 hw->mac.type == ixgbe_mac_X540 ||
3445 hw->mac.type == ixgbe_mac_X550 ||
3446 hw->mac.type == ixgbe_mac_X550EM_x ||
3447 hw->mac.type == ixgbe_mac_X550EM_a) {
3448 ixgbe->tx_head_wb_enable = B_FALSE;
3449 }
3450
3451 /*
3452 * ixgbe LSO needs the tx h/w checksum support.
3453 * LSO will be disabled if tx h/w checksum is not
3454 * enabled.
3455 */
3456 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3457 ixgbe->lso_enable = B_FALSE;
3458 }
3459
3460 /*
3461 * ixgbe LRO needs the rx h/w checksum support.
3462 * LRO will be disabled if rx h/w checksum is not
3463 * enabled.
3464 */
3465 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3466 ixgbe->lro_enable = B_FALSE;
3467 }
3468
3469 /*
3470 * ixgbe LRO only supported by 82599, X540 and X550
3471 */
3472 if (hw->mac.type == ixgbe_mac_82598EB) {
3473 ixgbe->lro_enable = B_FALSE;
3474 }
3475 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3476 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3477 DEFAULT_TX_COPY_THRESHOLD);
3478 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3479 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3480 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3481 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3482 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3483 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3484 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3485 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3486 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3487
3488 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3489 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3490 DEFAULT_RX_COPY_THRESHOLD);
3491 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3492 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3493 DEFAULT_RX_LIMIT_PER_INTR);
3494
3495 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3496 ixgbe->capab->min_intr_throttle,
3497 ixgbe->capab->max_intr_throttle,
3498 ixgbe->capab->def_intr_throttle);
3499 /*
3500 * 82599, X540 and X550 require the interrupt throttling rate is
3501 * a multiple of 8. This is enforced by the register definiton.
3502 */
3503 if (hw->mac.type == ixgbe_mac_82599EB ||
3504 hw->mac.type == ixgbe_mac_X540 ||
3505 hw->mac.type == ixgbe_mac_X550 ||
3506 hw->mac.type == ixgbe_mac_X550EM_x ||
3507 hw->mac.type == ixgbe_mac_X550EM_a)
3508 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3509
3510 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3511 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3512 }
3513
3514 static void
3515 ixgbe_init_params(ixgbe_t *ixgbe)
3516 {
3517 struct ixgbe_hw *hw = &ixgbe->hw;
3518 ixgbe_link_speed speeds_supported = 0;
3519 boolean_t negotiate;
3520
3521 /*
3522 * Get a list of speeds the adapter supports. If the hw struct hasn't
3523 * been populated with this information yet, retrieve it from the
3524 * adapter and save it to our own variable.
3525 *
3526 * On certain adapters, such as ones which use SFPs, the contents of
3527 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not
3528 * updated, so we must rely on calling ixgbe_get_link_capabilities()
3529 * in order to ascertain the speeds which we are capable of supporting,
3530 * and in the case of SFP-equipped adapters, which speed we are
3531 * advertising. If ixgbe_get_link_capabilities() fails for some reason,
3532 * we'll go with a default list of speeds as a last resort.
3533 */
3534 speeds_supported = hw->phy.speeds_supported;
3535
3536 if (speeds_supported == 0) {
3537 if (ixgbe_get_link_capabilities(hw, &speeds_supported,
3538 &negotiate) != IXGBE_SUCCESS) {
3539 if (hw->mac.type == ixgbe_mac_82598EB) {
3540 speeds_supported =
3541 IXGBE_LINK_SPEED_82598_AUTONEG;
3542 } else {
3543 speeds_supported =
3544 IXGBE_LINK_SPEED_82599_AUTONEG;
3545 }
3546 }
3547 }
3548 ixgbe->speeds_supported = speeds_supported;
3549
3550 /*
3551 * By default, all supported speeds are enabled and advertised.
3552 */
3553 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) {
3554 ixgbe->param_en_10000fdx_cap = 1;
3555 ixgbe->param_adv_10000fdx_cap = 1;
3556 } else {
3557 ixgbe->param_en_10000fdx_cap = 0;
3558 ixgbe->param_adv_10000fdx_cap = 0;
3559 }
3560
3561 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) {
3562 ixgbe->param_en_5000fdx_cap = 1;
3563 ixgbe->param_adv_5000fdx_cap = 1;
3564 } else {
3565 ixgbe->param_en_5000fdx_cap = 0;
3566 ixgbe->param_adv_5000fdx_cap = 0;
3567 }
3568
3569 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) {
3570 ixgbe->param_en_2500fdx_cap = 1;
3571 ixgbe->param_adv_2500fdx_cap = 1;
3572 } else {
3573 ixgbe->param_en_2500fdx_cap = 0;
3574 ixgbe->param_adv_2500fdx_cap = 0;
3575 }
3576
3577 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) {
3578 ixgbe->param_en_1000fdx_cap = 1;
3579 ixgbe->param_adv_1000fdx_cap = 1;
3580 } else {
3581 ixgbe->param_en_1000fdx_cap = 0;
3582 ixgbe->param_adv_1000fdx_cap = 0;
3583 }
3584
3585 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) {
3586 ixgbe->param_en_100fdx_cap = 1;
3587 ixgbe->param_adv_100fdx_cap = 1;
3588 } else {
3589 ixgbe->param_en_100fdx_cap = 0;
3590 ixgbe->param_adv_100fdx_cap = 0;
3591 }
3592
3593 ixgbe->param_pause_cap = 1;
3594 ixgbe->param_asym_pause_cap = 1;
3595 ixgbe->param_rem_fault = 0;
3596
3597 ixgbe->param_adv_autoneg_cap = 1;
3598 ixgbe->param_adv_pause_cap = 1;
3599 ixgbe->param_adv_asym_pause_cap = 1;
3600 ixgbe->param_adv_rem_fault = 0;
3601
3602 ixgbe->param_lp_10000fdx_cap = 0;
3603 ixgbe->param_lp_5000fdx_cap = 0;
3604 ixgbe->param_lp_2500fdx_cap = 0;
3605 ixgbe->param_lp_1000fdx_cap = 0;
3606 ixgbe->param_lp_100fdx_cap = 0;
3607 ixgbe->param_lp_autoneg_cap = 0;
3608 ixgbe->param_lp_pause_cap = 0;
3609 ixgbe->param_lp_asym_pause_cap = 0;
3610 ixgbe->param_lp_rem_fault = 0;
3611 }
3612
3613 /*
3614 * ixgbe_get_prop - Get a property value out of the configuration file
3615 * ixgbe.conf.
3616 *
3617 * Caller provides the name of the property, a default value, a minimum
3618 * value, and a maximum value.
3619 *
3620 * Return configured value of the property, with default, minimum and
3621 * maximum properly applied.
3622 */
3623 static int
3624 ixgbe_get_prop(ixgbe_t *ixgbe,
3625 char *propname, /* name of the property */
3626 int minval, /* minimum acceptable value */
3627 int maxval, /* maximim acceptable value */
3628 int defval) /* default value */
3629 {
3630 int value;
3631
3632 /*
3633 * Call ddi_prop_get_int() to read the conf settings
3634 */
3635 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3636 DDI_PROP_DONTPASS, propname, defval);
3637 if (value > maxval)
3638 value = maxval;
3639
3640 if (value < minval)
3641 value = minval;
3642
3643 return (value);
3644 }
3645
3646 /*
3647 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3648 */
3649 int
3650 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3651 {
3652 struct ixgbe_hw *hw = &ixgbe->hw;
3653 ixgbe_link_speed advertised = 0;
3654
3655 /*
3656 * Assemble a list of enabled speeds to auto-negotiate with.
3657 */
3658 if (ixgbe->param_en_10000fdx_cap == 1)
3659 advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3660
3661 if (ixgbe->param_en_5000fdx_cap == 1)
3662 advertised |= IXGBE_LINK_SPEED_5GB_FULL;
3663
3664 if (ixgbe->param_en_2500fdx_cap == 1)
3665 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL;
3666
3667 if (ixgbe->param_en_1000fdx_cap == 1)
3668 advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3669
3670 if (ixgbe->param_en_100fdx_cap == 1)
3671 advertised |= IXGBE_LINK_SPEED_100_FULL;
3672
3673 /*
3674 * As a last resort, autoneg with a default list of speeds.
3675 */
3676 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) {
3677 ixgbe_notice(ixgbe, "Invalid link settings. Setting link "
3678 "to autonegotiate with full capabilities.");
3679
3680 if (hw->mac.type == ixgbe_mac_82598EB)
3681 advertised = IXGBE_LINK_SPEED_82598_AUTONEG;
3682 else
3683 advertised = IXGBE_LINK_SPEED_82599_AUTONEG;
3684 }
3685
3686 if (setup_hw) {
3687 if (ixgbe_setup_link(&ixgbe->hw, advertised,
3688 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3689 ixgbe_notice(ixgbe, "Setup link failed on this "
3690 "device.");
3691 return (IXGBE_FAILURE);
3692 }
3693 }
3694
3695 return (IXGBE_SUCCESS);
3696 }
3697
3698 /*
3699 * ixgbe_driver_link_check - Link status processing.
3700 *
3701 * This function can be called in both kernel context and interrupt context
3702 */
3703 static void
3704 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3705 {
3706 struct ixgbe_hw *hw = &ixgbe->hw;
3707 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3708 boolean_t link_up = B_FALSE;
3709 boolean_t link_changed = B_FALSE;
3710
3711 ASSERT(mutex_owned(&ixgbe->gen_lock));
3712
3713 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3714 if (link_up) {
3715 ixgbe->link_check_complete = B_TRUE;
3716
3717 /* Link is up, enable flow control settings */
3718 (void) ixgbe_fc_enable(hw);
3719
3720 /*
3721 * The Link is up, check whether it was marked as down earlier
3722 */
3723 if (ixgbe->link_state != LINK_STATE_UP) {
3724 switch (speed) {
3725 case IXGBE_LINK_SPEED_10GB_FULL:
3726 ixgbe->link_speed = SPEED_10GB;
3727 break;
3728 case IXGBE_LINK_SPEED_5GB_FULL:
3729 ixgbe->link_speed = SPEED_5GB;
3730 break;
3731 case IXGBE_LINK_SPEED_2_5GB_FULL:
3732 ixgbe->link_speed = SPEED_2_5GB;
3733 break;
3734 case IXGBE_LINK_SPEED_1GB_FULL:
3735 ixgbe->link_speed = SPEED_1GB;
3736 break;
3737 case IXGBE_LINK_SPEED_100_FULL:
3738 ixgbe->link_speed = SPEED_100;
3739 }
3740 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3741 ixgbe->link_state = LINK_STATE_UP;
3742 link_changed = B_TRUE;
3743 }
3744 } else {
3745 if (ixgbe->link_check_complete == B_TRUE ||
3746 (ixgbe->link_check_complete == B_FALSE &&
3747 gethrtime() >= ixgbe->link_check_hrtime)) {
3748 /*
3749 * The link is really down
3750 */
3751 ixgbe->link_check_complete = B_TRUE;
3752
3753 if (ixgbe->link_state != LINK_STATE_DOWN) {
3754 ixgbe->link_speed = 0;
3755 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3756 ixgbe->link_state = LINK_STATE_DOWN;
3757 link_changed = B_TRUE;
3758 }
3759 }
3760 }
3761
3762 /*
3763 * If we are in an interrupt context, need to re-enable the
3764 * interrupt, which was automasked
3765 */
3766 if (servicing_interrupt() != 0) {
3767 ixgbe->eims |= IXGBE_EICR_LSC;
3768 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3769 }
3770
3771 if (link_changed) {
3772 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3773 }
3774 }
3775
3776 /*
3777 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3778 */
3779 static void
3780 ixgbe_sfp_check(void *arg)
3781 {
3782 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3783 uint32_t eicr = ixgbe->eicr;
3784 struct ixgbe_hw *hw = &ixgbe->hw;
3785
3786 mutex_enter(&ixgbe->gen_lock);
3787 (void) hw->phy.ops.identify_sfp(hw);
3788 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3789 /* clear the interrupt */
3790 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3791
3792 /* if link up, do multispeed fiber setup */
3793 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3794 B_TRUE);
3795 ixgbe_driver_link_check(ixgbe);
3796 ixgbe_get_hw_state(ixgbe);
3797 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3798 /* clear the interrupt */
3799 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3800
3801 /* if link up, do sfp module setup */
3802 (void) hw->mac.ops.setup_sfp(hw);
3803
3804 /* do multispeed fiber setup */
3805 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3806 B_TRUE);
3807 ixgbe_driver_link_check(ixgbe);
3808 ixgbe_get_hw_state(ixgbe);
3809 }
3810 mutex_exit(&ixgbe->gen_lock);
3811
3812 /*
3813 * We need to fully re-check the link later.
3814 */
3815 ixgbe->link_check_complete = B_FALSE;
3816 ixgbe->link_check_hrtime = gethrtime() +
3817 (IXGBE_LINK_UP_TIME * 100000000ULL);
3818 }
3819
3820 /*
3821 * ixgbe_overtemp_check - overtemp module processing done in taskq
3822 *
3823 * This routine will only be called on adapters with temperature sensor.
3824 * The indication of over-temperature can be either SDP0 interrupt or the link
3825 * status change interrupt.
3826 */
3827 static void
3828 ixgbe_overtemp_check(void *arg)
3829 {
3830 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3831 struct ixgbe_hw *hw = &ixgbe->hw;
3832 uint32_t eicr = ixgbe->eicr;
3833 ixgbe_link_speed speed;
3834 boolean_t link_up;
3835
3836 mutex_enter(&ixgbe->gen_lock);
3837
3838 /* make sure we know current state of link */
3839 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3840
3841 /* check over-temp condition */
3842 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3843 (eicr & IXGBE_EICR_LSC)) {
3844 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3845 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3846
3847 /*
3848 * Disable the adapter interrupts
3849 */
3850 ixgbe_disable_adapter_interrupts(ixgbe);
3851
3852 /*
3853 * Disable Rx/Tx units
3854 */
3855 (void) ixgbe_stop_adapter(hw);
3856
3857 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3858 ixgbe_error(ixgbe,
3859 "Problem: Network adapter has been stopped "
3860 "because it has overheated");
3861 ixgbe_error(ixgbe,
3862 "Action: Restart the computer. "
3863 "If the problem persists, power off the system "
3864 "and replace the adapter");
3865 }
3866 }
3867
3868 /* write to clear the interrupt */
3869 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3870
3871 mutex_exit(&ixgbe->gen_lock);
3872 }
3873
3874 /*
3875 * ixgbe_phy_check - taskq to process interrupts from an external PHY
3876 *
3877 * This routine will only be called on adapters with external PHYs
3878 * (such as X550) that may be trying to raise our attention to some event.
3879 * Currently, this is limited to claiming PHY overtemperature and link status
3880 * change (LSC) events, however this may expand to include other things in
3881 * future adapters.
3882 */
3883 static void
3884 ixgbe_phy_check(void *arg)
3885 {
3886 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3887 struct ixgbe_hw *hw = &ixgbe->hw;
3888 int rv;
3889
3890 mutex_enter(&ixgbe->gen_lock);
3891
3892 /*
3893 * X550 baseT PHY overtemp and LSC events are handled here.
3894 *
3895 * If an overtemp event occurs, it will be reflected in the
3896 * return value of phy.ops.handle_lasi() and the common code will
3897 * automatically power off the baseT PHY. This is our cue to trigger
3898 * an FMA event.
3899 *
3900 * If a link status change event occurs, phy.ops.handle_lasi() will
3901 * automatically initiate a link setup between the integrated KR PHY
3902 * and the external X557 PHY to ensure that the link speed between
3903 * them matches the link speed of the baseT link.
3904 */
3905 rv = ixgbe_handle_lasi(hw);
3906
3907 if (rv == IXGBE_ERR_OVERTEMP) {
3908 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3909
3910 /*
3911 * Disable the adapter interrupts
3912 */
3913 ixgbe_disable_adapter_interrupts(ixgbe);
3914
3915 /*
3916 * Disable Rx/Tx units
3917 */
3918 (void) ixgbe_stop_adapter(hw);
3919
3920 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3921 ixgbe_error(ixgbe,
3922 "Problem: Network adapter has been stopped due to a "
3923 "overtemperature event being detected.");
3924 ixgbe_error(ixgbe,
3925 "Action: Shut down or restart the computer. If the issue "
3926 "persists, please take action in accordance with the "
3927 "recommendations from your system vendor.");
3928 }
3929
3930 mutex_exit(&ixgbe->gen_lock);
3931 }
3932
3933 /*
3934 * ixgbe_link_timer - timer for link status detection
3935 */
3936 static void
3937 ixgbe_link_timer(void *arg)
3938 {
3939 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3940
3941 mutex_enter(&ixgbe->gen_lock);
3942 ixgbe_driver_link_check(ixgbe);
3943 mutex_exit(&ixgbe->gen_lock);
3944 }
3945
3946 /*
3947 * ixgbe_local_timer - Driver watchdog function.
3948 *
3949 * This function will handle the transmit stall check and other routines.
3950 */
3951 static void
3952 ixgbe_local_timer(void *arg)
3953 {
3954 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3955
3956 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3957 goto out;
3958
3959 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3960 ixgbe->reset_count++;
3961 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3962 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3963 goto out;
3964 }
3965
3966 if (ixgbe_stall_check(ixgbe)) {
3967 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3968 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3969
3970 ixgbe->reset_count++;
3971 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3972 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3973 }
3974
3975 out:
3976 ixgbe_restart_watchdog_timer(ixgbe);
3977 }
3978
3979 /*
3980 * ixgbe_stall_check - Check for transmit stall.
3981 *
3982 * This function checks if the adapter is stalled (in transmit).
3983 *
3984 * It is called each time the watchdog timeout is invoked.
3985 * If the transmit descriptor reclaim continuously fails,
3986 * the watchdog value will increment by 1. If the watchdog
3987 * value exceeds the threshold, the ixgbe is assumed to
3988 * have stalled and need to be reset.
3989 */
3990 static boolean_t
3991 ixgbe_stall_check(ixgbe_t *ixgbe)
3992 {
3993 ixgbe_tx_ring_t *tx_ring;
3994 boolean_t result;
3995 int i;
3996
3997 if (ixgbe->link_state != LINK_STATE_UP)
3998 return (B_FALSE);
3999
4000 /*
4001 * If any tx ring is stalled, we'll reset the chipset
4002 */
4003 result = B_FALSE;
4004 for (i = 0; i < ixgbe->num_tx_rings; i++) {
4005 tx_ring = &ixgbe->tx_rings[i];
4006 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
4007 tx_ring->tx_recycle(tx_ring);
4008 }
4009
4010 if (tx_ring->recycle_fail > 0)
4011 tx_ring->stall_watchdog++;
4012 else
4013 tx_ring->stall_watchdog = 0;
4014
4015 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
4016 result = B_TRUE;
4017 break;
4018 }
4019 }
4020
4021 if (result) {
4022 tx_ring->stall_watchdog = 0;
4023 tx_ring->recycle_fail = 0;
4024 }
4025
4026 return (result);
4027 }
4028
4029
4030 /*
4031 * is_valid_mac_addr - Check if the mac address is valid.
4032 */
4033 static boolean_t
4034 is_valid_mac_addr(uint8_t *mac_addr)
4035 {
4036 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
4037 const uint8_t addr_test2[6] =
4038 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
4039
4040 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
4041 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
4042 return (B_FALSE);
4043
4044 return (B_TRUE);
4045 }
4046
4047 static boolean_t
4048 ixgbe_find_mac_address(ixgbe_t *ixgbe)
4049 {
4050 #ifdef __sparc
4051 struct ixgbe_hw *hw = &ixgbe->hw;
4052 uchar_t *bytes;
4053 struct ether_addr sysaddr;
4054 uint_t nelts;
4055 int err;
4056 boolean_t found = B_FALSE;
4057
4058 /*
4059 * The "vendor's factory-set address" may already have
4060 * been extracted from the chip, but if the property
4061 * "local-mac-address" is set we use that instead.
4062 *
4063 * We check whether it looks like an array of 6
4064 * bytes (which it should, if OBP set it). If we can't
4065 * make sense of it this way, we'll ignore it.
4066 */
4067 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4068 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
4069 if (err == DDI_PROP_SUCCESS) {
4070 if (nelts == ETHERADDRL) {
4071 while (nelts--)
4072 hw->mac.addr[nelts] = bytes[nelts];
4073 found = B_TRUE;
4074 }
4075 ddi_prop_free(bytes);
4076 }
4077
4078 /*
4079 * Look up the OBP property "local-mac-address?". If the user has set
4080 * 'local-mac-address? = false', use "the system address" instead.
4081 */
4082 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
4083 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
4084 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
4085 if (localetheraddr(NULL, &sysaddr) != 0) {
4086 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
4087 found = B_TRUE;
4088 }
4089 }
4090 ddi_prop_free(bytes);
4091 }
4092
4093 /*
4094 * Finally(!), if there's a valid "mac-address" property (created
4095 * if we netbooted from this interface), we must use this instead
4096 * of any of the above to ensure that the NFS/install server doesn't
4097 * get confused by the address changing as illumos takes over!
4098 */
4099 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4100 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
4101 if (err == DDI_PROP_SUCCESS) {
4102 if (nelts == ETHERADDRL) {
4103 while (nelts--)
4104 hw->mac.addr[nelts] = bytes[nelts];
4105 found = B_TRUE;
4106 }
4107 ddi_prop_free(bytes);
4108 }
4109
4110 if (found) {
4111 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
4112 return (B_TRUE);
4113 }
4114 #else
4115 _NOTE(ARGUNUSED(ixgbe));
4116 #endif
4117
4118 return (B_TRUE);
4119 }
4120
4121 #pragma inline(ixgbe_arm_watchdog_timer)
4122 static void
4123 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
4124 {
4125 /*
4126 * Fire a watchdog timer
4127 */
4128 ixgbe->watchdog_tid =
4129 timeout(ixgbe_local_timer,
4130 (void *)ixgbe, 1 * drv_usectohz(1000000));
4131
4132 }
4133
4134 /*
4135 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
4136 */
4137 void
4138 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
4139 {
4140 mutex_enter(&ixgbe->watchdog_lock);
4141
4142 if (!ixgbe->watchdog_enable) {
4143 ixgbe->watchdog_enable = B_TRUE;
4144 ixgbe->watchdog_start = B_TRUE;
4145 ixgbe_arm_watchdog_timer(ixgbe);
4146 }
4147
4148 mutex_exit(&ixgbe->watchdog_lock);
4149 }
4150
4151 /*
4152 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
4153 */
4154 void
4155 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
4156 {
4157 timeout_id_t tid;
4158
4159 mutex_enter(&ixgbe->watchdog_lock);
4160
4161 ixgbe->watchdog_enable = B_FALSE;
4162 ixgbe->watchdog_start = B_FALSE;
4163 tid = ixgbe->watchdog_tid;
4164 ixgbe->watchdog_tid = 0;
4165
4166 mutex_exit(&ixgbe->watchdog_lock);
4167
4168 if (tid != 0)
4169 (void) untimeout(tid);
4170 }
4171
4172 /*
4173 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
4174 */
4175 void
4176 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
4177 {
4178 mutex_enter(&ixgbe->watchdog_lock);
4179
4180 if (ixgbe->watchdog_enable) {
4181 if (!ixgbe->watchdog_start) {
4182 ixgbe->watchdog_start = B_TRUE;
4183 ixgbe_arm_watchdog_timer(ixgbe);
4184 }
4185 }
4186
4187 mutex_exit(&ixgbe->watchdog_lock);
4188 }
4189
4190 /*
4191 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
4192 */
4193 static void
4194 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
4195 {
4196 mutex_enter(&ixgbe->watchdog_lock);
4197
4198 if (ixgbe->watchdog_start)
4199 ixgbe_arm_watchdog_timer(ixgbe);
4200
4201 mutex_exit(&ixgbe->watchdog_lock);
4202 }
4203
4204 /*
4205 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
4206 */
4207 void
4208 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
4209 {
4210 timeout_id_t tid;
4211
4212 mutex_enter(&ixgbe->watchdog_lock);
4213
4214 ixgbe->watchdog_start = B_FALSE;
4215 tid = ixgbe->watchdog_tid;
4216 ixgbe->watchdog_tid = 0;
4217
4218 mutex_exit(&ixgbe->watchdog_lock);
4219
4220 if (tid != 0)
4221 (void) untimeout(tid);
4222 }
4223
4224 /*
4225 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
4226 */
4227 static void
4228 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
4229 {
4230 struct ixgbe_hw *hw = &ixgbe->hw;
4231
4232 /*
4233 * mask all interrupts off
4234 */
4235 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
4236
4237 /*
4238 * for MSI-X, also disable autoclear
4239 */
4240 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4241 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
4242 }
4243
4244 IXGBE_WRITE_FLUSH(hw);
4245 }
4246
4247 /*
4248 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
4249 */
4250 static void
4251 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
4252 {
4253 struct ixgbe_hw *hw = &ixgbe->hw;
4254 uint32_t eiac, eiam;
4255 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4256
4257 /* interrupt types to enable */
4258 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
4259 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
4260 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
4261
4262 /* enable automask on "other" causes that this adapter can generate */
4263 eiam = ixgbe->capab->other_intr;
4264
4265 /*
4266 * msi-x mode
4267 */
4268 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4269 /* enable autoclear but not on bits 29:20 */
4270 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
4271
4272 /* general purpose interrupt enable */
4273 gpie |= (IXGBE_GPIE_MSIX_MODE
4274 | IXGBE_GPIE_PBA_SUPPORT
4275 | IXGBE_GPIE_OCD
4276 | IXGBE_GPIE_EIAME);
4277 /*
4278 * non-msi-x mode
4279 */
4280 } else {
4281
4282 /* disable autoclear, leave gpie at default */
4283 eiac = 0;
4284
4285 /*
4286 * General purpose interrupt enable.
4287 * For 82599, X540 and X550, extended interrupt
4288 * automask enable only in MSI or MSI-X mode
4289 */
4290 if ((hw->mac.type == ixgbe_mac_82598EB) ||
4291 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
4292 gpie |= IXGBE_GPIE_EIAME;
4293 }
4294 }
4295
4296 /* Enable specific "other" interrupt types */
4297 switch (hw->mac.type) {
4298 case ixgbe_mac_82598EB:
4299 gpie |= ixgbe->capab->other_gpie;
4300 break;
4301
4302 case ixgbe_mac_82599EB:
4303 case ixgbe_mac_X540:
4304 case ixgbe_mac_X550:
4305 case ixgbe_mac_X550EM_x:
4306 case ixgbe_mac_X550EM_a:
4307 gpie |= ixgbe->capab->other_gpie;
4308
4309 /* Enable RSC Delay 8us when LRO enabled */
4310 if (ixgbe->lro_enable) {
4311 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
4312 }
4313 break;
4314
4315 default:
4316 break;
4317 }
4318
4319 /* write to interrupt control registers */
4320 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4321 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
4322 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
4323 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4324 IXGBE_WRITE_FLUSH(hw);
4325 }
4326
4327 /*
4328 * ixgbe_loopback_ioctl - Loopback support.
4329 */
4330 enum ioc_reply
4331 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
4332 {
4333 lb_info_sz_t *lbsp;
4334 lb_property_t *lbpp;
4335 uint32_t *lbmp;
4336 uint32_t size;
4337 uint32_t value;
4338
4339 if (mp->b_cont == NULL)
4340 return (IOC_INVAL);
4341
4342 switch (iocp->ioc_cmd) {
4343 default:
4344 return (IOC_INVAL);
4345
4346 case LB_GET_INFO_SIZE:
4347 size = sizeof (lb_info_sz_t);
4348 if (iocp->ioc_count != size)
4349 return (IOC_INVAL);
4350
4351 value = sizeof (lb_normal);
4352 value += sizeof (lb_mac);
4353 value += sizeof (lb_external);
4354
4355 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
4356 *lbsp = value;
4357 break;
4358
4359 case LB_GET_INFO:
4360 value = sizeof (lb_normal);
4361 value += sizeof (lb_mac);
4362 value += sizeof (lb_external);
4363
4364 size = value;
4365 if (iocp->ioc_count != size)
4366 return (IOC_INVAL);
4367
4368 value = 0;
4369 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
4370
4371 lbpp[value++] = lb_normal;
4372 lbpp[value++] = lb_mac;
4373 lbpp[value++] = lb_external;
4374 break;
4375
4376 case LB_GET_MODE:
4377 size = sizeof (uint32_t);
4378 if (iocp->ioc_count != size)
4379 return (IOC_INVAL);
4380
4381 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4382 *lbmp = ixgbe->loopback_mode;
4383 break;
4384
4385 case LB_SET_MODE:
4386 size = 0;
4387 if (iocp->ioc_count != sizeof (uint32_t))
4388 return (IOC_INVAL);
4389
4390 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4391 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
4392 return (IOC_INVAL);
4393 break;
4394 }
4395
4396 iocp->ioc_count = size;
4397 iocp->ioc_error = 0;
4398
4399 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4400 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4401 return (IOC_INVAL);
4402 }
4403
4404 return (IOC_REPLY);
4405 }
4406
4407 /*
4408 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
4409 */
4410 static boolean_t
4411 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
4412 {
4413 if (mode == ixgbe->loopback_mode)
4414 return (B_TRUE);
4415
4416 ixgbe->loopback_mode = mode;
4417
4418 if (mode == IXGBE_LB_NONE) {
4419 /*
4420 * Reset the chip
4421 */
4422 (void) ixgbe_reset(ixgbe);
4423 return (B_TRUE);
4424 }
4425
4426 mutex_enter(&ixgbe->gen_lock);
4427
4428 switch (mode) {
4429 default:
4430 mutex_exit(&ixgbe->gen_lock);
4431 return (B_FALSE);
4432
4433 case IXGBE_LB_EXTERNAL:
4434 break;
4435
4436 case IXGBE_LB_INTERNAL_MAC:
4437 ixgbe_set_internal_mac_loopback(ixgbe);
4438 break;
4439 }
4440
4441 mutex_exit(&ixgbe->gen_lock);
4442
4443 return (B_TRUE);
4444 }
4445
4446 /*
4447 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4448 */
4449 static void
4450 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4451 {
4452 struct ixgbe_hw *hw;
4453 uint32_t reg;
4454 uint8_t atlas;
4455
4456 hw = &ixgbe->hw;
4457
4458 /*
4459 * Setup MAC loopback
4460 */
4461 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4462 reg |= IXGBE_HLREG0_LPBK;
4463 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4464
4465 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4466 reg &= ~IXGBE_AUTOC_LMS_MASK;
4467 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4468
4469 /*
4470 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4471 */
4472 switch (hw->mac.type) {
4473 case ixgbe_mac_82598EB:
4474 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4475 &atlas);
4476 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4477 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4478 atlas);
4479
4480 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4481 &atlas);
4482 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4483 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4484 atlas);
4485
4486 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4487 &atlas);
4488 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4489 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4490 atlas);
4491
4492 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4493 &atlas);
4494 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4495 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4496 atlas);
4497 break;
4498
4499 case ixgbe_mac_82599EB:
4500 case ixgbe_mac_X540:
4501 case ixgbe_mac_X550:
4502 case ixgbe_mac_X550EM_x:
4503 case ixgbe_mac_X550EM_a:
4504 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4505 reg |= (IXGBE_AUTOC_FLU |
4506 IXGBE_AUTOC_10G_KX4);
4507 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4508
4509 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4510 B_FALSE);
4511 break;
4512
4513 default:
4514 break;
4515 }
4516 }
4517
4518 #pragma inline(ixgbe_intr_rx_work)
4519 /*
4520 * ixgbe_intr_rx_work - RX processing of ISR.
4521 */
4522 static void
4523 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4524 {
4525 mblk_t *mp;
4526
4527 mutex_enter(&rx_ring->rx_lock);
4528
4529 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4530 mutex_exit(&rx_ring->rx_lock);
4531
4532 if (mp != NULL)
4533 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4534 rx_ring->ring_gen_num);
4535 }
4536
4537 #pragma inline(ixgbe_intr_tx_work)
4538 /*
4539 * ixgbe_intr_tx_work - TX processing of ISR.
4540 */
4541 static void
4542 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4543 {
4544 ixgbe_t *ixgbe = tx_ring->ixgbe;
4545
4546 /*
4547 * Recycle the tx descriptors
4548 */
4549 tx_ring->tx_recycle(tx_ring);
4550
4551 /*
4552 * Schedule the re-transmit
4553 */
4554 if (tx_ring->reschedule &&
4555 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4556 tx_ring->reschedule = B_FALSE;
4557 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4558 tx_ring->ring_handle);
4559 tx_ring->stat_reschedule++;
4560 }
4561 }
4562
4563 #pragma inline(ixgbe_intr_other_work)
4564 /*
4565 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4566 */
4567 static void
4568 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4569 {
4570 struct ixgbe_hw *hw = &ixgbe->hw;
4571
4572 ASSERT(mutex_owned(&ixgbe->gen_lock));
4573
4574 /*
4575 * handle link status change
4576 */
4577 if (eicr & IXGBE_EICR_LSC) {
4578 ixgbe_driver_link_check(ixgbe);
4579 ixgbe_get_hw_state(ixgbe);
4580 }
4581
4582 /*
4583 * check for fan failure on adapters with fans
4584 */
4585 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4586 (eicr & IXGBE_EICR_GPI_SDP1)) {
4587 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4588
4589 /*
4590 * Disable the adapter interrupts
4591 */
4592 ixgbe_disable_adapter_interrupts(ixgbe);
4593
4594 /*
4595 * Disable Rx/Tx units
4596 */
4597 (void) ixgbe_stop_adapter(&ixgbe->hw);
4598
4599 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4600 ixgbe_error(ixgbe,
4601 "Problem: Network adapter has been stopped "
4602 "because the fan has stopped.\n");
4603 ixgbe_error(ixgbe,
4604 "Action: Replace the adapter.\n");
4605
4606 /* re-enable the interrupt, which was automasked */
4607 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4608 }
4609
4610 /*
4611 * Do SFP check for adapters with hot-plug capability
4612 */
4613 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4614 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4615 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4616 ixgbe->eicr = eicr;
4617 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4618 ixgbe_sfp_check, (void *)ixgbe,
4619 DDI_NOSLEEP)) != DDI_SUCCESS) {
4620 ixgbe_log(ixgbe, "No memory available to dispatch "
4621 "taskq for SFP check");
4622 }
4623 }
4624
4625 /*
4626 * Do over-temperature check for adapters with temp sensor
4627 */
4628 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4629 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) ||
4630 (eicr & IXGBE_EICR_LSC))) {
4631 ixgbe->eicr = eicr;
4632 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4633 ixgbe_overtemp_check, (void *)ixgbe,
4634 DDI_NOSLEEP)) != DDI_SUCCESS) {
4635 ixgbe_log(ixgbe, "No memory available to dispatch "
4636 "taskq for overtemp check");
4637 }
4638 }
4639
4640 /*
4641 * Process an external PHY interrupt
4642 */
4643 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
4644 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
4645 ixgbe->eicr = eicr;
4646 if ((ddi_taskq_dispatch(ixgbe->phy_taskq,
4647 ixgbe_phy_check, (void *)ixgbe,
4648 DDI_NOSLEEP)) != DDI_SUCCESS) {
4649 ixgbe_log(ixgbe, "No memory available to dispatch "
4650 "taskq for PHY check");
4651 }
4652 }
4653 }
4654
4655 /*
4656 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4657 */
4658 static uint_t
4659 ixgbe_intr_legacy(void *arg1, void *arg2)
4660 {
4661 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4662 struct ixgbe_hw *hw = &ixgbe->hw;
4663 ixgbe_tx_ring_t *tx_ring;
4664 ixgbe_rx_ring_t *rx_ring;
4665 uint32_t eicr;
4666 mblk_t *mp;
4667 boolean_t tx_reschedule;
4668 uint_t result;
4669
4670 _NOTE(ARGUNUSED(arg2));
4671
4672 mutex_enter(&ixgbe->gen_lock);
4673 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4674 mutex_exit(&ixgbe->gen_lock);
4675 return (DDI_INTR_UNCLAIMED);
4676 }
4677
4678 mp = NULL;
4679 tx_reschedule = B_FALSE;
4680
4681 /*
4682 * Any bit set in eicr: claim this interrupt
4683 */
4684 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4685
4686 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4687 mutex_exit(&ixgbe->gen_lock);
4688 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4689 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4690 return (DDI_INTR_CLAIMED);
4691 }
4692
4693 if (eicr) {
4694 /*
4695 * For legacy interrupt, we have only one interrupt,
4696 * so we have only one rx ring and one tx ring enabled.
4697 */
4698 ASSERT(ixgbe->num_rx_rings == 1);
4699 ASSERT(ixgbe->num_tx_rings == 1);
4700
4701 /*
4702 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4703 */
4704 if (eicr & 0x1) {
4705 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4706 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4707 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4708 /*
4709 * Clean the rx descriptors
4710 */
4711 rx_ring = &ixgbe->rx_rings[0];
4712 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4713 }
4714
4715 /*
4716 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4717 */
4718 if (eicr & 0x2) {
4719 /*
4720 * Recycle the tx descriptors
4721 */
4722 tx_ring = &ixgbe->tx_rings[0];
4723 tx_ring->tx_recycle(tx_ring);
4724
4725 /*
4726 * Schedule the re-transmit
4727 */
4728 tx_reschedule = (tx_ring->reschedule &&
4729 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4730 }
4731
4732 /* any interrupt type other than tx/rx */
4733 if (eicr & ixgbe->capab->other_intr) {
4734 switch (hw->mac.type) {
4735 case ixgbe_mac_82598EB:
4736 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4737 break;
4738
4739 case ixgbe_mac_82599EB:
4740 case ixgbe_mac_X540:
4741 case ixgbe_mac_X550:
4742 case ixgbe_mac_X550EM_x:
4743 case ixgbe_mac_X550EM_a:
4744 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4745 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4746 break;
4747
4748 default:
4749 break;
4750 }
4751 ixgbe_intr_other_work(ixgbe, eicr);
4752 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4753 }
4754
4755 mutex_exit(&ixgbe->gen_lock);
4756
4757 result = DDI_INTR_CLAIMED;
4758 } else {
4759 mutex_exit(&ixgbe->gen_lock);
4760
4761 /*
4762 * No interrupt cause bits set: don't claim this interrupt.
4763 */
4764 result = DDI_INTR_UNCLAIMED;
4765 }
4766
4767 /* re-enable the interrupts which were automasked */
4768 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4769
4770 /*
4771 * Do the following work outside of the gen_lock
4772 */
4773 if (mp != NULL) {
4774 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4775 rx_ring->ring_gen_num);
4776 }
4777
4778 if (tx_reschedule) {
4779 tx_ring->reschedule = B_FALSE;
4780 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4781 tx_ring->stat_reschedule++;
4782 }
4783
4784 return (result);
4785 }
4786
4787 /*
4788 * ixgbe_intr_msi - Interrupt handler for MSI.
4789 */
4790 static uint_t
4791 ixgbe_intr_msi(void *arg1, void *arg2)
4792 {
4793 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4794 struct ixgbe_hw *hw = &ixgbe->hw;
4795 uint32_t eicr;
4796
4797 _NOTE(ARGUNUSED(arg2));
4798
4799 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4800
4801 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4802 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4803 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4804 return (DDI_INTR_CLAIMED);
4805 }
4806
4807 /*
4808 * For MSI interrupt, we have only one vector,
4809 * so we have only one rx ring and one tx ring enabled.
4810 */
4811 ASSERT(ixgbe->num_rx_rings == 1);
4812 ASSERT(ixgbe->num_tx_rings == 1);
4813
4814 /*
4815 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4816 */
4817 if (eicr & 0x1) {
4818 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4819 }
4820
4821 /*
4822 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4823 */
4824 if (eicr & 0x2) {
4825 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4826 }
4827
4828 /* any interrupt type other than tx/rx */
4829 if (eicr & ixgbe->capab->other_intr) {
4830 mutex_enter(&ixgbe->gen_lock);
4831 switch (hw->mac.type) {
4832 case ixgbe_mac_82598EB:
4833 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4834 break;
4835
4836 case ixgbe_mac_82599EB:
4837 case ixgbe_mac_X540:
4838 case ixgbe_mac_X550:
4839 case ixgbe_mac_X550EM_x:
4840 case ixgbe_mac_X550EM_a:
4841 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4842 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4843 break;
4844
4845 default:
4846 break;
4847 }
4848 ixgbe_intr_other_work(ixgbe, eicr);
4849 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4850 mutex_exit(&ixgbe->gen_lock);
4851 }
4852
4853 /* re-enable the interrupts which were automasked */
4854 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4855
4856 return (DDI_INTR_CLAIMED);
4857 }
4858
4859 /*
4860 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4861 */
4862 static uint_t
4863 ixgbe_intr_msix(void *arg1, void *arg2)
4864 {
4865 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4866 ixgbe_t *ixgbe = vect->ixgbe;
4867 struct ixgbe_hw *hw = &ixgbe->hw;
4868 uint32_t eicr;
4869 int r_idx = 0;
4870
4871 _NOTE(ARGUNUSED(arg2));
4872
4873 /*
4874 * Clean each rx ring that has its bit set in the map
4875 */
4876 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4877 while (r_idx >= 0) {
4878 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4879 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4880 (ixgbe->num_rx_rings - 1));
4881 }
4882
4883 /*
4884 * Clean each tx ring that has its bit set in the map
4885 */
4886 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4887 while (r_idx >= 0) {
4888 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4889 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4890 (ixgbe->num_tx_rings - 1));
4891 }
4892
4893
4894 /*
4895 * Clean other interrupt (link change) that has its bit set in the map
4896 */
4897 if (BT_TEST(vect->other_map, 0) == 1) {
4898 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4899
4900 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4901 DDI_FM_OK) {
4902 ddi_fm_service_impact(ixgbe->dip,
4903 DDI_SERVICE_DEGRADED);
4904 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4905 return (DDI_INTR_CLAIMED);
4906 }
4907
4908 /*
4909 * Check "other" cause bits: any interrupt type other than tx/rx
4910 */
4911 if (eicr & ixgbe->capab->other_intr) {
4912 mutex_enter(&ixgbe->gen_lock);
4913 switch (hw->mac.type) {
4914 case ixgbe_mac_82598EB:
4915 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4916 ixgbe_intr_other_work(ixgbe, eicr);
4917 break;
4918
4919 case ixgbe_mac_82599EB:
4920 case ixgbe_mac_X540:
4921 case ixgbe_mac_X550:
4922 case ixgbe_mac_X550EM_x:
4923 case ixgbe_mac_X550EM_a:
4924 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4925 ixgbe_intr_other_work(ixgbe, eicr);
4926 break;
4927
4928 default:
4929 break;
4930 }
4931 mutex_exit(&ixgbe->gen_lock);
4932 }
4933
4934 /* re-enable the interrupts which were automasked */
4935 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4936 }
4937
4938 return (DDI_INTR_CLAIMED);
4939 }
4940
4941 /*
4942 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4943 *
4944 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4945 * if not successful, try Legacy.
4946 * ixgbe->intr_force can be used to force sequence to start with
4947 * any of the 3 types.
4948 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4949 */
4950 static int
4951 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4952 {
4953 dev_info_t *devinfo;
4954 int intr_types;
4955 int rc;
4956
4957 devinfo = ixgbe->dip;
4958
4959 /*
4960 * Get supported interrupt types
4961 */
4962 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4963
4964 if (rc != DDI_SUCCESS) {
4965 ixgbe_log(ixgbe,
4966 "Get supported interrupt types failed: %d", rc);
4967 return (IXGBE_FAILURE);
4968 }
4969 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4970
4971 ixgbe->intr_type = 0;
4972
4973 /*
4974 * Install MSI-X interrupts
4975 */
4976 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4977 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4978 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4979 if (rc == IXGBE_SUCCESS)
4980 return (IXGBE_SUCCESS);
4981
4982 ixgbe_log(ixgbe,
4983 "Allocate MSI-X failed, trying MSI interrupts...");
4984 }
4985
4986 /*
4987 * MSI-X not used, force rings and groups to 1
4988 */
4989 ixgbe->num_rx_rings = 1;
4990 ixgbe->num_rx_groups = 1;
4991 ixgbe->num_tx_rings = 1;
4992 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4993 ixgbe_log(ixgbe,
4994 "MSI-X not used, force rings and groups number to 1");
4995
4996 /*
4997 * Install MSI interrupts
4998 */
4999 if ((intr_types & DDI_INTR_TYPE_MSI) &&
5000 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
5001 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
5002 if (rc == IXGBE_SUCCESS)
5003 return (IXGBE_SUCCESS);
5004
5005 ixgbe_log(ixgbe,
5006 "Allocate MSI failed, trying Legacy interrupts...");
5007 }
5008
5009 /*
5010 * Install legacy interrupts
5011 */
5012 if (intr_types & DDI_INTR_TYPE_FIXED) {
5013 /*
5014 * Disallow legacy interrupts for X550. X550 has a silicon
5015 * bug which prevents Shared Legacy interrupts from working.
5016 * For details, please reference:
5017 *
5018 * Intel Ethernet Controller X550 Specification Update rev. 2.1
5019 * May 2016, erratum 22: PCIe Interrupt Status Bit
5020 */
5021 if (ixgbe->hw.mac.type == ixgbe_mac_X550 ||
5022 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x ||
5023 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a ||
5024 ixgbe->hw.mac.type == ixgbe_mac_X550_vf ||
5025 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf ||
5026 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a_vf) {
5027 ixgbe_log(ixgbe,
5028 "Legacy interrupts are not supported on this "
5029 "adapter. Please use MSI or MSI-X instead.");
5030 return (IXGBE_FAILURE);
5031 }
5032 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
5033 if (rc == IXGBE_SUCCESS)
5034 return (IXGBE_SUCCESS);
5035
5036 ixgbe_log(ixgbe,
5037 "Allocate Legacy interrupts failed");
5038 }
5039
5040 /*
5041 * If none of the 3 types succeeded, return failure
5042 */
5043 return (IXGBE_FAILURE);
5044 }
5045
5046 /*
5047 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
5048 *
5049 * For legacy and MSI, only 1 handle is needed. For MSI-X,
5050 * if fewer than 2 handles are available, return failure.
5051 * Upon success, this maps the vectors to rx and tx rings for
5052 * interrupts.
5053 */
5054 static int
5055 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
5056 {
5057 dev_info_t *devinfo;
5058 int request, count, actual;
5059 int minimum;
5060 int rc;
5061 uint32_t ring_per_group;
5062
5063 devinfo = ixgbe->dip;
5064
5065 switch (intr_type) {
5066 case DDI_INTR_TYPE_FIXED:
5067 request = 1; /* Request 1 legacy interrupt handle */
5068 minimum = 1;
5069 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
5070 break;
5071
5072 case DDI_INTR_TYPE_MSI:
5073 request = 1; /* Request 1 MSI interrupt handle */
5074 minimum = 1;
5075 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
5076 break;
5077
5078 case DDI_INTR_TYPE_MSIX:
5079 /*
5080 * Best number of vectors for the adapter is
5081 * (# rx rings + # tx rings), however we will
5082 * limit the request number.
5083 */
5084 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
5085 if (request > ixgbe->capab->max_ring_vect)
5086 request = ixgbe->capab->max_ring_vect;
5087 minimum = 1;
5088 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
5089 break;
5090
5091 default:
5092 ixgbe_log(ixgbe,
5093 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
5094 intr_type);
5095 return (IXGBE_FAILURE);
5096 }
5097 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
5098 request, minimum);
5099
5100 /*
5101 * Get number of supported interrupts
5102 */
5103 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
5104 if ((rc != DDI_SUCCESS) || (count < minimum)) {
5105 ixgbe_log(ixgbe,
5106 "Get interrupt number failed. Return: %d, count: %d",
5107 rc, count);
5108 return (IXGBE_FAILURE);
5109 }
5110 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
5111
5112 actual = 0;
5113 ixgbe->intr_cnt = 0;
5114 ixgbe->intr_cnt_max = 0;
5115 ixgbe->intr_cnt_min = 0;
5116
5117 /*
5118 * Allocate an array of interrupt handles
5119 */
5120 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
5121 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
5122
5123 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
5124 request, &actual, DDI_INTR_ALLOC_NORMAL);
5125 if (rc != DDI_SUCCESS) {
5126 ixgbe_log(ixgbe, "Allocate interrupts failed. "
5127 "return: %d, request: %d, actual: %d",
5128 rc, request, actual);
5129 goto alloc_handle_fail;
5130 }
5131 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
5132
5133 /*
5134 * upper/lower limit of interrupts
5135 */
5136 ixgbe->intr_cnt = actual;
5137 ixgbe->intr_cnt_max = request;
5138 ixgbe->intr_cnt_min = minimum;
5139
5140 /*
5141 * rss number per group should not exceed the rx interrupt number,
5142 * else need to adjust rx ring number.
5143 */
5144 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5145 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
5146 if (actual < ring_per_group) {
5147 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
5148 ixgbe_setup_vmdq_rss_conf(ixgbe);
5149 }
5150
5151 /*
5152 * Now we know the actual number of vectors. Here we map the vector
5153 * to other, rx rings and tx ring.
5154 */
5155 if (actual < minimum) {
5156 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
5157 actual);
5158 goto alloc_handle_fail;
5159 }
5160
5161 /*
5162 * Get priority for first vector, assume remaining are all the same
5163 */
5164 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
5165 if (rc != DDI_SUCCESS) {
5166 ixgbe_log(ixgbe,
5167 "Get interrupt priority failed: %d", rc);
5168 goto alloc_handle_fail;
5169 }
5170
5171 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
5172 if (rc != DDI_SUCCESS) {
5173 ixgbe_log(ixgbe,
5174 "Get interrupt cap failed: %d", rc);
5175 goto alloc_handle_fail;
5176 }
5177
5178 ixgbe->intr_type = intr_type;
5179
5180 return (IXGBE_SUCCESS);
5181
5182 alloc_handle_fail:
5183 ixgbe_rem_intrs(ixgbe);
5184
5185 return (IXGBE_FAILURE);
5186 }
5187
5188 /*
5189 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
5190 *
5191 * Before adding the interrupt handlers, the interrupt vectors have
5192 * been allocated, and the rx/tx rings have also been allocated.
5193 */
5194 static int
5195 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
5196 {
5197 int vector = 0;
5198 int rc;
5199
5200 switch (ixgbe->intr_type) {
5201 case DDI_INTR_TYPE_MSIX:
5202 /*
5203 * Add interrupt handler for all vectors
5204 */
5205 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
5206 /*
5207 * install pointer to vect_map[vector]
5208 */
5209 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5210 (ddi_intr_handler_t *)ixgbe_intr_msix,
5211 (void *)&ixgbe->vect_map[vector], NULL);
5212
5213 if (rc != DDI_SUCCESS) {
5214 ixgbe_log(ixgbe,
5215 "Add interrupt handler failed. "
5216 "return: %d, vector: %d", rc, vector);
5217 for (vector--; vector >= 0; vector--) {
5218 (void) ddi_intr_remove_handler(
5219 ixgbe->htable[vector]);
5220 }
5221 return (IXGBE_FAILURE);
5222 }
5223 }
5224
5225 break;
5226
5227 case DDI_INTR_TYPE_MSI:
5228 /*
5229 * Add interrupt handlers for the only vector
5230 */
5231 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5232 (ddi_intr_handler_t *)ixgbe_intr_msi,
5233 (void *)ixgbe, NULL);
5234
5235 if (rc != DDI_SUCCESS) {
5236 ixgbe_log(ixgbe,
5237 "Add MSI interrupt handler failed: %d", rc);
5238 return (IXGBE_FAILURE);
5239 }
5240
5241 break;
5242
5243 case DDI_INTR_TYPE_FIXED:
5244 /*
5245 * Add interrupt handlers for the only vector
5246 */
5247 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5248 (ddi_intr_handler_t *)ixgbe_intr_legacy,
5249 (void *)ixgbe, NULL);
5250
5251 if (rc != DDI_SUCCESS) {
5252 ixgbe_log(ixgbe,
5253 "Add legacy interrupt handler failed: %d", rc);
5254 return (IXGBE_FAILURE);
5255 }
5256
5257 break;
5258
5259 default:
5260 return (IXGBE_FAILURE);
5261 }
5262
5263 return (IXGBE_SUCCESS);
5264 }
5265
5266 #pragma inline(ixgbe_map_rxring_to_vector)
5267 /*
5268 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
5269 */
5270 static void
5271 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
5272 {
5273 /*
5274 * Set bit in map
5275 */
5276 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5277
5278 /*
5279 * Count bits set
5280 */
5281 ixgbe->vect_map[v_idx].rxr_cnt++;
5282
5283 /*
5284 * Remember bit position
5285 */
5286 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
5287 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
5288 }
5289
5290 #pragma inline(ixgbe_map_txring_to_vector)
5291 /*
5292 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
5293 */
5294 static void
5295 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
5296 {
5297 /*
5298 * Set bit in map
5299 */
5300 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
5301
5302 /*
5303 * Count bits set
5304 */
5305 ixgbe->vect_map[v_idx].txr_cnt++;
5306
5307 /*
5308 * Remember bit position
5309 */
5310 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
5311 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
5312 }
5313
5314 /*
5315 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
5316 * allocation register (IVAR).
5317 * cause:
5318 * -1 : other cause
5319 * 0 : rx
5320 * 1 : tx
5321 */
5322 static void
5323 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
5324 int8_t cause)
5325 {
5326 struct ixgbe_hw *hw = &ixgbe->hw;
5327 u32 ivar, index;
5328
5329 switch (hw->mac.type) {
5330 case ixgbe_mac_82598EB:
5331 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5332 if (cause == -1) {
5333 cause = 0;
5334 }
5335 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5336 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5337 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
5338 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
5339 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5340 break;
5341
5342 case ixgbe_mac_82599EB:
5343 case ixgbe_mac_X540:
5344 case ixgbe_mac_X550:
5345 case ixgbe_mac_X550EM_x:
5346 case ixgbe_mac_X550EM_a:
5347 if (cause == -1) {
5348 /* other causes */
5349 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5350 index = (intr_alloc_entry & 1) * 8;
5351 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5352 ivar &= ~(0xFF << index);
5353 ivar |= (msix_vector << index);
5354 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5355 } else {
5356 /* tx or rx causes */
5357 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5358 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5359 ivar = IXGBE_READ_REG(hw,
5360 IXGBE_IVAR(intr_alloc_entry >> 1));
5361 ivar &= ~(0xFF << index);
5362 ivar |= (msix_vector << index);
5363 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5364 ivar);
5365 }
5366 break;
5367
5368 default:
5369 break;
5370 }
5371 }
5372
5373 /*
5374 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
5375 * given interrupt vector allocation register (IVAR).
5376 * cause:
5377 * -1 : other cause
5378 * 0 : rx
5379 * 1 : tx
5380 */
5381 static void
5382 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5383 {
5384 struct ixgbe_hw *hw = &ixgbe->hw;
5385 u32 ivar, index;
5386
5387 switch (hw->mac.type) {
5388 case ixgbe_mac_82598EB:
5389 if (cause == -1) {
5390 cause = 0;
5391 }
5392 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5393 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5394 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
5395 (intr_alloc_entry & 0x3)));
5396 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5397 break;
5398
5399 case ixgbe_mac_82599EB:
5400 case ixgbe_mac_X540:
5401 case ixgbe_mac_X550:
5402 case ixgbe_mac_X550EM_x:
5403 case ixgbe_mac_X550EM_a:
5404 if (cause == -1) {
5405 /* other causes */
5406 index = (intr_alloc_entry & 1) * 8;
5407 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5408 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5409 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5410 } else {
5411 /* tx or rx causes */
5412 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5413 ivar = IXGBE_READ_REG(hw,
5414 IXGBE_IVAR(intr_alloc_entry >> 1));
5415 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5416 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5417 ivar);
5418 }
5419 break;
5420
5421 default:
5422 break;
5423 }
5424 }
5425
5426 /*
5427 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
5428 * given interrupt vector allocation register (IVAR).
5429 * cause:
5430 * -1 : other cause
5431 * 0 : rx
5432 * 1 : tx
5433 */
5434 static void
5435 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5436 {
5437 struct ixgbe_hw *hw = &ixgbe->hw;
5438 u32 ivar, index;
5439
5440 switch (hw->mac.type) {
5441 case ixgbe_mac_82598EB:
5442 if (cause == -1) {
5443 cause = 0;
5444 }
5445 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5446 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5447 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5448 (intr_alloc_entry & 0x3)));
5449 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5450 break;
5451
5452 case ixgbe_mac_82599EB:
5453 case ixgbe_mac_X540:
5454 case ixgbe_mac_X550:
5455 case ixgbe_mac_X550EM_x:
5456 case ixgbe_mac_X550EM_a:
5457 if (cause == -1) {
5458 /* other causes */
5459 index = (intr_alloc_entry & 1) * 8;
5460 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5461 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5462 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5463 } else {
5464 /* tx or rx causes */
5465 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5466 ivar = IXGBE_READ_REG(hw,
5467 IXGBE_IVAR(intr_alloc_entry >> 1));
5468 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5469 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5470 ivar);
5471 }
5472 break;
5473
5474 default:
5475 break;
5476 }
5477 }
5478
5479 /*
5480 * Convert the rx ring index driver maintained to the rx ring index
5481 * in h/w.
5482 */
5483 static uint32_t
5484 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5485 {
5486
5487 struct ixgbe_hw *hw = &ixgbe->hw;
5488 uint32_t rx_ring_per_group, hw_rx_index;
5489
5490 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5491 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5492 return (sw_rx_index);
5493 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5494 switch (hw->mac.type) {
5495 case ixgbe_mac_82598EB:
5496 return (sw_rx_index);
5497
5498 case ixgbe_mac_82599EB:
5499 case ixgbe_mac_X540:
5500 case ixgbe_mac_X550:
5501 case ixgbe_mac_X550EM_x:
5502 case ixgbe_mac_X550EM_a:
5503 return (sw_rx_index * 2);
5504
5505 default:
5506 break;
5507 }
5508 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5509 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5510
5511 switch (hw->mac.type) {
5512 case ixgbe_mac_82598EB:
5513 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5514 16 + (sw_rx_index % rx_ring_per_group);
5515 return (hw_rx_index);
5516
5517 case ixgbe_mac_82599EB:
5518 case ixgbe_mac_X540:
5519 case ixgbe_mac_X550:
5520 case ixgbe_mac_X550EM_x:
5521 case ixgbe_mac_X550EM_a:
5522 if (ixgbe->num_rx_groups > 32) {
5523 hw_rx_index = (sw_rx_index /
5524 rx_ring_per_group) * 2 +
5525 (sw_rx_index % rx_ring_per_group);
5526 } else {
5527 hw_rx_index = (sw_rx_index /
5528 rx_ring_per_group) * 4 +
5529 (sw_rx_index % rx_ring_per_group);
5530 }
5531 return (hw_rx_index);
5532
5533 default:
5534 break;
5535 }
5536 }
5537
5538 /*
5539 * Should never reach. Just to make compiler happy.
5540 */
5541 return (sw_rx_index);
5542 }
5543
5544 /*
5545 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5546 *
5547 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5548 * to vector[0 - (intr_cnt -1)].
5549 */
5550 static int
5551 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5552 {
5553 int i, vector = 0;
5554
5555 /* initialize vector map */
5556 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5557 for (i = 0; i < ixgbe->intr_cnt; i++) {
5558 ixgbe->vect_map[i].ixgbe = ixgbe;
5559 }
5560
5561 /*
5562 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5563 * tx rings[0] on RTxQ[1].
5564 */
5565 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5566 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5567 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5568 return (IXGBE_SUCCESS);
5569 }
5570
5571 /*
5572 * Interrupts/vectors mapping for MSI-X
5573 */
5574
5575 /*
5576 * Map other interrupt to vector 0,
5577 * Set bit in map and count the bits set.
5578 */
5579 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5580 ixgbe->vect_map[vector].other_cnt++;
5581
5582 /*
5583 * Map rx ring interrupts to vectors
5584 */
5585 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5586 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5587 vector = (vector +1) % ixgbe->intr_cnt;
5588 }
5589
5590 /*
5591 * Map tx ring interrupts to vectors
5592 */
5593 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5594 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5595 vector = (vector +1) % ixgbe->intr_cnt;
5596 }
5597
5598 return (IXGBE_SUCCESS);
5599 }
5600
5601 /*
5602 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5603 *
5604 * This relies on ring/vector mapping already set up in the
5605 * vect_map[] structures
5606 */
5607 static void
5608 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5609 {
5610 struct ixgbe_hw *hw = &ixgbe->hw;
5611 ixgbe_intr_vector_t *vect; /* vector bitmap */
5612 int r_idx; /* ring index */
5613 int v_idx; /* vector index */
5614 uint32_t hw_index;
5615
5616 /*
5617 * Clear any previous entries
5618 */
5619 switch (hw->mac.type) {
5620 case ixgbe_mac_82598EB:
5621 for (v_idx = 0; v_idx < 25; v_idx++)
5622 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5623 break;
5624
5625 case ixgbe_mac_82599EB:
5626 case ixgbe_mac_X540:
5627 case ixgbe_mac_X550:
5628 case ixgbe_mac_X550EM_x:
5629 case ixgbe_mac_X550EM_a:
5630 for (v_idx = 0; v_idx < 64; v_idx++)
5631 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5632 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5633 break;
5634
5635 default:
5636 break;
5637 }
5638
5639 /*
5640 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5641 * tx rings[0] will use RTxQ[1].
5642 */
5643 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5644 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5645 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5646 return;
5647 }
5648
5649 /*
5650 * For MSI-X interrupt, "Other" is always on vector[0].
5651 */
5652 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5653
5654 /*
5655 * For each interrupt vector, populate the IVAR table
5656 */
5657 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5658 vect = &ixgbe->vect_map[v_idx];
5659
5660 /*
5661 * For each rx ring bit set
5662 */
5663 r_idx = bt_getlowbit(vect->rx_map, 0,
5664 (ixgbe->num_rx_rings - 1));
5665
5666 while (r_idx >= 0) {
5667 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5668 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5669 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5670 (ixgbe->num_rx_rings - 1));
5671 }
5672
5673 /*
5674 * For each tx ring bit set
5675 */
5676 r_idx = bt_getlowbit(vect->tx_map, 0,
5677 (ixgbe->num_tx_rings - 1));
5678
5679 while (r_idx >= 0) {
5680 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5681 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5682 (ixgbe->num_tx_rings - 1));
5683 }
5684 }
5685 }
5686
5687 /*
5688 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5689 */
5690 static void
5691 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5692 {
5693 int i;
5694 int rc;
5695
5696 for (i = 0; i < ixgbe->intr_cnt; i++) {
5697 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5698 if (rc != DDI_SUCCESS) {
5699 IXGBE_DEBUGLOG_1(ixgbe,
5700 "Remove intr handler failed: %d", rc);
5701 }
5702 }
5703 }
5704
5705 /*
5706 * ixgbe_rem_intrs - Remove the allocated interrupts.
5707 */
5708 static void
5709 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5710 {
5711 int i;
5712 int rc;
5713
5714 for (i = 0; i < ixgbe->intr_cnt; i++) {
5715 rc = ddi_intr_free(ixgbe->htable[i]);
5716 if (rc != DDI_SUCCESS) {
5717 IXGBE_DEBUGLOG_1(ixgbe,
5718 "Free intr failed: %d", rc);
5719 }
5720 }
5721
5722 kmem_free(ixgbe->htable, ixgbe->intr_size);
5723 ixgbe->htable = NULL;
5724 }
5725
5726 /*
5727 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5728 */
5729 static int
5730 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5731 {
5732 int i;
5733 int rc;
5734
5735 /*
5736 * Enable interrupts
5737 */
5738 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5739 /*
5740 * Call ddi_intr_block_enable() for MSI
5741 */
5742 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5743 if (rc != DDI_SUCCESS) {
5744 ixgbe_log(ixgbe,
5745 "Enable block intr failed: %d", rc);
5746 return (IXGBE_FAILURE);
5747 }
5748 } else {
5749 /*
5750 * Call ddi_intr_enable() for Legacy/MSI non block enable
5751 */
5752 for (i = 0; i < ixgbe->intr_cnt; i++) {
5753 rc = ddi_intr_enable(ixgbe->htable[i]);
5754 if (rc != DDI_SUCCESS) {
5755 ixgbe_log(ixgbe,
5756 "Enable intr failed: %d", rc);
5757 return (IXGBE_FAILURE);
5758 }
5759 }
5760 }
5761
5762 return (IXGBE_SUCCESS);
5763 }
5764
5765 /*
5766 * ixgbe_disable_intrs - Disable all the interrupts.
5767 */
5768 static int
5769 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5770 {
5771 int i;
5772 int rc;
5773
5774 /*
5775 * Disable all interrupts
5776 */
5777 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5778 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5779 if (rc != DDI_SUCCESS) {
5780 ixgbe_log(ixgbe,
5781 "Disable block intr failed: %d", rc);
5782 return (IXGBE_FAILURE);
5783 }
5784 } else {
5785 for (i = 0; i < ixgbe->intr_cnt; i++) {
5786 rc = ddi_intr_disable(ixgbe->htable[i]);
5787 if (rc != DDI_SUCCESS) {
5788 ixgbe_log(ixgbe,
5789 "Disable intr failed: %d", rc);
5790 return (IXGBE_FAILURE);
5791 }
5792 }
5793 }
5794
5795 return (IXGBE_SUCCESS);
5796 }
5797
5798 /*
5799 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5800 */
5801 static void
5802 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5803 {
5804 struct ixgbe_hw *hw = &ixgbe->hw;
5805 ixgbe_link_speed speed = 0;
5806 boolean_t link_up = B_FALSE;
5807 uint32_t pcs1g_anlp = 0;
5808
5809 ASSERT(mutex_owned(&ixgbe->gen_lock));
5810 ixgbe->param_lp_1000fdx_cap = 0;
5811 ixgbe->param_lp_100fdx_cap = 0;
5812
5813 /* check for link, don't wait */
5814 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
5815
5816 /*
5817 * Update the observed Link Partner's capabilities. Not all adapters
5818 * can provide full information on the LP's capable speeds, so we
5819 * provide what we can.
5820 */
5821 if (link_up) {
5822 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5823
5824 ixgbe->param_lp_1000fdx_cap =
5825 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5826 ixgbe->param_lp_100fdx_cap =
5827 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5828 }
5829
5830 /*
5831 * Update GLD's notion of the adapter's currently advertised speeds.
5832 * Since the common code doesn't always record the current autonegotiate
5833 * settings in the phy struct for all parts (specifically, adapters with
5834 * SFPs) we first test to see if it is 0, and if so, we fall back to
5835 * using the adapter's speed capabilities which we saved during instance
5836 * init in ixgbe_init_params().
5837 *
5838 * Adapters with SFPs will always be shown as advertising all of their
5839 * supported speeds, and adapters with baseT PHYs (where the phy struct
5840 * is maintained by the common code) will always have a factual view of
5841 * their currently-advertised speeds. In the case of SFPs, this is
5842 * acceptable as we default to advertising all speeds that the adapter
5843 * claims to support, and those properties are immutable; unlike on
5844 * baseT (copper) PHYs, where speeds can be enabled or disabled at will.
5845 */
5846 speed = hw->phy.autoneg_advertised;
5847 if (speed == 0)
5848 speed = ixgbe->speeds_supported;
5849
5850 ixgbe->param_adv_10000fdx_cap =
5851 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0;
5852 ixgbe->param_adv_5000fdx_cap =
5853 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0;
5854 ixgbe->param_adv_2500fdx_cap =
5855 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0;
5856 ixgbe->param_adv_1000fdx_cap =
5857 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0;
5858 ixgbe->param_adv_100fdx_cap =
5859 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0;
5860 }
5861
5862 /*
5863 * ixgbe_get_driver_control - Notify that driver is in control of device.
5864 */
5865 static void
5866 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5867 {
5868 uint32_t ctrl_ext;
5869
5870 /*
5871 * Notify firmware that driver is in control of device
5872 */
5873 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5874 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5875 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5876 }
5877
5878 /*
5879 * ixgbe_release_driver_control - Notify that driver is no longer in control
5880 * of device.
5881 */
5882 static void
5883 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5884 {
5885 uint32_t ctrl_ext;
5886
5887 /*
5888 * Notify firmware that driver is no longer in control of device
5889 */
5890 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5891 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5892 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5893 }
5894
5895 /*
5896 * ixgbe_atomic_reserve - Atomic decrease operation.
5897 */
5898 int
5899 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5900 {
5901 uint32_t oldval;
5902 uint32_t newval;
5903
5904 /*
5905 * ATOMICALLY
5906 */
5907 do {
5908 oldval = *count_p;
5909 if (oldval < n)
5910 return (-1);
5911 newval = oldval - n;
5912 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5913
5914 return (newval);
5915 }
5916
5917 /*
5918 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5919 */
5920 static uint8_t *
5921 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5922 {
5923 uint8_t *addr = *upd_ptr;
5924 uint8_t *new_ptr;
5925
5926 _NOTE(ARGUNUSED(hw));
5927 _NOTE(ARGUNUSED(vmdq));
5928
5929 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5930 *upd_ptr = new_ptr;
5931 return (addr);
5932 }
5933
5934 /*
5935 * FMA support
5936 */
5937 int
5938 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5939 {
5940 ddi_fm_error_t de;
5941
5942 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5943 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5944 return (de.fme_status);
5945 }
5946
5947 int
5948 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5949 {
5950 ddi_fm_error_t de;
5951
5952 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5953 return (de.fme_status);
5954 }
5955
5956 /*
5957 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5958 */
5959 static int
5960 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5961 {
5962 _NOTE(ARGUNUSED(impl_data));
5963 /*
5964 * as the driver can always deal with an error in any dma or
5965 * access handle, we can just return the fme_status value.
5966 */
5967 pci_ereport_post(dip, err, NULL);
5968 return (err->fme_status);
5969 }
5970
5971 static void
5972 ixgbe_fm_init(ixgbe_t *ixgbe)
5973 {
5974 ddi_iblock_cookie_t iblk;
5975 int fma_dma_flag;
5976
5977 /*
5978 * Only register with IO Fault Services if we have some capability
5979 */
5980 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5981 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5982 } else {
5983 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5984 }
5985
5986 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5987 fma_dma_flag = 1;
5988 } else {
5989 fma_dma_flag = 0;
5990 }
5991
5992 ixgbe_set_fma_flags(fma_dma_flag);
5993
5994 if (ixgbe->fm_capabilities) {
5995
5996 /*
5997 * Register capabilities with IO Fault Services
5998 */
5999 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
6000
6001 /*
6002 * Initialize pci ereport capabilities if ereport capable
6003 */
6004 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
6005 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6006 pci_ereport_setup(ixgbe->dip);
6007
6008 /*
6009 * Register error callback if error callback capable
6010 */
6011 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6012 ddi_fm_handler_register(ixgbe->dip,
6013 ixgbe_fm_error_cb, (void*) ixgbe);
6014 }
6015 }
6016
6017 static void
6018 ixgbe_fm_fini(ixgbe_t *ixgbe)
6019 {
6020 /*
6021 * Only unregister FMA capabilities if they are registered
6022 */
6023 if (ixgbe->fm_capabilities) {
6024
6025 /*
6026 * Release any resources allocated by pci_ereport_setup()
6027 */
6028 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
6029 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6030 pci_ereport_teardown(ixgbe->dip);
6031
6032 /*
6033 * Un-register error callback if error callback capable
6034 */
6035 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6036 ddi_fm_handler_unregister(ixgbe->dip);
6037
6038 /*
6039 * Unregister from IO Fault Service
6040 */
6041 ddi_fm_fini(ixgbe->dip);
6042 }
6043 }
6044
6045 void
6046 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
6047 {
6048 uint64_t ena;
6049 char buf[FM_MAX_CLASS];
6050
6051 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
6052 ena = fm_ena_generate(0, FM_ENA_FMT1);
6053 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
6054 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
6055 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
6056 }
6057 }
6058
6059 static int
6060 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
6061 {
6062 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
6063
6064 mutex_enter(&rx_ring->rx_lock);
6065 rx_ring->ring_gen_num = mr_gen_num;
6066 mutex_exit(&rx_ring->rx_lock);
6067 return (0);
6068 }
6069
6070 /*
6071 * Get the global ring index by a ring index within a group.
6072 */
6073 static int
6074 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
6075 {
6076 ixgbe_rx_ring_t *rx_ring;
6077 int i;
6078
6079 for (i = 0; i < ixgbe->num_rx_rings; i++) {
6080 rx_ring = &ixgbe->rx_rings[i];
6081 if (rx_ring->group_index == gindex)
6082 rindex--;
6083 if (rindex < 0)
6084 return (i);
6085 }
6086
6087 return (-1);
6088 }
6089
6090 /*
6091 * Callback funtion for MAC layer to register all rings.
6092 */
6093 /* ARGSUSED */
6094 void
6095 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
6096 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
6097 {
6098 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6099 mac_intr_t *mintr = &infop->mri_intr;
6100
6101 switch (rtype) {
6102 case MAC_RING_TYPE_RX: {
6103 /*
6104 * 'index' is the ring index within the group.
6105 * Need to get the global ring index by searching in groups.
6106 */
6107 int global_ring_index = ixgbe_get_rx_ring_index(
6108 ixgbe, group_index, ring_index);
6109
6110 ASSERT(global_ring_index >= 0);
6111
6112 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
6113 rx_ring->ring_handle = rh;
6114
6115 infop->mri_driver = (mac_ring_driver_t)rx_ring;
6116 infop->mri_start = ixgbe_ring_start;
6117 infop->mri_stop = NULL;
6118 infop->mri_poll = ixgbe_ring_rx_poll;
6119 infop->mri_stat = ixgbe_rx_ring_stat;
6120
6121 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
6122 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
6123 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
6124 if (ixgbe->intr_type &
6125 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6126 mintr->mi_ddi_handle =
6127 ixgbe->htable[rx_ring->intr_vector];
6128 }
6129
6130 break;
6131 }
6132 case MAC_RING_TYPE_TX: {
6133 ASSERT(group_index == -1);
6134 ASSERT(ring_index < ixgbe->num_tx_rings);
6135
6136 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
6137 tx_ring->ring_handle = rh;
6138
6139 infop->mri_driver = (mac_ring_driver_t)tx_ring;
6140 infop->mri_start = NULL;
6141 infop->mri_stop = NULL;
6142 infop->mri_tx = ixgbe_ring_tx;
6143 infop->mri_stat = ixgbe_tx_ring_stat;
6144 if (ixgbe->intr_type &
6145 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6146 mintr->mi_ddi_handle =
6147 ixgbe->htable[tx_ring->intr_vector];
6148 }
6149 break;
6150 }
6151 default:
6152 break;
6153 }
6154 }
6155
6156 /*
6157 * Callback funtion for MAC layer to register all groups.
6158 */
6159 void
6160 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
6161 mac_group_info_t *infop, mac_group_handle_t gh)
6162 {
6163 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6164
6165 switch (rtype) {
6166 case MAC_RING_TYPE_RX: {
6167 ixgbe_rx_group_t *rx_group;
6168
6169 rx_group = &ixgbe->rx_groups[index];
6170 rx_group->group_handle = gh;
6171
6172 infop->mgi_driver = (mac_group_driver_t)rx_group;
6173 infop->mgi_start = NULL;
6174 infop->mgi_stop = NULL;
6175 infop->mgi_addmac = ixgbe_addmac;
6176 infop->mgi_remmac = ixgbe_remmac;
6177 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
6178
6179 break;
6180 }
6181 case MAC_RING_TYPE_TX:
6182 break;
6183 default:
6184 break;
6185 }
6186 }
6187
6188 /*
6189 * Enable interrupt on the specificed rx ring.
6190 */
6191 int
6192 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
6193 {
6194 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6195 ixgbe_t *ixgbe = rx_ring->ixgbe;
6196 int r_idx = rx_ring->index;
6197 int hw_r_idx = rx_ring->hw_index;
6198 int v_idx = rx_ring->intr_vector;
6199
6200 mutex_enter(&ixgbe->gen_lock);
6201 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6202 mutex_exit(&ixgbe->gen_lock);
6203 /*
6204 * Simply return 0.
6205 * Interrupts are being adjusted. ixgbe_intr_adjust()
6206 * will eventually re-enable the interrupt when it's
6207 * done with the adjustment.
6208 */
6209 return (0);
6210 }
6211
6212 /*
6213 * To enable interrupt by setting the VAL bit of given interrupt
6214 * vector allocation register (IVAR).
6215 */
6216 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
6217
6218 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
6219
6220 /*
6221 * Trigger a Rx interrupt on this ring
6222 */
6223 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
6224 IXGBE_WRITE_FLUSH(&ixgbe->hw);
6225
6226 mutex_exit(&ixgbe->gen_lock);
6227
6228 return (0);
6229 }
6230
6231 /*
6232 * Disable interrupt on the specificed rx ring.
6233 */
6234 int
6235 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
6236 {
6237 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6238 ixgbe_t *ixgbe = rx_ring->ixgbe;
6239 int r_idx = rx_ring->index;
6240 int hw_r_idx = rx_ring->hw_index;
6241 int v_idx = rx_ring->intr_vector;
6242
6243 mutex_enter(&ixgbe->gen_lock);
6244 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6245 mutex_exit(&ixgbe->gen_lock);
6246 /*
6247 * Simply return 0.
6248 * In the rare case where an interrupt is being
6249 * disabled while interrupts are being adjusted,
6250 * we don't fail the operation. No interrupts will
6251 * be generated while they are adjusted, and
6252 * ixgbe_intr_adjust() will cause the interrupts
6253 * to be re-enabled once it completes. Note that
6254 * in this case, packets may be delivered to the
6255 * stack via interrupts before xgbe_rx_ring_intr_enable()
6256 * is called again. This is acceptable since interrupt
6257 * adjustment is infrequent, and the stack will be
6258 * able to handle these packets.
6259 */
6260 return (0);
6261 }
6262
6263 /*
6264 * To disable interrupt by clearing the VAL bit of given interrupt
6265 * vector allocation register (IVAR).
6266 */
6267 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
6268
6269 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
6270
6271 mutex_exit(&ixgbe->gen_lock);
6272
6273 return (0);
6274 }
6275
6276 /*
6277 * Add a mac address.
6278 */
6279 static int
6280 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
6281 {
6282 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6283 ixgbe_t *ixgbe = rx_group->ixgbe;
6284 struct ixgbe_hw *hw = &ixgbe->hw;
6285 int slot, i;
6286
6287 mutex_enter(&ixgbe->gen_lock);
6288
6289 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6290 mutex_exit(&ixgbe->gen_lock);
6291 return (ECANCELED);
6292 }
6293
6294 if (ixgbe->unicst_avail == 0) {
6295 /* no slots available */
6296 mutex_exit(&ixgbe->gen_lock);
6297 return (ENOSPC);
6298 }
6299
6300 /*
6301 * The first ixgbe->num_rx_groups slots are reserved for each respective
6302 * group. The rest slots are shared by all groups. While adding a
6303 * MAC address, reserved slots are firstly checked then the shared
6304 * slots are searched.
6305 */
6306 slot = -1;
6307 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
6308 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
6309 if (ixgbe->unicst_addr[i].mac.set == 0) {
6310 slot = i;
6311 break;
6312 }
6313 }
6314 } else {
6315 slot = rx_group->index;
6316 }
6317
6318 if (slot == -1) {
6319 /* no slots available */
6320 mutex_exit(&ixgbe->gen_lock);
6321 return (ENOSPC);
6322 }
6323
6324 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6325 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
6326 rx_group->index, IXGBE_RAH_AV);
6327 ixgbe->unicst_addr[slot].mac.set = 1;
6328 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
6329 ixgbe->unicst_avail--;
6330
6331 mutex_exit(&ixgbe->gen_lock);
6332
6333 return (0);
6334 }
6335
6336 /*
6337 * Remove a mac address.
6338 */
6339 static int
6340 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
6341 {
6342 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6343 ixgbe_t *ixgbe = rx_group->ixgbe;
6344 struct ixgbe_hw *hw = &ixgbe->hw;
6345 int slot;
6346
6347 mutex_enter(&ixgbe->gen_lock);
6348
6349 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6350 mutex_exit(&ixgbe->gen_lock);
6351 return (ECANCELED);
6352 }
6353
6354 slot = ixgbe_unicst_find(ixgbe, mac_addr);
6355 if (slot == -1) {
6356 mutex_exit(&ixgbe->gen_lock);
6357 return (EINVAL);
6358 }
6359
6360 if (ixgbe->unicst_addr[slot].mac.set == 0) {
6361 mutex_exit(&ixgbe->gen_lock);
6362 return (EINVAL);
6363 }
6364
6365 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6366 (void) ixgbe_clear_rar(hw, slot);
6367 ixgbe->unicst_addr[slot].mac.set = 0;
6368 ixgbe->unicst_avail++;
6369
6370 mutex_exit(&ixgbe->gen_lock);
6371
6372 return (0);
6373 }