1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright 2019 Joyent, Inc.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
33 */
34
35 #include "ixgbe_sw.h"
36
37 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
38
39 /*
40 * Local function protoypes
41 */
42 static int ixgbe_register_mac(ixgbe_t *);
43 static int ixgbe_identify_hardware(ixgbe_t *);
44 static int ixgbe_regs_map(ixgbe_t *);
45 static void ixgbe_init_properties(ixgbe_t *);
46 static int ixgbe_init_driver_settings(ixgbe_t *);
47 static void ixgbe_init_locks(ixgbe_t *);
48 static void ixgbe_destroy_locks(ixgbe_t *);
49 static int ixgbe_init(ixgbe_t *);
50 static int ixgbe_chip_start(ixgbe_t *);
51 static void ixgbe_chip_stop(ixgbe_t *);
52 static int ixgbe_reset(ixgbe_t *);
53 static void ixgbe_tx_clean(ixgbe_t *);
54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
56 static int ixgbe_alloc_rings(ixgbe_t *);
57 static void ixgbe_free_rings(ixgbe_t *);
58 static int ixgbe_alloc_rx_data(ixgbe_t *);
59 static void ixgbe_free_rx_data(ixgbe_t *);
60 static int ixgbe_setup_rings(ixgbe_t *);
61 static int ixgbe_setup_rx(ixgbe_t *);
62 static void ixgbe_setup_tx(ixgbe_t *);
63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
65 static void ixgbe_setup_rss(ixgbe_t *);
66 static void ixgbe_setup_vmdq(ixgbe_t *);
67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
68 static void ixgbe_setup_rss_table(ixgbe_t *);
69 static void ixgbe_init_unicst(ixgbe_t *);
70 static int ixgbe_init_vlan(ixgbe_t *);
71 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
72 static void ixgbe_setup_multicst(ixgbe_t *);
73 static void ixgbe_get_hw_state(ixgbe_t *);
74 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
75 static void ixgbe_get_conf(ixgbe_t *);
76 static void ixgbe_init_params(ixgbe_t *);
77 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
78 static void ixgbe_driver_link_check(ixgbe_t *);
79 static void ixgbe_sfp_check(void *);
80 static void ixgbe_overtemp_check(void *);
81 static void ixgbe_phy_check(void *);
82 static void ixgbe_link_timer(void *);
83 static void ixgbe_local_timer(void *);
84 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
85 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
86 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
87 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
88 static boolean_t is_valid_mac_addr(uint8_t *);
89 static boolean_t ixgbe_stall_check(ixgbe_t *);
90 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
91 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
92 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
93 static int ixgbe_alloc_intrs(ixgbe_t *);
94 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
95 static int ixgbe_add_intr_handlers(ixgbe_t *);
96 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
97 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
98 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
99 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
100 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
101 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
102 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
103 static void ixgbe_setup_adapter_vector(ixgbe_t *);
104 static void ixgbe_rem_intr_handlers(ixgbe_t *);
105 static void ixgbe_rem_intrs(ixgbe_t *);
106 static int ixgbe_enable_intrs(ixgbe_t *);
107 static int ixgbe_disable_intrs(ixgbe_t *);
108 static uint_t ixgbe_intr_legacy(void *, void *);
109 static uint_t ixgbe_intr_msi(void *, void *);
110 static uint_t ixgbe_intr_msix(void *, void *);
111 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
112 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
113 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
114 static void ixgbe_get_driver_control(struct ixgbe_hw *);
115 static int ixgbe_addmac(void *, const uint8_t *);
116 static int ixgbe_remmac(void *, const uint8_t *);
117 static int ixgbe_addvlan(mac_group_driver_t, uint16_t);
118 static int ixgbe_remvlan(mac_group_driver_t, uint16_t);
119 static void ixgbe_release_driver_control(struct ixgbe_hw *);
120
121 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
122 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
123 static int ixgbe_resume(dev_info_t *);
124 static int ixgbe_suspend(dev_info_t *);
125 static int ixgbe_quiesce(dev_info_t *);
126 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
127 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
128 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
129 static int ixgbe_intr_cb_register(ixgbe_t *);
130 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
131
132 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
133 const void *impl_data);
134 static void ixgbe_fm_init(ixgbe_t *);
135 static void ixgbe_fm_fini(ixgbe_t *);
136
137 char *ixgbe_priv_props[] = {
138 "_tx_copy_thresh",
139 "_tx_recycle_thresh",
140 "_tx_overload_thresh",
141 "_tx_resched_thresh",
142 "_rx_copy_thresh",
143 "_rx_limit_per_intr",
144 "_intr_throttling",
145 "_adv_pause_cap",
146 "_adv_asym_pause_cap",
147 NULL
148 };
149
150 #define IXGBE_MAX_PRIV_PROPS \
151 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
152
153 static struct cb_ops ixgbe_cb_ops = {
154 nulldev, /* cb_open */
155 nulldev, /* cb_close */
156 nodev, /* cb_strategy */
157 nodev, /* cb_print */
158 nodev, /* cb_dump */
159 nodev, /* cb_read */
160 nodev, /* cb_write */
161 nodev, /* cb_ioctl */
162 nodev, /* cb_devmap */
163 nodev, /* cb_mmap */
164 nodev, /* cb_segmap */
165 nochpoll, /* cb_chpoll */
166 ddi_prop_op, /* cb_prop_op */
167 NULL, /* cb_stream */
168 D_MP | D_HOTPLUG, /* cb_flag */
169 CB_REV, /* cb_rev */
170 nodev, /* cb_aread */
171 nodev /* cb_awrite */
172 };
173
174 static struct dev_ops ixgbe_dev_ops = {
175 DEVO_REV, /* devo_rev */
176 0, /* devo_refcnt */
177 NULL, /* devo_getinfo */
178 nulldev, /* devo_identify */
179 nulldev, /* devo_probe */
180 ixgbe_attach, /* devo_attach */
181 ixgbe_detach, /* devo_detach */
182 nodev, /* devo_reset */
183 &ixgbe_cb_ops, /* devo_cb_ops */
184 NULL, /* devo_bus_ops */
185 ddi_power, /* devo_power */
186 ixgbe_quiesce, /* devo_quiesce */
187 };
188
189 static struct modldrv ixgbe_modldrv = {
190 &mod_driverops, /* Type of module. This one is a driver */
191 ixgbe_ident, /* Discription string */
192 &ixgbe_dev_ops /* driver ops */
193 };
194
195 static struct modlinkage ixgbe_modlinkage = {
196 MODREV_1, &ixgbe_modldrv, NULL
197 };
198
199 /*
200 * Access attributes for register mapping
201 */
202 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
203 DDI_DEVICE_ATTR_V1,
204 DDI_STRUCTURE_LE_ACC,
205 DDI_STRICTORDER_ACC,
206 DDI_FLAGERR_ACC
207 };
208
209 /*
210 * Loopback property
211 */
212 static lb_property_t lb_normal = {
213 normal, "normal", IXGBE_LB_NONE
214 };
215
216 static lb_property_t lb_mac = {
217 internal, "MAC", IXGBE_LB_INTERNAL_MAC
218 };
219
220 static lb_property_t lb_external = {
221 external, "External", IXGBE_LB_EXTERNAL
222 };
223
224 #define IXGBE_M_CALLBACK_FLAGS \
225 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
226
227 static mac_callbacks_t ixgbe_m_callbacks = {
228 IXGBE_M_CALLBACK_FLAGS,
229 ixgbe_m_stat,
230 ixgbe_m_start,
231 ixgbe_m_stop,
232 ixgbe_m_promisc,
233 ixgbe_m_multicst,
234 NULL,
235 NULL,
236 NULL,
237 ixgbe_m_ioctl,
238 ixgbe_m_getcapab,
239 NULL,
240 NULL,
241 ixgbe_m_setprop,
242 ixgbe_m_getprop,
243 ixgbe_m_propinfo
244 };
245
246 /*
247 * Initialize capabilities of each supported adapter type
248 */
249 static adapter_info_t ixgbe_82598eb_cap = {
250 64, /* maximum number of rx queues */
251 1, /* minimum number of rx queues */
252 64, /* default number of rx queues */
253 16, /* maximum number of rx groups */
254 1, /* minimum number of rx groups */
255 1, /* default number of rx groups */
256 32, /* maximum number of tx queues */
257 1, /* minimum number of tx queues */
258 8, /* default number of tx queues */
259 16366, /* maximum MTU size */
260 0xFFFF, /* maximum interrupt throttle rate */
261 0, /* minimum interrupt throttle rate */
262 200, /* default interrupt throttle rate */
263 18, /* maximum total msix vectors */
264 16, /* maximum number of ring vectors */
265 2, /* maximum number of other vectors */
266 IXGBE_EICR_LSC, /* "other" interrupt types handled */
267 0, /* "other" interrupt types enable mask */
268 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
269 | IXGBE_FLAG_RSS_CAPABLE
270 | IXGBE_FLAG_VMDQ_CAPABLE)
271 };
272
273 static adapter_info_t ixgbe_82599eb_cap = {
274 128, /* maximum number of rx queues */
275 1, /* minimum number of rx queues */
276 128, /* default number of rx queues */
277 64, /* maximum number of rx groups */
278 1, /* minimum number of rx groups */
279 1, /* default number of rx groups */
280 128, /* maximum number of tx queues */
281 1, /* minimum number of tx queues */
282 8, /* default number of tx queues */
283 15500, /* maximum MTU size */
284 0xFF8, /* maximum interrupt throttle rate */
285 0, /* minimum interrupt throttle rate */
286 200, /* default interrupt throttle rate */
287 64, /* maximum total msix vectors */
288 16, /* maximum number of ring vectors */
289 2, /* maximum number of other vectors */
290 (IXGBE_EICR_LSC
291 | IXGBE_EICR_GPI_SDP1
292 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
293
294 (IXGBE_SDP1_GPIEN
295 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
296
297 (IXGBE_FLAG_DCA_CAPABLE
298 | IXGBE_FLAG_RSS_CAPABLE
299 | IXGBE_FLAG_VMDQ_CAPABLE
300 | IXGBE_FLAG_RSC_CAPABLE
301 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
302 };
303
304 static adapter_info_t ixgbe_X540_cap = {
305 128, /* maximum number of rx queues */
306 1, /* minimum number of rx queues */
307 128, /* default number of rx queues */
308 64, /* maximum number of rx groups */
309 1, /* minimum number of rx groups */
310 1, /* default number of rx groups */
311 128, /* maximum number of tx queues */
312 1, /* minimum number of tx queues */
313 8, /* default number of tx queues */
314 15500, /* maximum MTU size */
315 0xFF8, /* maximum interrupt throttle rate */
316 0, /* minimum interrupt throttle rate */
317 200, /* default interrupt throttle rate */
318 64, /* maximum total msix vectors */
319 16, /* maximum number of ring vectors */
320 2, /* maximum number of other vectors */
321 (IXGBE_EICR_LSC
322 | IXGBE_EICR_GPI_SDP1_X540
323 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
324
325 (IXGBE_SDP1_GPIEN_X540
326 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
327
328 (IXGBE_FLAG_DCA_CAPABLE
329 | IXGBE_FLAG_RSS_CAPABLE
330 | IXGBE_FLAG_VMDQ_CAPABLE
331 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
332 };
333
334 static adapter_info_t ixgbe_X550_cap = {
335 128, /* maximum number of rx queues */
336 1, /* minimum number of rx queues */
337 128, /* default number of rx queues */
338 64, /* maximum number of rx groups */
339 1, /* minimum number of rx groups */
340 1, /* default number of rx groups */
341 128, /* maximum number of tx queues */
342 1, /* minimum number of tx queues */
343 8, /* default number of tx queues */
344 15500, /* maximum MTU size */
345 0xFF8, /* maximum interrupt throttle rate */
346 0, /* minimum interrupt throttle rate */
347 0x200, /* default interrupt throttle rate */
348 64, /* maximum total msix vectors */
349 16, /* maximum number of ring vectors */
350 2, /* maximum number of other vectors */
351 IXGBE_EICR_LSC, /* "other" interrupt types handled */
352 0, /* "other" interrupt types enable mask */
353 (IXGBE_FLAG_RSS_CAPABLE
354 | IXGBE_FLAG_VMDQ_CAPABLE
355 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
356 };
357
358 /*
359 * Module Initialization Functions.
360 */
361
362 int
363 _init(void)
364 {
365 int status;
366
367 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
368
369 status = mod_install(&ixgbe_modlinkage);
370
371 if (status != DDI_SUCCESS) {
372 mac_fini_ops(&ixgbe_dev_ops);
373 }
374
375 return (status);
376 }
377
378 int
379 _fini(void)
380 {
381 int status;
382
383 status = mod_remove(&ixgbe_modlinkage);
384
385 if (status == DDI_SUCCESS) {
386 mac_fini_ops(&ixgbe_dev_ops);
387 }
388
389 return (status);
390 }
391
392 int
393 _info(struct modinfo *modinfop)
394 {
395 int status;
396
397 status = mod_info(&ixgbe_modlinkage, modinfop);
398
399 return (status);
400 }
401
402 /*
403 * ixgbe_attach - Driver attach.
404 *
405 * This function is the device specific initialization entry
406 * point. This entry point is required and must be written.
407 * The DDI_ATTACH command must be provided in the attach entry
408 * point. When attach() is called with cmd set to DDI_ATTACH,
409 * all normal kernel services (such as kmem_alloc(9F)) are
410 * available for use by the driver.
411 *
412 * The attach() function will be called once for each instance
413 * of the device on the system with cmd set to DDI_ATTACH.
414 * Until attach() succeeds, the only driver entry points which
415 * may be called are open(9E) and getinfo(9E).
416 */
417 static int
418 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
419 {
420 ixgbe_t *ixgbe;
421 struct ixgbe_osdep *osdep;
422 struct ixgbe_hw *hw;
423 int instance;
424 char taskqname[32];
425
426 /*
427 * Check the command and perform corresponding operations
428 */
429 switch (cmd) {
430 default:
431 return (DDI_FAILURE);
432
433 case DDI_RESUME:
434 return (ixgbe_resume(devinfo));
435
436 case DDI_ATTACH:
437 break;
438 }
439
440 /* Get the device instance */
441 instance = ddi_get_instance(devinfo);
442
443 /* Allocate memory for the instance data structure */
444 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
445
446 ixgbe->dip = devinfo;
447 ixgbe->instance = instance;
448
449 hw = &ixgbe->hw;
450 osdep = &ixgbe->osdep;
451 hw->back = osdep;
452 osdep->ixgbe = ixgbe;
453
454 /* Attach the instance pointer to the dev_info data structure */
455 ddi_set_driver_private(devinfo, ixgbe);
456
457 /*
458 * Initialize for FMA support
459 */
460 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
461 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
462 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
463 ixgbe_fm_init(ixgbe);
464 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
465
466 /*
467 * Map PCI config space registers
468 */
469 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
470 ixgbe_error(ixgbe, "Failed to map PCI configurations");
471 goto attach_fail;
472 }
473 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
474
475 /*
476 * Identify the chipset family
477 */
478 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
479 ixgbe_error(ixgbe, "Failed to identify hardware");
480 goto attach_fail;
481 }
482
483 /*
484 * Map device registers
485 */
486 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
487 ixgbe_error(ixgbe, "Failed to map device registers");
488 goto attach_fail;
489 }
490 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
491
492 /*
493 * Initialize driver parameters
494 */
495 ixgbe_init_properties(ixgbe);
496 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
497
498 /*
499 * Register interrupt callback
500 */
501 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
502 ixgbe_error(ixgbe, "Failed to register interrupt callback");
503 goto attach_fail;
504 }
505
506 /*
507 * Allocate interrupts
508 */
509 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
510 ixgbe_error(ixgbe, "Failed to allocate interrupts");
511 goto attach_fail;
512 }
513 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
514
515 /*
516 * Allocate rx/tx rings based on the ring numbers.
517 * The actual numbers of rx/tx rings are decided by the number of
518 * allocated interrupt vectors, so we should allocate the rings after
519 * interrupts are allocated.
520 */
521 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
522 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
523 goto attach_fail;
524 }
525 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
526
527 /*
528 * Map rings to interrupt vectors
529 */
530 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
531 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
532 goto attach_fail;
533 }
534
535 /*
536 * Add interrupt handlers
537 */
538 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
539 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
540 goto attach_fail;
541 }
542 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
543
544 /*
545 * Create a taskq for sfp-change
546 */
547 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
548 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
549 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
550 ixgbe_error(ixgbe, "sfp_taskq create failed");
551 goto attach_fail;
552 }
553 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
554
555 /*
556 * Create a taskq for over-temp
557 */
558 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
559 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
560 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
561 ixgbe_error(ixgbe, "overtemp_taskq create failed");
562 goto attach_fail;
563 }
564 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
565
566 /*
567 * Create a taskq for processing external PHY interrupts
568 */
569 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance);
570 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname,
571 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
572 ixgbe_error(ixgbe, "phy_taskq create failed");
573 goto attach_fail;
574 }
575 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ;
576
577 /*
578 * Initialize driver parameters
579 */
580 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
581 ixgbe_error(ixgbe, "Failed to initialize driver settings");
582 goto attach_fail;
583 }
584
585 /*
586 * Initialize mutexes for this device.
587 * Do this before enabling the interrupt handler and
588 * register the softint to avoid the condition where
589 * interrupt handler can try using uninitialized mutex.
590 */
591 ixgbe_init_locks(ixgbe);
592 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
593
594 /*
595 * Initialize chipset hardware
596 */
597 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
598 ixgbe_error(ixgbe, "Failed to initialize adapter");
599 goto attach_fail;
600 }
601 ixgbe->link_check_complete = B_FALSE;
602 ixgbe->link_check_hrtime = gethrtime() +
603 (IXGBE_LINK_UP_TIME * 100000000ULL);
604 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
605
606 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
607 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
608 goto attach_fail;
609 }
610
611 /*
612 * Initialize adapter capabilities
613 */
614 ixgbe_init_params(ixgbe);
615
616 /*
617 * Initialize statistics
618 */
619 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
620 ixgbe_error(ixgbe, "Failed to initialize statistics");
621 goto attach_fail;
622 }
623 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
624
625 /*
626 * Register the driver to the MAC
627 */
628 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
629 ixgbe_error(ixgbe, "Failed to register MAC");
630 goto attach_fail;
631 }
632 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
633 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
634
635 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
636 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
637 if (ixgbe->periodic_id == 0) {
638 ixgbe_error(ixgbe, "Failed to add the link check timer");
639 goto attach_fail;
640 }
641 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
642
643 /*
644 * Now that mutex locks are initialized, and the chip is also
645 * initialized, enable interrupts.
646 */
647 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
648 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
649 goto attach_fail;
650 }
651 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
652
653 ixgbe_log(ixgbe, "%s", ixgbe_ident);
654 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
655
656 return (DDI_SUCCESS);
657
658 attach_fail:
659 ixgbe_unconfigure(devinfo, ixgbe);
660 return (DDI_FAILURE);
661 }
662
663 /*
664 * ixgbe_detach - Driver detach.
665 *
666 * The detach() function is the complement of the attach routine.
667 * If cmd is set to DDI_DETACH, detach() is used to remove the
668 * state associated with a given instance of a device node
669 * prior to the removal of that instance from the system.
670 *
671 * The detach() function will be called once for each instance
672 * of the device for which there has been a successful attach()
673 * once there are no longer any opens on the device.
674 *
675 * Interrupts routine are disabled, All memory allocated by this
676 * driver are freed.
677 */
678 static int
679 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
680 {
681 ixgbe_t *ixgbe;
682
683 /*
684 * Check detach command
685 */
686 switch (cmd) {
687 default:
688 return (DDI_FAILURE);
689
690 case DDI_SUSPEND:
691 return (ixgbe_suspend(devinfo));
692
693 case DDI_DETACH:
694 break;
695 }
696
697 /*
698 * Get the pointer to the driver private data structure
699 */
700 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
701 if (ixgbe == NULL)
702 return (DDI_FAILURE);
703
704 /*
705 * If the device is still running, it needs to be stopped first.
706 * This check is necessary because under some specific circumstances,
707 * the detach routine can be called without stopping the interface
708 * first.
709 */
710 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
711 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
712 mutex_enter(&ixgbe->gen_lock);
713 ixgbe_stop(ixgbe, B_TRUE);
714 mutex_exit(&ixgbe->gen_lock);
715 /* Disable and stop the watchdog timer */
716 ixgbe_disable_watchdog_timer(ixgbe);
717 }
718
719 /*
720 * Check if there are still rx buffers held by the upper layer.
721 * If so, fail the detach.
722 */
723 if (!ixgbe_rx_drain(ixgbe))
724 return (DDI_FAILURE);
725
726 /*
727 * Do the remaining unconfigure routines
728 */
729 ixgbe_unconfigure(devinfo, ixgbe);
730
731 return (DDI_SUCCESS);
732 }
733
734 /*
735 * quiesce(9E) entry point.
736 *
737 * This function is called when the system is single-threaded at high
738 * PIL with preemption disabled. Therefore, this function must not be
739 * blocked.
740 *
741 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
742 * DDI_FAILURE indicates an error condition and should almost never happen.
743 */
744 static int
745 ixgbe_quiesce(dev_info_t *devinfo)
746 {
747 ixgbe_t *ixgbe;
748 struct ixgbe_hw *hw;
749
750 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
751
752 if (ixgbe == NULL)
753 return (DDI_FAILURE);
754
755 hw = &ixgbe->hw;
756
757 /*
758 * Disable the adapter interrupts
759 */
760 ixgbe_disable_adapter_interrupts(ixgbe);
761
762 /*
763 * Tell firmware driver is no longer in control
764 */
765 ixgbe_release_driver_control(hw);
766
767 /*
768 * Reset the chipset
769 */
770 (void) ixgbe_reset_hw(hw);
771
772 /*
773 * Reset PHY
774 */
775 (void) ixgbe_reset_phy(hw);
776
777 return (DDI_SUCCESS);
778 }
779
780 static void
781 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
782 {
783 /*
784 * Disable interrupt
785 */
786 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
787 (void) ixgbe_disable_intrs(ixgbe);
788 }
789
790 /*
791 * remove the link check timer
792 */
793 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
794 if (ixgbe->periodic_id != NULL) {
795 ddi_periodic_delete(ixgbe->periodic_id);
796 ixgbe->periodic_id = NULL;
797 }
798 }
799
800 /*
801 * Unregister MAC
802 */
803 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
804 (void) mac_unregister(ixgbe->mac_hdl);
805 }
806
807 /*
808 * Free statistics
809 */
810 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
811 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
812 }
813
814 /*
815 * Remove interrupt handlers
816 */
817 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
818 ixgbe_rem_intr_handlers(ixgbe);
819 }
820
821 /*
822 * Remove taskq for sfp-status-change
823 */
824 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
825 ddi_taskq_destroy(ixgbe->sfp_taskq);
826 }
827
828 /*
829 * Remove taskq for over-temp
830 */
831 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
832 ddi_taskq_destroy(ixgbe->overtemp_taskq);
833 }
834
835 /*
836 * Remove taskq for external PHYs
837 */
838 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) {
839 ddi_taskq_destroy(ixgbe->phy_taskq);
840 }
841
842 /*
843 * Remove interrupts
844 */
845 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
846 ixgbe_rem_intrs(ixgbe);
847 }
848
849 /*
850 * Unregister interrupt callback handler
851 */
852 if (ixgbe->cb_hdl != NULL) {
853 (void) ddi_cb_unregister(ixgbe->cb_hdl);
854 }
855
856 /*
857 * Remove driver properties
858 */
859 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
860 (void) ddi_prop_remove_all(devinfo);
861 }
862
863 /*
864 * Stop the chipset
865 */
866 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
867 mutex_enter(&ixgbe->gen_lock);
868 ixgbe_chip_stop(ixgbe);
869 mutex_exit(&ixgbe->gen_lock);
870 }
871
872 /*
873 * Free register handle
874 */
875 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
876 if (ixgbe->osdep.reg_handle != NULL)
877 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
878 }
879
880 /*
881 * Free PCI config handle
882 */
883 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
884 if (ixgbe->osdep.cfg_handle != NULL)
885 pci_config_teardown(&ixgbe->osdep.cfg_handle);
886 }
887
888 /*
889 * Free locks
890 */
891 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
892 ixgbe_destroy_locks(ixgbe);
893 }
894
895 /*
896 * Free the rx/tx rings
897 */
898 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
899 ixgbe_free_rings(ixgbe);
900 }
901
902 /*
903 * Unregister FMA capabilities
904 */
905 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
906 ixgbe_fm_fini(ixgbe);
907 }
908
909 /*
910 * Free the driver data structure
911 */
912 kmem_free(ixgbe, sizeof (ixgbe_t));
913
914 ddi_set_driver_private(devinfo, NULL);
915 }
916
917 /*
918 * ixgbe_register_mac - Register the driver and its function pointers with
919 * the GLD interface.
920 */
921 static int
922 ixgbe_register_mac(ixgbe_t *ixgbe)
923 {
924 struct ixgbe_hw *hw = &ixgbe->hw;
925 mac_register_t *mac;
926 int status;
927
928 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
929 return (IXGBE_FAILURE);
930
931 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
932 mac->m_driver = ixgbe;
933 mac->m_dip = ixgbe->dip;
934 mac->m_src_addr = hw->mac.addr;
935 mac->m_callbacks = &ixgbe_m_callbacks;
936 mac->m_min_sdu = 0;
937 mac->m_max_sdu = ixgbe->default_mtu;
938 mac->m_margin = VLAN_TAGSZ;
939 mac->m_priv_props = ixgbe_priv_props;
940 mac->m_v12n = MAC_VIRT_LEVEL1;
941
942 status = mac_register(mac, &ixgbe->mac_hdl);
943
944 mac_free(mac);
945
946 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
947 }
948
949 /*
950 * ixgbe_identify_hardware - Identify the type of the chipset.
951 */
952 static int
953 ixgbe_identify_hardware(ixgbe_t *ixgbe)
954 {
955 struct ixgbe_hw *hw = &ixgbe->hw;
956 struct ixgbe_osdep *osdep = &ixgbe->osdep;
957
958 /*
959 * Get the device id
960 */
961 hw->vendor_id =
962 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
963 hw->device_id =
964 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
965 hw->revision_id =
966 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
967 hw->subsystem_device_id =
968 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
969 hw->subsystem_vendor_id =
970 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
971
972 /*
973 * Set the mac type of the adapter based on the device id
974 */
975 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
976 return (IXGBE_FAILURE);
977 }
978
979 /*
980 * Install adapter capabilities
981 */
982 switch (hw->mac.type) {
983 case ixgbe_mac_82598EB:
984 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
985 ixgbe->capab = &ixgbe_82598eb_cap;
986
987 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
988 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
989 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
990 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
991 }
992 break;
993
994 case ixgbe_mac_82599EB:
995 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
996 ixgbe->capab = &ixgbe_82599eb_cap;
997
998 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
999 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
1000 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
1001 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
1002 }
1003 break;
1004
1005 case ixgbe_mac_X540:
1006 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
1007 ixgbe->capab = &ixgbe_X540_cap;
1008 /*
1009 * For now, X540 is all set in its capab structure.
1010 * As other X540 variants show up, things can change here.
1011 */
1012 break;
1013
1014 case ixgbe_mac_X550:
1015 case ixgbe_mac_X550EM_x:
1016 case ixgbe_mac_X550EM_a:
1017 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
1018 ixgbe->capab = &ixgbe_X550_cap;
1019
1020 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1021 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
1022 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
1023 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP ||
1024 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP_N) {
1025 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
1026 }
1027
1028 /*
1029 * Link detection on X552 SFP+ and X552/X557-AT
1030 */
1031 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1032 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
1033 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
1034 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
1035 ixgbe->capab->other_intr |=
1036 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
1037 }
1038 if (hw->phy.type == ixgbe_phy_x550em_ext_t) {
1039 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540;
1040 }
1041 break;
1042
1043 default:
1044 IXGBE_DEBUGLOG_1(ixgbe,
1045 "adapter not supported in ixgbe_identify_hardware(): %d\n",
1046 hw->mac.type);
1047 return (IXGBE_FAILURE);
1048 }
1049
1050 return (IXGBE_SUCCESS);
1051 }
1052
1053 /*
1054 * ixgbe_regs_map - Map the device registers.
1055 *
1056 */
1057 static int
1058 ixgbe_regs_map(ixgbe_t *ixgbe)
1059 {
1060 dev_info_t *devinfo = ixgbe->dip;
1061 struct ixgbe_hw *hw = &ixgbe->hw;
1062 struct ixgbe_osdep *osdep = &ixgbe->osdep;
1063 off_t mem_size;
1064
1065 /*
1066 * First get the size of device registers to be mapped.
1067 */
1068 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
1069 != DDI_SUCCESS) {
1070 return (IXGBE_FAILURE);
1071 }
1072
1073 /*
1074 * Call ddi_regs_map_setup() to map registers
1075 */
1076 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
1077 (caddr_t *)&hw->hw_addr, 0,
1078 mem_size, &ixgbe_regs_acc_attr,
1079 &osdep->reg_handle)) != DDI_SUCCESS) {
1080 return (IXGBE_FAILURE);
1081 }
1082
1083 return (IXGBE_SUCCESS);
1084 }
1085
1086 /*
1087 * ixgbe_init_properties - Initialize driver properties.
1088 */
1089 static void
1090 ixgbe_init_properties(ixgbe_t *ixgbe)
1091 {
1092 /*
1093 * Get conf file properties, including link settings
1094 * jumbo frames, ring number, descriptor number, etc.
1095 */
1096 ixgbe_get_conf(ixgbe);
1097 }
1098
1099 /*
1100 * ixgbe_init_driver_settings - Initialize driver settings.
1101 *
1102 * The settings include hardware function pointers, bus information,
1103 * rx/tx rings settings, link state, and any other parameters that
1104 * need to be setup during driver initialization.
1105 */
1106 static int
1107 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1108 {
1109 struct ixgbe_hw *hw = &ixgbe->hw;
1110 dev_info_t *devinfo = ixgbe->dip;
1111 ixgbe_rx_ring_t *rx_ring;
1112 ixgbe_rx_group_t *rx_group;
1113 ixgbe_tx_ring_t *tx_ring;
1114 uint32_t rx_size;
1115 uint32_t tx_size;
1116 uint32_t ring_per_group;
1117 int i;
1118
1119 /*
1120 * Initialize chipset specific hardware function pointers
1121 */
1122 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1123 return (IXGBE_FAILURE);
1124 }
1125
1126 /*
1127 * Get the system page size
1128 */
1129 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1130
1131 /*
1132 * Set rx buffer size
1133 *
1134 * The IP header alignment room is counted in the calculation.
1135 * The rx buffer size is in unit of 1K that is required by the
1136 * chipset hardware.
1137 */
1138 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1139 ixgbe->rx_buf_size = ((rx_size >> 10) +
1140 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1141
1142 /*
1143 * Set tx buffer size
1144 */
1145 tx_size = ixgbe->max_frame_size;
1146 ixgbe->tx_buf_size = ((tx_size >> 10) +
1147 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1148
1149 /*
1150 * Initialize rx/tx rings/groups parameters
1151 */
1152 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1153 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1154 rx_ring = &ixgbe->rx_rings[i];
1155 rx_ring->index = i;
1156 rx_ring->ixgbe = ixgbe;
1157 rx_ring->group_index = i / ring_per_group;
1158 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1159 }
1160
1161 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1162 rx_group = &ixgbe->rx_groups[i];
1163 rx_group->index = i;
1164 rx_group->ixgbe = ixgbe;
1165 list_create(&rx_group->vlans, sizeof (ixgbe_vlan_t),
1166 offsetof(ixgbe_vlan_t, ixvl_link));
1167 }
1168
1169 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1170 tx_ring = &ixgbe->tx_rings[i];
1171 tx_ring->index = i;
1172 tx_ring->ixgbe = ixgbe;
1173 if (ixgbe->tx_head_wb_enable)
1174 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1175 else
1176 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1177
1178 tx_ring->ring_size = ixgbe->tx_ring_size;
1179 tx_ring->free_list_size = ixgbe->tx_ring_size +
1180 (ixgbe->tx_ring_size >> 1);
1181 }
1182
1183 /*
1184 * Initialize values of interrupt throttling rate
1185 */
1186 for (i = 1; i < MAX_INTR_VECTOR; i++)
1187 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1188
1189 /*
1190 * The initial link state should be "unknown"
1191 */
1192 ixgbe->link_state = LINK_STATE_UNKNOWN;
1193
1194 return (IXGBE_SUCCESS);
1195 }
1196
1197 /*
1198 * ixgbe_init_locks - Initialize locks.
1199 */
1200 static void
1201 ixgbe_init_locks(ixgbe_t *ixgbe)
1202 {
1203 ixgbe_rx_ring_t *rx_ring;
1204 ixgbe_tx_ring_t *tx_ring;
1205 int i;
1206
1207 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1208 rx_ring = &ixgbe->rx_rings[i];
1209 mutex_init(&rx_ring->rx_lock, NULL,
1210 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1211 }
1212
1213 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1214 tx_ring = &ixgbe->tx_rings[i];
1215 mutex_init(&tx_ring->tx_lock, NULL,
1216 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1217 mutex_init(&tx_ring->recycle_lock, NULL,
1218 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1219 mutex_init(&tx_ring->tcb_head_lock, NULL,
1220 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1221 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1222 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1223 }
1224
1225 mutex_init(&ixgbe->gen_lock, NULL,
1226 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1227
1228 mutex_init(&ixgbe->watchdog_lock, NULL,
1229 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1230 }
1231
1232 /*
1233 * ixgbe_destroy_locks - Destroy locks.
1234 */
1235 static void
1236 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1237 {
1238 ixgbe_rx_ring_t *rx_ring;
1239 ixgbe_tx_ring_t *tx_ring;
1240 int i;
1241
1242 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1243 rx_ring = &ixgbe->rx_rings[i];
1244 mutex_destroy(&rx_ring->rx_lock);
1245 }
1246
1247 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1248 tx_ring = &ixgbe->tx_rings[i];
1249 mutex_destroy(&tx_ring->tx_lock);
1250 mutex_destroy(&tx_ring->recycle_lock);
1251 mutex_destroy(&tx_ring->tcb_head_lock);
1252 mutex_destroy(&tx_ring->tcb_tail_lock);
1253 }
1254
1255 mutex_destroy(&ixgbe->gen_lock);
1256 mutex_destroy(&ixgbe->watchdog_lock);
1257 }
1258
1259 /*
1260 * We need to try and determine which LED index in hardware corresponds to the
1261 * link/activity LED. This is the one that'll be overwritten when we perform
1262 * GLDv3 LED activity.
1263 */
1264 static void
1265 ixgbe_led_init(ixgbe_t *ixgbe)
1266 {
1267 uint32_t reg, i;
1268 struct ixgbe_hw *hw = &ixgbe->hw;
1269
1270 reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL);
1271 for (i = 0; i < 4; i++) {
1272 if (((reg >> IXGBE_LED_MODE_SHIFT(i)) &
1273 IXGBE_LED_MODE_MASK_BASE) == IXGBE_LED_LINK_ACTIVE) {
1274 ixgbe->ixgbe_led_index = i;
1275 return;
1276 }
1277 }
1278
1279 /*
1280 * If we couldn't determine this, we use the default for various MACs
1281 * based on information Intel has inserted into other drivers over the
1282 * years.
1283 */
1284 switch (hw->mac.type) {
1285 case ixgbe_mac_X550EM_a:
1286 ixgbe->ixgbe_led_index = 0;
1287 break;
1288 case ixgbe_mac_X550EM_x:
1289 ixgbe->ixgbe_led_index = 1;
1290 break;
1291 default:
1292 ixgbe->ixgbe_led_index = 2;
1293 break;
1294 }
1295 }
1296
1297 static int
1298 ixgbe_resume(dev_info_t *devinfo)
1299 {
1300 ixgbe_t *ixgbe;
1301 int i;
1302
1303 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1304 if (ixgbe == NULL)
1305 return (DDI_FAILURE);
1306
1307 mutex_enter(&ixgbe->gen_lock);
1308
1309 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1310 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1311 mutex_exit(&ixgbe->gen_lock);
1312 return (DDI_FAILURE);
1313 }
1314
1315 /*
1316 * Enable and start the watchdog timer
1317 */
1318 ixgbe_enable_watchdog_timer(ixgbe);
1319 }
1320
1321 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1322
1323 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1324 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1325 mac_tx_ring_update(ixgbe->mac_hdl,
1326 ixgbe->tx_rings[i].ring_handle);
1327 }
1328 }
1329
1330 mutex_exit(&ixgbe->gen_lock);
1331
1332 return (DDI_SUCCESS);
1333 }
1334
1335 static int
1336 ixgbe_suspend(dev_info_t *devinfo)
1337 {
1338 ixgbe_t *ixgbe;
1339
1340 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1341 if (ixgbe == NULL)
1342 return (DDI_FAILURE);
1343
1344 mutex_enter(&ixgbe->gen_lock);
1345
1346 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1347 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1348 mutex_exit(&ixgbe->gen_lock);
1349 return (DDI_SUCCESS);
1350 }
1351 ixgbe_stop(ixgbe, B_FALSE);
1352
1353 mutex_exit(&ixgbe->gen_lock);
1354
1355 /*
1356 * Disable and stop the watchdog timer
1357 */
1358 ixgbe_disable_watchdog_timer(ixgbe);
1359
1360 return (DDI_SUCCESS);
1361 }
1362
1363 /*
1364 * ixgbe_init - Initialize the device.
1365 */
1366 static int
1367 ixgbe_init(ixgbe_t *ixgbe)
1368 {
1369 struct ixgbe_hw *hw = &ixgbe->hw;
1370 u8 pbanum[IXGBE_PBANUM_LENGTH];
1371 int rv;
1372
1373 mutex_enter(&ixgbe->gen_lock);
1374
1375 /*
1376 * Configure/Initialize hardware
1377 */
1378 rv = ixgbe_init_hw(hw);
1379 if (rv != IXGBE_SUCCESS) {
1380 switch (rv) {
1381
1382 /*
1383 * The first three errors are not prohibitive to us progressing
1384 * further, and are maily advisory in nature. In the case of a
1385 * SFP module not being present or not deemed supported by the
1386 * common code, we adivse the operator of this fact but carry on
1387 * instead of failing hard, as SFPs can be inserted or replaced
1388 * while the driver is running. In the case of a unknown error,
1389 * we fail-hard, logging the reason and emitting a FMA event.
1390 */
1391 case IXGBE_ERR_EEPROM_VERSION:
1392 ixgbe_error(ixgbe,
1393 "This Intel 10Gb Ethernet device is pre-release and"
1394 " contains outdated firmware. Please contact your"
1395 " hardware vendor for a replacement.");
1396 break;
1397 case IXGBE_ERR_SFP_NOT_PRESENT:
1398 ixgbe_error(ixgbe,
1399 "No SFP+ module detected on this interface. Please "
1400 "install a supported SFP+ module for this "
1401 "interface to become operational.");
1402 break;
1403 case IXGBE_ERR_SFP_NOT_SUPPORTED:
1404 ixgbe_error(ixgbe,
1405 "Unsupported SFP+ module detected. Please replace "
1406 "it with a supported SFP+ module per Intel "
1407 "documentation, or bypass this check with "
1408 "allow_unsupported_sfp=1 in ixgbe.conf.");
1409 break;
1410 default:
1411 ixgbe_error(ixgbe,
1412 "Failed to initialize hardware. ixgbe_init_hw "
1413 "returned %d", rv);
1414 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1415 goto init_fail;
1416 }
1417 }
1418
1419 /*
1420 * Need to init eeprom before validating the checksum.
1421 */
1422 if (ixgbe_init_eeprom_params(hw) < 0) {
1423 ixgbe_error(ixgbe,
1424 "Unable to intitialize the eeprom interface.");
1425 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1426 goto init_fail;
1427 }
1428
1429 /*
1430 * NVM validation
1431 */
1432 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1433 /*
1434 * Some PCI-E parts fail the first check due to
1435 * the link being in sleep state. Call it again,
1436 * if it fails a second time it's a real issue.
1437 */
1438 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1439 ixgbe_error(ixgbe,
1440 "Invalid NVM checksum. Please contact "
1441 "the vendor to update the NVM.");
1442 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1443 goto init_fail;
1444 }
1445 }
1446
1447 /*
1448 * Setup default flow control thresholds - enable/disable
1449 * & flow control type is controlled by ixgbe.conf
1450 */
1451 hw->fc.high_water[0] = DEFAULT_FCRTH;
1452 hw->fc.low_water[0] = DEFAULT_FCRTL;
1453 hw->fc.pause_time = DEFAULT_FCPAUSE;
1454 hw->fc.send_xon = B_TRUE;
1455
1456 /*
1457 * Initialize flow control
1458 */
1459 (void) ixgbe_start_hw(hw);
1460
1461 /*
1462 * Initialize link settings
1463 */
1464 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1465
1466 /*
1467 * Initialize the chipset hardware
1468 */
1469 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1470 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1471 goto init_fail;
1472 }
1473
1474 /*
1475 * Read identifying information and place in devinfo.
1476 */
1477 pbanum[0] = '\0';
1478 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1479 if (*pbanum != '\0') {
1480 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1481 "printed-board-assembly", (char *)pbanum);
1482 }
1483
1484 /*
1485 * Determine LED index.
1486 */
1487 ixgbe_led_init(ixgbe);
1488
1489 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1490 goto init_fail;
1491 }
1492
1493 mutex_exit(&ixgbe->gen_lock);
1494 return (IXGBE_SUCCESS);
1495
1496 init_fail:
1497 /*
1498 * Reset PHY
1499 */
1500 (void) ixgbe_reset_phy(hw);
1501
1502 mutex_exit(&ixgbe->gen_lock);
1503 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1504 return (IXGBE_FAILURE);
1505 }
1506
1507 /*
1508 * ixgbe_chip_start - Initialize and start the chipset hardware.
1509 */
1510 static int
1511 ixgbe_chip_start(ixgbe_t *ixgbe)
1512 {
1513 struct ixgbe_hw *hw = &ixgbe->hw;
1514 int i;
1515
1516 ASSERT(mutex_owned(&ixgbe->gen_lock));
1517
1518 /*
1519 * Get the mac address
1520 * This function should handle SPARC case correctly.
1521 */
1522 if (!ixgbe_find_mac_address(ixgbe)) {
1523 ixgbe_error(ixgbe, "Failed to get the mac address");
1524 return (IXGBE_FAILURE);
1525 }
1526
1527 /*
1528 * Validate the mac address
1529 */
1530 (void) ixgbe_init_rx_addrs(hw);
1531 if (!is_valid_mac_addr(hw->mac.addr)) {
1532 ixgbe_error(ixgbe, "Invalid mac address");
1533 return (IXGBE_FAILURE);
1534 }
1535
1536 /*
1537 * Re-enable relaxed ordering for performance. It is disabled
1538 * by default in the hardware init.
1539 */
1540 if (ixgbe->relax_order_enable == B_TRUE)
1541 ixgbe_enable_relaxed_ordering(hw);
1542
1543 /*
1544 * Setup adapter interrupt vectors
1545 */
1546 ixgbe_setup_adapter_vector(ixgbe);
1547
1548 /*
1549 * Initialize unicast addresses.
1550 */
1551 ixgbe_init_unicst(ixgbe);
1552
1553 /*
1554 * Setup and initialize the mctable structures.
1555 */
1556 ixgbe_setup_multicst(ixgbe);
1557
1558 /*
1559 * Set interrupt throttling rate
1560 */
1561 for (i = 0; i < ixgbe->intr_cnt; i++) {
1562 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1563 }
1564
1565 /*
1566 * Disable Wake-on-LAN
1567 */
1568 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
1569
1570 /*
1571 * Some adapters offer Energy Efficient Ethernet (EEE) support.
1572 * Due to issues with EEE in e1000g/igb, we disable this by default
1573 * as a precautionary measure.
1574 *
1575 * Currently, this is present on a number of the X550 family parts.
1576 */
1577 (void) ixgbe_setup_eee(hw, B_FALSE);
1578
1579 /*
1580 * Turn on any present SFP Tx laser
1581 */
1582 ixgbe_enable_tx_laser(hw);
1583
1584 /*
1585 * Power on the PHY
1586 */
1587 (void) ixgbe_set_phy_power(hw, B_TRUE);
1588
1589 /*
1590 * Save the state of the PHY
1591 */
1592 ixgbe_get_hw_state(ixgbe);
1593
1594 /*
1595 * Make sure driver has control
1596 */
1597 ixgbe_get_driver_control(hw);
1598
1599 return (IXGBE_SUCCESS);
1600 }
1601
1602 /*
1603 * ixgbe_chip_stop - Stop the chipset hardware
1604 */
1605 static void
1606 ixgbe_chip_stop(ixgbe_t *ixgbe)
1607 {
1608 struct ixgbe_hw *hw = &ixgbe->hw;
1609 int rv;
1610
1611 ASSERT(mutex_owned(&ixgbe->gen_lock));
1612
1613 /*
1614 * Stop interupt generation and disable Tx unit
1615 */
1616 hw->adapter_stopped = B_FALSE;
1617 (void) ixgbe_stop_adapter(hw);
1618
1619 /*
1620 * Reset the chipset
1621 */
1622 (void) ixgbe_reset_hw(hw);
1623
1624 /*
1625 * Reset PHY
1626 */
1627 (void) ixgbe_reset_phy(hw);
1628
1629 /*
1630 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting
1631 * the PHY while doing so. Else, just power down the PHY.
1632 */
1633 if (hw->phy.ops.enter_lplu != NULL) {
1634 hw->phy.reset_disable = B_TRUE;
1635 rv = hw->phy.ops.enter_lplu(hw);
1636 if (rv != IXGBE_SUCCESS)
1637 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv);
1638 hw->phy.reset_disable = B_FALSE;
1639 } else {
1640 (void) ixgbe_set_phy_power(hw, B_FALSE);
1641 }
1642
1643 /*
1644 * Turn off any present SFP Tx laser
1645 * Expected for health and safety reasons
1646 */
1647 ixgbe_disable_tx_laser(hw);
1648
1649 /*
1650 * Tell firmware driver is no longer in control
1651 */
1652 ixgbe_release_driver_control(hw);
1653
1654 }
1655
1656 /*
1657 * ixgbe_reset - Reset the chipset and re-start the driver.
1658 *
1659 * It involves stopping and re-starting the chipset,
1660 * and re-configuring the rx/tx rings.
1661 */
1662 static int
1663 ixgbe_reset(ixgbe_t *ixgbe)
1664 {
1665 int i;
1666
1667 /*
1668 * Disable and stop the watchdog timer
1669 */
1670 ixgbe_disable_watchdog_timer(ixgbe);
1671
1672 mutex_enter(&ixgbe->gen_lock);
1673
1674 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1675 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1676
1677 ixgbe_stop(ixgbe, B_FALSE);
1678
1679 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1680 mutex_exit(&ixgbe->gen_lock);
1681 return (IXGBE_FAILURE);
1682 }
1683
1684 /*
1685 * After resetting, need to recheck the link status.
1686 */
1687 ixgbe->link_check_complete = B_FALSE;
1688 ixgbe->link_check_hrtime = gethrtime() +
1689 (IXGBE_LINK_UP_TIME * 100000000ULL);
1690
1691 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1692
1693 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1694 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1695 mac_tx_ring_update(ixgbe->mac_hdl,
1696 ixgbe->tx_rings[i].ring_handle);
1697 }
1698 }
1699
1700 mutex_exit(&ixgbe->gen_lock);
1701
1702 /*
1703 * Enable and start the watchdog timer
1704 */
1705 ixgbe_enable_watchdog_timer(ixgbe);
1706
1707 return (IXGBE_SUCCESS);
1708 }
1709
1710 /*
1711 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1712 */
1713 static void
1714 ixgbe_tx_clean(ixgbe_t *ixgbe)
1715 {
1716 ixgbe_tx_ring_t *tx_ring;
1717 tx_control_block_t *tcb;
1718 link_list_t pending_list;
1719 uint32_t desc_num;
1720 int i, j;
1721
1722 LINK_LIST_INIT(&pending_list);
1723
1724 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1725 tx_ring = &ixgbe->tx_rings[i];
1726
1727 mutex_enter(&tx_ring->recycle_lock);
1728
1729 /*
1730 * Clean the pending tx data - the pending packets in the
1731 * work_list that have no chances to be transmitted again.
1732 *
1733 * We must ensure the chipset is stopped or the link is down
1734 * before cleaning the transmit packets.
1735 */
1736 desc_num = 0;
1737 for (j = 0; j < tx_ring->ring_size; j++) {
1738 tcb = tx_ring->work_list[j];
1739 if (tcb != NULL) {
1740 desc_num += tcb->desc_num;
1741
1742 tx_ring->work_list[j] = NULL;
1743
1744 ixgbe_free_tcb(tcb);
1745
1746 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1747 }
1748 }
1749
1750 if (desc_num > 0) {
1751 atomic_add_32(&tx_ring->tbd_free, desc_num);
1752 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1753
1754 /*
1755 * Reset the head and tail pointers of the tbd ring;
1756 * Reset the writeback head if it's enable.
1757 */
1758 tx_ring->tbd_head = 0;
1759 tx_ring->tbd_tail = 0;
1760 if (ixgbe->tx_head_wb_enable)
1761 *tx_ring->tbd_head_wb = 0;
1762
1763 IXGBE_WRITE_REG(&ixgbe->hw,
1764 IXGBE_TDH(tx_ring->index), 0);
1765 IXGBE_WRITE_REG(&ixgbe->hw,
1766 IXGBE_TDT(tx_ring->index), 0);
1767 }
1768
1769 mutex_exit(&tx_ring->recycle_lock);
1770
1771 /*
1772 * Add the tx control blocks in the pending list to
1773 * the free list.
1774 */
1775 ixgbe_put_free_list(tx_ring, &pending_list);
1776 }
1777 }
1778
1779 /*
1780 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1781 * transmitted.
1782 */
1783 static boolean_t
1784 ixgbe_tx_drain(ixgbe_t *ixgbe)
1785 {
1786 ixgbe_tx_ring_t *tx_ring;
1787 boolean_t done;
1788 int i, j;
1789
1790 /*
1791 * Wait for a specific time to allow pending tx packets
1792 * to be transmitted.
1793 *
1794 * Check the counter tbd_free to see if transmission is done.
1795 * No lock protection is needed here.
1796 *
1797 * Return B_TRUE if all pending packets have been transmitted;
1798 * Otherwise return B_FALSE;
1799 */
1800 for (i = 0; i < TX_DRAIN_TIME; i++) {
1801
1802 done = B_TRUE;
1803 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1804 tx_ring = &ixgbe->tx_rings[j];
1805 done = done &&
1806 (tx_ring->tbd_free == tx_ring->ring_size);
1807 }
1808
1809 if (done)
1810 break;
1811
1812 msec_delay(1);
1813 }
1814
1815 return (done);
1816 }
1817
1818 /*
1819 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1820 */
1821 static boolean_t
1822 ixgbe_rx_drain(ixgbe_t *ixgbe)
1823 {
1824 boolean_t done = B_TRUE;
1825 int i;
1826
1827 /*
1828 * Polling the rx free list to check if those rx buffers held by
1829 * the upper layer are released.
1830 *
1831 * Check the counter rcb_free to see if all pending buffers are
1832 * released. No lock protection is needed here.
1833 *
1834 * Return B_TRUE if all pending buffers have been released;
1835 * Otherwise return B_FALSE;
1836 */
1837 for (i = 0; i < RX_DRAIN_TIME; i++) {
1838 done = (ixgbe->rcb_pending == 0);
1839
1840 if (done)
1841 break;
1842
1843 msec_delay(1);
1844 }
1845
1846 return (done);
1847 }
1848
1849 /*
1850 * ixgbe_start - Start the driver/chipset.
1851 */
1852 int
1853 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1854 {
1855 struct ixgbe_hw *hw = &ixgbe->hw;
1856 int i;
1857
1858 ASSERT(mutex_owned(&ixgbe->gen_lock));
1859
1860 if (alloc_buffer) {
1861 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1862 ixgbe_error(ixgbe,
1863 "Failed to allocate software receive rings");
1864 return (IXGBE_FAILURE);
1865 }
1866
1867 /* Allocate buffers for all the rx/tx rings */
1868 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1869 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1870 return (IXGBE_FAILURE);
1871 }
1872
1873 ixgbe->tx_ring_init = B_TRUE;
1874 } else {
1875 ixgbe->tx_ring_init = B_FALSE;
1876 }
1877
1878 for (i = 0; i < ixgbe->num_rx_rings; i++)
1879 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1880 for (i = 0; i < ixgbe->num_tx_rings; i++)
1881 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1882
1883 /*
1884 * Start the chipset hardware
1885 */
1886 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1887 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1888 goto start_failure;
1889 }
1890
1891 /*
1892 * Configure link now for X550
1893 *
1894 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the
1895 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550,
1896 * the resting state of the link would be the maximum speed that
1897 * autonegotiation will allow (usually 10Gb, infrastructure allowing)
1898 * so we never bothered with explicitly setting the link to 10Gb as it
1899 * would already be at that state on driver attach. With X550, we must
1900 * trigger a re-negotiation of the link in order to switch from a LPLU
1901 * 1Gb link to 10Gb (cable and link partner permitting.)
1902 */
1903 if (hw->mac.type == ixgbe_mac_X550 ||
1904 hw->mac.type == ixgbe_mac_X550EM_a ||
1905 hw->mac.type == ixgbe_mac_X550EM_x) {
1906 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE);
1907 ixgbe_get_hw_state(ixgbe);
1908 }
1909
1910 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1911 goto start_failure;
1912 }
1913
1914 /*
1915 * Setup the rx/tx rings
1916 */
1917 if (ixgbe_setup_rings(ixgbe) != IXGBE_SUCCESS)
1918 goto start_failure;
1919
1920 /*
1921 * ixgbe_start() will be called when resetting, however if reset
1922 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1923 * before enabling the interrupts.
1924 */
1925 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1926 | IXGBE_STALL| IXGBE_OVERTEMP));
1927
1928 /*
1929 * Enable adapter interrupts
1930 * The interrupts must be enabled after the driver state is START
1931 */
1932 ixgbe_enable_adapter_interrupts(ixgbe);
1933
1934 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1935 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1936 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1937 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1938
1939 return (IXGBE_SUCCESS);
1940
1941 start_failure:
1942 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1943 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1944 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1945 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1946
1947 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1948
1949 return (IXGBE_FAILURE);
1950 }
1951
1952 /*
1953 * ixgbe_stop - Stop the driver/chipset.
1954 */
1955 void
1956 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1957 {
1958 int i;
1959
1960 ASSERT(mutex_owned(&ixgbe->gen_lock));
1961
1962 /*
1963 * Disable the adapter interrupts
1964 */
1965 ixgbe_disable_adapter_interrupts(ixgbe);
1966
1967 /*
1968 * Drain the pending tx packets
1969 */
1970 (void) ixgbe_tx_drain(ixgbe);
1971
1972 for (i = 0; i < ixgbe->num_rx_rings; i++)
1973 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1974 for (i = 0; i < ixgbe->num_tx_rings; i++)
1975 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1976
1977 /*
1978 * Stop the chipset hardware
1979 */
1980 ixgbe_chip_stop(ixgbe);
1981
1982 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1983 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1984 }
1985
1986 /*
1987 * Clean the pending tx data/resources
1988 */
1989 ixgbe_tx_clean(ixgbe);
1990
1991 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1992 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1993 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1994 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1995
1996 if (ixgbe->link_state == LINK_STATE_UP) {
1997 ixgbe->link_state = LINK_STATE_UNKNOWN;
1998 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1999 }
2000
2001 if (free_buffer) {
2002 /*
2003 * Release the DMA/memory resources of rx/tx rings
2004 */
2005 ixgbe_free_dma(ixgbe);
2006 ixgbe_free_rx_data(ixgbe);
2007 }
2008 }
2009
2010 /*
2011 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
2012 */
2013 /* ARGSUSED */
2014 static int
2015 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
2016 void *arg1, void *arg2)
2017 {
2018 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
2019
2020 switch (cbaction) {
2021 /* IRM callback */
2022 int count;
2023 case DDI_CB_INTR_ADD:
2024 case DDI_CB_INTR_REMOVE:
2025 count = (int)(uintptr_t)cbarg;
2026 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
2027 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
2028 int, ixgbe->intr_cnt);
2029 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
2030 DDI_SUCCESS) {
2031 ixgbe_error(ixgbe,
2032 "IRM CB: Failed to adjust interrupts");
2033 goto cb_fail;
2034 }
2035 break;
2036 default:
2037 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
2038 cbaction);
2039 return (DDI_ENOTSUP);
2040 }
2041 return (DDI_SUCCESS);
2042 cb_fail:
2043 return (DDI_FAILURE);
2044 }
2045
2046 /*
2047 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
2048 */
2049 static int
2050 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
2051 {
2052 int i, rc, actual;
2053
2054 if (count == 0)
2055 return (DDI_SUCCESS);
2056
2057 if ((cbaction == DDI_CB_INTR_ADD &&
2058 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
2059 (cbaction == DDI_CB_INTR_REMOVE &&
2060 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
2061 return (DDI_FAILURE);
2062
2063 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
2064 return (DDI_FAILURE);
2065 }
2066
2067 for (i = 0; i < ixgbe->num_rx_rings; i++)
2068 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
2069 for (i = 0; i < ixgbe->num_tx_rings; i++)
2070 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
2071
2072 mutex_enter(&ixgbe->gen_lock);
2073 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
2074 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
2075 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
2076 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
2077
2078 ixgbe_stop(ixgbe, B_FALSE);
2079 /*
2080 * Disable interrupts
2081 */
2082 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
2083 rc = ixgbe_disable_intrs(ixgbe);
2084 ASSERT(rc == IXGBE_SUCCESS);
2085 }
2086 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
2087
2088 /*
2089 * Remove interrupt handlers
2090 */
2091 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
2092 ixgbe_rem_intr_handlers(ixgbe);
2093 }
2094 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
2095
2096 /*
2097 * Clear vect_map
2098 */
2099 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
2100 switch (cbaction) {
2101 case DDI_CB_INTR_ADD:
2102 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
2103 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
2104 DDI_INTR_ALLOC_NORMAL);
2105 if (rc != DDI_SUCCESS || actual != count) {
2106 ixgbe_log(ixgbe, "Adjust interrupts failed."
2107 "return: %d, irm cb size: %d, actual: %d",
2108 rc, count, actual);
2109 goto intr_adjust_fail;
2110 }
2111 ixgbe->intr_cnt += count;
2112 break;
2113
2114 case DDI_CB_INTR_REMOVE:
2115 for (i = ixgbe->intr_cnt - count;
2116 i < ixgbe->intr_cnt; i ++) {
2117 rc = ddi_intr_free(ixgbe->htable[i]);
2118 ixgbe->htable[i] = NULL;
2119 if (rc != DDI_SUCCESS) {
2120 ixgbe_log(ixgbe, "Adjust interrupts failed."
2121 "return: %d, irm cb size: %d, actual: %d",
2122 rc, count, actual);
2123 goto intr_adjust_fail;
2124 }
2125 }
2126 ixgbe->intr_cnt -= count;
2127 break;
2128 }
2129
2130 /*
2131 * Get priority for first vector, assume remaining are all the same
2132 */
2133 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
2134 if (rc != DDI_SUCCESS) {
2135 ixgbe_log(ixgbe,
2136 "Get interrupt priority failed: %d", rc);
2137 goto intr_adjust_fail;
2138 }
2139 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
2140 if (rc != DDI_SUCCESS) {
2141 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
2142 goto intr_adjust_fail;
2143 }
2144 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
2145
2146 /*
2147 * Map rings to interrupt vectors
2148 */
2149 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
2150 ixgbe_error(ixgbe,
2151 "IRM CB: Failed to map interrupts to vectors");
2152 goto intr_adjust_fail;
2153 }
2154
2155 /*
2156 * Add interrupt handlers
2157 */
2158 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
2159 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
2160 goto intr_adjust_fail;
2161 }
2162 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
2163
2164 /*
2165 * Now that mutex locks are initialized, and the chip is also
2166 * initialized, enable interrupts.
2167 */
2168 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
2169 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
2170 goto intr_adjust_fail;
2171 }
2172 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
2173 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
2174 ixgbe_error(ixgbe, "IRM CB: Failed to start");
2175 goto intr_adjust_fail;
2176 }
2177 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
2178 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
2179 ixgbe->ixgbe_state |= IXGBE_STARTED;
2180 mutex_exit(&ixgbe->gen_lock);
2181
2182 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2183 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
2184 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
2185 }
2186 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2187 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
2188 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
2189 }
2190
2191 /* Wakeup all Tx rings */
2192 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2193 mac_tx_ring_update(ixgbe->mac_hdl,
2194 ixgbe->tx_rings[i].ring_handle);
2195 }
2196
2197 IXGBE_DEBUGLOG_3(ixgbe,
2198 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
2199 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
2200 return (DDI_SUCCESS);
2201
2202 intr_adjust_fail:
2203 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2204 mutex_exit(&ixgbe->gen_lock);
2205 return (DDI_FAILURE);
2206 }
2207
2208 /*
2209 * ixgbe_intr_cb_register - Register interrupt callback function.
2210 */
2211 static int
2212 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
2213 {
2214 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
2215 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
2216 return (IXGBE_FAILURE);
2217 }
2218 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
2219 return (IXGBE_SUCCESS);
2220 }
2221
2222 /*
2223 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2224 */
2225 static int
2226 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2227 {
2228 /*
2229 * Allocate memory space for rx rings
2230 */
2231 ixgbe->rx_rings = kmem_zalloc(
2232 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2233 KM_NOSLEEP);
2234
2235 if (ixgbe->rx_rings == NULL) {
2236 return (IXGBE_FAILURE);
2237 }
2238
2239 /*
2240 * Allocate memory space for tx rings
2241 */
2242 ixgbe->tx_rings = kmem_zalloc(
2243 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2244 KM_NOSLEEP);
2245
2246 if (ixgbe->tx_rings == NULL) {
2247 kmem_free(ixgbe->rx_rings,
2248 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2249 ixgbe->rx_rings = NULL;
2250 return (IXGBE_FAILURE);
2251 }
2252
2253 /*
2254 * Allocate memory space for rx ring groups
2255 */
2256 ixgbe->rx_groups = kmem_zalloc(
2257 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2258 KM_NOSLEEP);
2259
2260 if (ixgbe->rx_groups == NULL) {
2261 kmem_free(ixgbe->rx_rings,
2262 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2263 kmem_free(ixgbe->tx_rings,
2264 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2265 ixgbe->rx_rings = NULL;
2266 ixgbe->tx_rings = NULL;
2267 return (IXGBE_FAILURE);
2268 }
2269
2270 return (IXGBE_SUCCESS);
2271 }
2272
2273 /*
2274 * ixgbe_free_rings - Free the memory space of rx/tx rings.
2275 */
2276 static void
2277 ixgbe_free_rings(ixgbe_t *ixgbe)
2278 {
2279 if (ixgbe->rx_rings != NULL) {
2280 kmem_free(ixgbe->rx_rings,
2281 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2282 ixgbe->rx_rings = NULL;
2283 }
2284
2285 if (ixgbe->tx_rings != NULL) {
2286 kmem_free(ixgbe->tx_rings,
2287 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2288 ixgbe->tx_rings = NULL;
2289 }
2290
2291 for (uint_t i = 0; i < ixgbe->num_rx_groups; i++) {
2292 ixgbe_vlan_t *vlp;
2293 ixgbe_rx_group_t *rx_group = &ixgbe->rx_groups[i];
2294
2295 while ((vlp = list_remove_head(&rx_group->vlans)) != NULL)
2296 kmem_free(vlp, sizeof (ixgbe_vlan_t));
2297
2298 list_destroy(&rx_group->vlans);
2299 }
2300
2301 if (ixgbe->rx_groups != NULL) {
2302 kmem_free(ixgbe->rx_groups,
2303 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2304 ixgbe->rx_groups = NULL;
2305 }
2306 }
2307
2308 static int
2309 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2310 {
2311 ixgbe_rx_ring_t *rx_ring;
2312 int i;
2313
2314 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2315 rx_ring = &ixgbe->rx_rings[i];
2316 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2317 goto alloc_rx_rings_failure;
2318 }
2319 return (IXGBE_SUCCESS);
2320
2321 alloc_rx_rings_failure:
2322 ixgbe_free_rx_data(ixgbe);
2323 return (IXGBE_FAILURE);
2324 }
2325
2326 static void
2327 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2328 {
2329 ixgbe_rx_ring_t *rx_ring;
2330 ixgbe_rx_data_t *rx_data;
2331 int i;
2332
2333 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2334 rx_ring = &ixgbe->rx_rings[i];
2335
2336 mutex_enter(&ixgbe->rx_pending_lock);
2337 rx_data = rx_ring->rx_data;
2338
2339 if (rx_data != NULL) {
2340 rx_data->flag |= IXGBE_RX_STOPPED;
2341
2342 if (rx_data->rcb_pending == 0) {
2343 ixgbe_free_rx_ring_data(rx_data);
2344 rx_ring->rx_data = NULL;
2345 }
2346 }
2347
2348 mutex_exit(&ixgbe->rx_pending_lock);
2349 }
2350 }
2351
2352 /*
2353 * ixgbe_setup_rings - Setup rx/tx rings.
2354 */
2355 static int
2356 ixgbe_setup_rings(ixgbe_t *ixgbe)
2357 {
2358 /*
2359 * Setup the rx/tx rings, including the following:
2360 *
2361 * 1. Setup the descriptor ring and the control block buffers;
2362 * 2. Initialize necessary registers for receive/transmit;
2363 * 3. Initialize software pointers/parameters for receive/transmit;
2364 */
2365 if (ixgbe_setup_rx(ixgbe) != IXGBE_SUCCESS)
2366 return (IXGBE_FAILURE);
2367
2368 ixgbe_setup_tx(ixgbe);
2369
2370 return (IXGBE_SUCCESS);
2371 }
2372
2373 static void
2374 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2375 {
2376 ixgbe_t *ixgbe = rx_ring->ixgbe;
2377 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2378 struct ixgbe_hw *hw = &ixgbe->hw;
2379 rx_control_block_t *rcb;
2380 union ixgbe_adv_rx_desc *rbd;
2381 uint32_t size;
2382 uint32_t buf_low;
2383 uint32_t buf_high;
2384 uint32_t reg_val;
2385 int i;
2386
2387 ASSERT(mutex_owned(&rx_ring->rx_lock));
2388 ASSERT(mutex_owned(&ixgbe->gen_lock));
2389
2390 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2391 rcb = rx_data->work_list[i];
2392 rbd = &rx_data->rbd_ring[i];
2393
2394 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2395 rbd->read.hdr_addr = 0;
2396 }
2397
2398 /*
2399 * Initialize the length register
2400 */
2401 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2402 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2403
2404 /*
2405 * Initialize the base address registers
2406 */
2407 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2408 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2409 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2410 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2411
2412 /*
2413 * Setup head & tail pointers
2414 */
2415 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2416 rx_data->ring_size - 1);
2417 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2418
2419 rx_data->rbd_next = 0;
2420 rx_data->lro_first = 0;
2421
2422 /*
2423 * Setup the Receive Descriptor Control Register (RXDCTL)
2424 * PTHRESH=32 descriptors (half the internal cache)
2425 * HTHRESH=0 descriptors (to minimize latency on fetch)
2426 * WTHRESH defaults to 1 (writeback each descriptor)
2427 */
2428 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2429 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2430
2431 /* Not a valid value for 82599, X540 or X550 */
2432 if (hw->mac.type == ixgbe_mac_82598EB) {
2433 reg_val |= 0x0020; /* pthresh */
2434 }
2435 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2436
2437 if (hw->mac.type == ixgbe_mac_82599EB ||
2438 hw->mac.type == ixgbe_mac_X540 ||
2439 hw->mac.type == ixgbe_mac_X550 ||
2440 hw->mac.type == ixgbe_mac_X550EM_x ||
2441 hw->mac.type == ixgbe_mac_X550EM_a) {
2442 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2443 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2444 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2445 }
2446
2447 /*
2448 * Setup the Split and Replication Receive Control Register.
2449 * Set the rx buffer size and the advanced descriptor type.
2450 */
2451 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2452 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2453 reg_val |= IXGBE_SRRCTL_DROP_EN;
2454 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2455 }
2456
2457 static int
2458 ixgbe_setup_rx(ixgbe_t *ixgbe)
2459 {
2460 ixgbe_rx_ring_t *rx_ring;
2461 struct ixgbe_hw *hw = &ixgbe->hw;
2462 uint32_t reg_val;
2463 uint32_t i;
2464 uint32_t psrtype_rss_bit;
2465
2466 /*
2467 * Ensure that Rx is disabled while setting up
2468 * the Rx unit and Rx descriptor ring(s)
2469 */
2470 ixgbe_disable_rx(hw);
2471
2472 /* PSRTYPE must be configured for 82599 */
2473 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2474 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2475 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2476 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2477 reg_val |= IXGBE_PSRTYPE_L2HDR;
2478 reg_val |= 0x80000000;
2479 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2480 } else {
2481 if (ixgbe->num_rx_groups > 32) {
2482 psrtype_rss_bit = 0x20000000;
2483 } else {
2484 psrtype_rss_bit = 0x40000000;
2485 }
2486 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2487 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2488 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2489 reg_val |= IXGBE_PSRTYPE_L2HDR;
2490 reg_val |= psrtype_rss_bit;
2491 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2492 }
2493 }
2494
2495 /*
2496 * Set filter control in FCTRL to determine types of packets are passed
2497 * up to the driver.
2498 * - Pass broadcast packets.
2499 * - Do not pass flow control pause frames (82598-specific)
2500 */
2501 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2502 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */
2503 if (hw->mac.type == ixgbe_mac_82598EB) {
2504 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */
2505 }
2506 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2507
2508 /*
2509 * Hardware checksum settings
2510 */
2511 if (ixgbe->rx_hcksum_enable) {
2512 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2513 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2514 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2515 }
2516
2517 /*
2518 * Setup VMDq and RSS for multiple receive queues
2519 */
2520 switch (ixgbe->classify_mode) {
2521 case IXGBE_CLASSIFY_RSS:
2522 /*
2523 * One group, only RSS is needed when more than
2524 * one ring enabled.
2525 */
2526 ixgbe_setup_rss(ixgbe);
2527 break;
2528
2529 case IXGBE_CLASSIFY_VMDQ:
2530 /*
2531 * Multiple groups, each group has one ring,
2532 * only VMDq is needed.
2533 */
2534 ixgbe_setup_vmdq(ixgbe);
2535 break;
2536
2537 case IXGBE_CLASSIFY_VMDQ_RSS:
2538 /*
2539 * Multiple groups and multiple rings, both
2540 * VMDq and RSS are needed.
2541 */
2542 ixgbe_setup_vmdq_rss(ixgbe);
2543 break;
2544
2545 default:
2546 break;
2547 }
2548
2549 /*
2550 * Initialize VLAN SW and HW state if VLAN filtering is
2551 * enabled.
2552 */
2553 if (ixgbe->vlft_enabled) {
2554 if (ixgbe_init_vlan(ixgbe) != IXGBE_SUCCESS)
2555 return (IXGBE_FAILURE);
2556 }
2557
2558 /*
2559 * Enable the receive unit. This must be done after filter
2560 * control is set in FCTRL. On 82598, we disable the descriptor monitor.
2561 * 82598 is the only adapter which defines this RXCTRL option.
2562 */
2563 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
2564 if (hw->mac.type == ixgbe_mac_82598EB)
2565 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */
2566 reg_val |= IXGBE_RXCTRL_RXEN;
2567 (void) ixgbe_enable_rx_dma(hw, reg_val);
2568
2569 /*
2570 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2571 */
2572 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2573 rx_ring = &ixgbe->rx_rings[i];
2574 ixgbe_setup_rx_ring(rx_ring);
2575 }
2576
2577 /*
2578 * The 82598 controller gives us the RNBC (Receive No Buffer
2579 * Count) register to determine the number of frames dropped
2580 * due to no available descriptors on the destination queue.
2581 * However, this register was removed starting with 82599 and
2582 * it was replaced with the RQSMR/QPRDC registers. The nice
2583 * thing about the new registers is that they allow you to map
2584 * groups of queues to specific stat registers. The bad thing
2585 * is there are only 16 slots in the stat registers, so this
2586 * won't work when we have 32 Rx groups. Instead, we map all
2587 * queues to the zero slot of the stat registers, giving us a
2588 * global counter at QPRDC[0] (with the equivalent semantics
2589 * of RNBC). Perhaps future controllers will have more slots
2590 * and we can implement per-group counters.
2591 */
2592 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2593 uint32_t index = ixgbe->rx_rings[i].hw_index;
2594 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), 0);
2595 }
2596
2597 /*
2598 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2599 * by four bytes if the packet has a VLAN field, so includes MTU,
2600 * ethernet header and frame check sequence.
2601 * Register is MAXFRS in 82599.
2602 */
2603 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD);
2604 reg_val &= ~IXGBE_MHADD_MFS_MASK;
2605 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header)
2606 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2607 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2608
2609 /*
2610 * Setup Jumbo Frame enable bit
2611 */
2612 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2613 if (ixgbe->default_mtu > ETHERMTU)
2614 reg_val |= IXGBE_HLREG0_JUMBOEN;
2615 else
2616 reg_val &= ~IXGBE_HLREG0_JUMBOEN;
2617 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2618
2619 /*
2620 * Setup RSC for multiple receive queues.
2621 */
2622 if (ixgbe->lro_enable) {
2623 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2624 /*
2625 * Make sure rx_buf_size * MAXDESC not greater
2626 * than 65535.
2627 * Intel recommends 4 for MAXDESC field value.
2628 */
2629 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2630 reg_val |= IXGBE_RSCCTL_RSCEN;
2631 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2632 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2633 else
2634 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2635 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2636 }
2637
2638 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2639 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2640 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2641
2642 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2643 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2644 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2645 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2646
2647 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2648 }
2649
2650 return (IXGBE_SUCCESS);
2651 }
2652
2653 static void
2654 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2655 {
2656 ixgbe_t *ixgbe = tx_ring->ixgbe;
2657 struct ixgbe_hw *hw = &ixgbe->hw;
2658 uint32_t size;
2659 uint32_t buf_low;
2660 uint32_t buf_high;
2661 uint32_t reg_val;
2662
2663 ASSERT(mutex_owned(&tx_ring->tx_lock));
2664 ASSERT(mutex_owned(&ixgbe->gen_lock));
2665
2666 /*
2667 * Initialize the length register
2668 */
2669 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2670 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2671
2672 /*
2673 * Initialize the base address registers
2674 */
2675 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2676 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2677 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2678 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2679
2680 /*
2681 * Setup head & tail pointers
2682 */
2683 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2684 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2685
2686 /*
2687 * Setup head write-back
2688 */
2689 if (ixgbe->tx_head_wb_enable) {
2690 /*
2691 * The memory of the head write-back is allocated using
2692 * the extra tbd beyond the tail of the tbd ring.
2693 */
2694 tx_ring->tbd_head_wb = (uint32_t *)
2695 ((uintptr_t)tx_ring->tbd_area.address + size);
2696 *tx_ring->tbd_head_wb = 0;
2697
2698 buf_low = (uint32_t)
2699 (tx_ring->tbd_area.dma_address + size);
2700 buf_high = (uint32_t)
2701 ((tx_ring->tbd_area.dma_address + size) >> 32);
2702
2703 /* Set the head write-back enable bit */
2704 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2705
2706 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2707 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2708
2709 /*
2710 * Turn off relaxed ordering for head write back or it will
2711 * cause problems with the tx recycling
2712 */
2713
2714 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2715 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2716 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2717 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2718 if (hw->mac.type == ixgbe_mac_82598EB) {
2719 IXGBE_WRITE_REG(hw,
2720 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2721 } else {
2722 IXGBE_WRITE_REG(hw,
2723 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2724 }
2725 } else {
2726 tx_ring->tbd_head_wb = NULL;
2727 }
2728
2729 tx_ring->tbd_head = 0;
2730 tx_ring->tbd_tail = 0;
2731 tx_ring->tbd_free = tx_ring->ring_size;
2732
2733 if (ixgbe->tx_ring_init == B_TRUE) {
2734 tx_ring->tcb_head = 0;
2735 tx_ring->tcb_tail = 0;
2736 tx_ring->tcb_free = tx_ring->free_list_size;
2737 }
2738
2739 /*
2740 * Initialize the s/w context structure
2741 */
2742 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2743 }
2744
2745 static void
2746 ixgbe_setup_tx(ixgbe_t *ixgbe)
2747 {
2748 struct ixgbe_hw *hw = &ixgbe->hw;
2749 ixgbe_tx_ring_t *tx_ring;
2750 uint32_t reg_val;
2751 int i;
2752
2753 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2754 tx_ring = &ixgbe->tx_rings[i];
2755 ixgbe_setup_tx_ring(tx_ring);
2756 }
2757
2758 /*
2759 * Setup the per-ring statistics mapping. We map all Tx queues
2760 * to slot 0 to stay consistent with Rx.
2761 */
2762 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2763 switch (hw->mac.type) {
2764 case ixgbe_mac_82598EB:
2765 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 0);
2766 break;
2767
2768 default:
2769 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 0);
2770 break;
2771 }
2772 }
2773
2774 /*
2775 * Enable CRC appending and TX padding (for short tx frames)
2776 */
2777 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2778 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2779 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2780
2781 /*
2782 * enable DMA for 82599, X540 and X550 parts
2783 */
2784 if (hw->mac.type == ixgbe_mac_82599EB ||
2785 hw->mac.type == ixgbe_mac_X540 ||
2786 hw->mac.type == ixgbe_mac_X550 ||
2787 hw->mac.type == ixgbe_mac_X550EM_x ||
2788 hw->mac.type == ixgbe_mac_X550EM_a) {
2789 /* DMATXCTL.TE must be set after all Tx config is complete */
2790 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2791 reg_val |= IXGBE_DMATXCTL_TE;
2792 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2793
2794 /* Disable arbiter to set MTQC */
2795 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2796 reg_val |= IXGBE_RTTDCS_ARBDIS;
2797 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2798 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2799 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2800 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2801 }
2802
2803 /*
2804 * Enabling tx queues ..
2805 * For 82599 must be done after DMATXCTL.TE is set
2806 */
2807 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2808 tx_ring = &ixgbe->tx_rings[i];
2809 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2810 reg_val |= IXGBE_TXDCTL_ENABLE;
2811 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2812 }
2813 }
2814
2815 /*
2816 * ixgbe_setup_rss - Setup receive-side scaling feature.
2817 */
2818 static void
2819 ixgbe_setup_rss(ixgbe_t *ixgbe)
2820 {
2821 struct ixgbe_hw *hw = &ixgbe->hw;
2822 uint32_t mrqc;
2823
2824 /*
2825 * Initialize RETA/ERETA table
2826 */
2827 ixgbe_setup_rss_table(ixgbe);
2828
2829 /*
2830 * Enable RSS & perform hash on these packet types
2831 */
2832 mrqc = IXGBE_MRQC_RSSEN |
2833 IXGBE_MRQC_RSS_FIELD_IPV4 |
2834 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2835 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2836 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2837 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2838 IXGBE_MRQC_RSS_FIELD_IPV6 |
2839 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2840 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2841 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2842 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2843 }
2844
2845 /*
2846 * ixgbe_setup_vmdq - Setup MAC classification feature
2847 */
2848 static void
2849 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2850 {
2851 struct ixgbe_hw *hw = &ixgbe->hw;
2852 uint32_t vmdctl, i, vtctl, vlnctl;
2853
2854 /*
2855 * Setup the VMDq Control register, enable VMDq based on
2856 * packet destination MAC address:
2857 */
2858 switch (hw->mac.type) {
2859 case ixgbe_mac_82598EB:
2860 /*
2861 * VMDq Enable = 1;
2862 * VMDq Filter = 0; MAC filtering
2863 * Default VMDq output index = 0;
2864 */
2865 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2866 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2867 break;
2868
2869 case ixgbe_mac_82599EB:
2870 case ixgbe_mac_X540:
2871 case ixgbe_mac_X550:
2872 case ixgbe_mac_X550EM_x:
2873 case ixgbe_mac_X550EM_a:
2874 /*
2875 * Enable VMDq-only.
2876 */
2877 vmdctl = IXGBE_MRQC_VMDQEN;
2878 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2879
2880 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2881 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2882 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2883 }
2884
2885 /*
2886 * Enable Virtualization and Replication.
2887 */
2888 vtctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2889 ixgbe->rx_def_group = vtctl & IXGBE_VT_CTL_POOL_MASK;
2890 vtctl |= IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2891 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2892
2893 /*
2894 * Enable VLAN filtering and switching (VFTA and VLVF).
2895 */
2896 vlnctl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
2897 vlnctl |= IXGBE_VLNCTRL_VFE;
2898 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctl);
2899 ixgbe->vlft_enabled = B_TRUE;
2900
2901 /*
2902 * Enable receiving packets to all VFs
2903 */
2904 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2905 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2906 break;
2907
2908 default:
2909 break;
2910 }
2911 }
2912
2913 /*
2914 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2915 */
2916 static void
2917 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2918 {
2919 struct ixgbe_hw *hw = &ixgbe->hw;
2920 uint32_t i, mrqc;
2921 uint32_t vtctl, vmdctl, vlnctl;
2922
2923 /*
2924 * Initialize RETA/ERETA table
2925 */
2926 ixgbe_setup_rss_table(ixgbe);
2927
2928 /*
2929 * Enable and setup RSS and VMDq
2930 */
2931 switch (hw->mac.type) {
2932 case ixgbe_mac_82598EB:
2933 /*
2934 * Enable RSS & Setup RSS Hash functions
2935 */
2936 mrqc = IXGBE_MRQC_RSSEN |
2937 IXGBE_MRQC_RSS_FIELD_IPV4 |
2938 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2939 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2940 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2941 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2942 IXGBE_MRQC_RSS_FIELD_IPV6 |
2943 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2944 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2945 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2946 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2947
2948 /*
2949 * Enable and Setup VMDq
2950 * VMDq Filter = 0; MAC filtering
2951 * Default VMDq output index = 0;
2952 */
2953 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2954 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2955 break;
2956
2957 case ixgbe_mac_82599EB:
2958 case ixgbe_mac_X540:
2959 case ixgbe_mac_X550:
2960 case ixgbe_mac_X550EM_x:
2961 case ixgbe_mac_X550EM_a:
2962 /*
2963 * Enable RSS & Setup RSS Hash functions
2964 */
2965 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2966 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2967 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2968 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2969 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2970 IXGBE_MRQC_RSS_FIELD_IPV6 |
2971 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2972 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2973 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2974
2975 /*
2976 * Enable VMDq+RSS.
2977 */
2978 if (ixgbe->num_rx_groups > 32) {
2979 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2980 } else {
2981 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2982 }
2983
2984 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2985
2986 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2987 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2988 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2989 }
2990 break;
2991
2992 default:
2993 break;
2994
2995 }
2996
2997 if (hw->mac.type == ixgbe_mac_82599EB ||
2998 hw->mac.type == ixgbe_mac_X540 ||
2999 hw->mac.type == ixgbe_mac_X550 ||
3000 hw->mac.type == ixgbe_mac_X550EM_x ||
3001 hw->mac.type == ixgbe_mac_X550EM_a) {
3002 /*
3003 * Enable Virtualization and Replication.
3004 */
3005 vtctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3006 ixgbe->rx_def_group = vtctl & IXGBE_VT_CTL_POOL_MASK;
3007 vtctl |= IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3008 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3009 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
3010
3011 /*
3012 * Enable VLAN filtering and switching (VFTA and VLVF).
3013 */
3014 vlnctl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3015 vlnctl |= IXGBE_VLNCTRL_VFE;
3016 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctl);
3017 ixgbe->vlft_enabled = B_TRUE;
3018
3019 /*
3020 * Enable receiving packets to all VFs
3021 */
3022 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
3023 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
3024 }
3025 }
3026
3027 /*
3028 * ixgbe_setup_rss_table - Setup RSS table
3029 */
3030 static void
3031 ixgbe_setup_rss_table(ixgbe_t *ixgbe)
3032 {
3033 struct ixgbe_hw *hw = &ixgbe->hw;
3034 uint32_t i, j;
3035 uint32_t random;
3036 uint32_t reta;
3037 uint32_t ring_per_group;
3038 uint32_t ring;
3039 uint32_t table_size;
3040 uint32_t index_mult;
3041 uint32_t rxcsum;
3042
3043 /*
3044 * Set multiplier for RETA setup and table size based on MAC type.
3045 * RETA table sizes vary by model:
3046 *
3047 * 82598, 82599, X540: 128 table entries.
3048 * X550: 512 table entries.
3049 */
3050 index_mult = 0x1;
3051 table_size = 128;
3052 switch (ixgbe->hw.mac.type) {
3053 case ixgbe_mac_82598EB:
3054 index_mult = 0x11;
3055 break;
3056 case ixgbe_mac_X550:
3057 case ixgbe_mac_X550EM_x:
3058 case ixgbe_mac_X550EM_a:
3059 table_size = 512;
3060 break;
3061 default:
3062 break;
3063 }
3064
3065 /*
3066 * Fill out RSS redirection table. The configuation of the indices is
3067 * hardware-dependent.
3068 *
3069 * 82598: 8 bits wide containing two 4 bit RSS indices
3070 * 82599, X540: 8 bits wide containing one 4 bit RSS index
3071 * X550: 8 bits wide containing one 6 bit RSS index
3072 */
3073 reta = 0;
3074 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3075
3076 for (i = 0, j = 0; i < table_size; i++, j++) {
3077 if (j == ring_per_group) j = 0;
3078
3079 /*
3080 * The low 8 bits are for hash value (n+0);
3081 * The next 8 bits are for hash value (n+1), etc.
3082 */
3083 ring = (j * index_mult);
3084 reta = reta >> 8;
3085 reta = reta | (((uint32_t)ring) << 24);
3086
3087 if ((i & 3) == 3) {
3088 /*
3089 * The first 128 table entries are programmed into the
3090 * RETA register, with any beyond that (eg; on X550)
3091 * into ERETA.
3092 */
3093 if (i < 128)
3094 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3095 else
3096 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3097 reta);
3098 reta = 0;
3099 }
3100 }
3101
3102 /*
3103 * Fill out hash function seeds with a random constant
3104 */
3105 for (i = 0; i < 10; i++) {
3106 (void) random_get_pseudo_bytes((uint8_t *)&random,
3107 sizeof (uint32_t));
3108 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
3109 }
3110
3111 /*
3112 * Disable Packet Checksum to enable RSS for multiple receive queues.
3113 * It is an adapter hardware limitation that Packet Checksum is
3114 * mutually exclusive with RSS.
3115 */
3116 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3117 rxcsum |= IXGBE_RXCSUM_PCSD;
3118 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3119 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3120 }
3121
3122 /*
3123 * ixgbe_init_unicst - Initialize the unicast addresses.
3124 */
3125 static void
3126 ixgbe_init_unicst(ixgbe_t *ixgbe)
3127 {
3128 struct ixgbe_hw *hw = &ixgbe->hw;
3129 uint8_t *mac_addr;
3130 int slot;
3131 /*
3132 * Here we should consider two situations:
3133 *
3134 * 1. Chipset is initialized at the first time,
3135 * Clear all the multiple unicast addresses.
3136 *
3137 * 2. Chipset is reset
3138 * Recover the multiple unicast addresses from the
3139 * software data structure to the RAR registers.
3140 */
3141 if (!ixgbe->unicst_init) {
3142 /*
3143 * Initialize the multiple unicast addresses
3144 */
3145 ixgbe->unicst_total = hw->mac.num_rar_entries;
3146 ixgbe->unicst_avail = ixgbe->unicst_total;
3147 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3148 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3149 bzero(mac_addr, ETHERADDRL);
3150 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3151 ixgbe->unicst_addr[slot].mac.set = 0;
3152 }
3153 ixgbe->unicst_init = B_TRUE;
3154 } else {
3155 /* Re-configure the RAR registers */
3156 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3157 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3158 if (ixgbe->unicst_addr[slot].mac.set == 1) {
3159 (void) ixgbe_set_rar(hw, slot, mac_addr,
3160 ixgbe->unicst_addr[slot].mac.group_index,
3161 IXGBE_RAH_AV);
3162 } else {
3163 bzero(mac_addr, ETHERADDRL);
3164 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3165 }
3166 }
3167 }
3168 }
3169
3170 /*
3171 * ixgbe_unicst_find - Find the slot for the specified unicast address
3172 */
3173 int
3174 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
3175 {
3176 int slot;
3177
3178 ASSERT(mutex_owned(&ixgbe->gen_lock));
3179
3180 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3181 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
3182 mac_addr, ETHERADDRL) == 0)
3183 return (slot);
3184 }
3185
3186 return (-1);
3187 }
3188
3189 /*
3190 * Restore the HW state to match the SW state during restart.
3191 */
3192 static int
3193 ixgbe_init_vlan(ixgbe_t *ixgbe)
3194 {
3195 /*
3196 * The device is starting for the first time; there is nothing
3197 * to do.
3198 */
3199 if (!ixgbe->vlft_init) {
3200 ixgbe->vlft_init = B_TRUE;
3201 return (IXGBE_SUCCESS);
3202 }
3203
3204 for (uint_t i = 0; i < ixgbe->num_rx_groups; i++) {
3205 int ret;
3206 boolean_t vlvf_bypass;
3207 ixgbe_rx_group_t *rxg = &ixgbe->rx_groups[i];
3208 struct ixgbe_hw *hw = &ixgbe->hw;
3209
3210 if (rxg->aupe) {
3211 uint32_t vml2flt;
3212
3213 vml2flt = IXGBE_READ_REG(hw, IXGBE_VMOLR(rxg->index));
3214 vml2flt |= IXGBE_VMOLR_AUPE;
3215 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(rxg->index), vml2flt);
3216 }
3217
3218 vlvf_bypass = (rxg->index == ixgbe->rx_def_group);
3219 for (ixgbe_vlan_t *vlp = list_head(&rxg->vlans); vlp != NULL;
3220 vlp = list_next(&rxg->vlans, vlp)) {
3221 ret = ixgbe_set_vfta(hw, vlp->ixvl_vid, rxg->index,
3222 B_TRUE, vlvf_bypass);
3223
3224 if (ret != IXGBE_SUCCESS) {
3225 ixgbe_error(ixgbe, "Failed to program VFTA"
3226 " for group %u, VID: %u, ret: %d.",
3227 rxg->index, vlp->ixvl_vid, ret);
3228 return (IXGBE_FAILURE);
3229 }
3230 }
3231 }
3232
3233 return (IXGBE_SUCCESS);
3234 }
3235
3236 /*
3237 * ixgbe_multicst_add - Add a multicst address.
3238 */
3239 int
3240 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3241 {
3242 ASSERT(mutex_owned(&ixgbe->gen_lock));
3243
3244 if ((multiaddr[0] & 01) == 0) {
3245 return (EINVAL);
3246 }
3247
3248 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
3249 return (ENOENT);
3250 }
3251
3252 bcopy(multiaddr,
3253 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
3254 ixgbe->mcast_count++;
3255
3256 /*
3257 * Update the multicast table in the hardware
3258 */
3259 ixgbe_setup_multicst(ixgbe);
3260
3261 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3262 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3263 return (EIO);
3264 }
3265
3266 return (0);
3267 }
3268
3269 /*
3270 * ixgbe_multicst_remove - Remove a multicst address.
3271 */
3272 int
3273 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3274 {
3275 int i;
3276
3277 ASSERT(mutex_owned(&ixgbe->gen_lock));
3278
3279 for (i = 0; i < ixgbe->mcast_count; i++) {
3280 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
3281 ETHERADDRL) == 0) {
3282 for (i++; i < ixgbe->mcast_count; i++) {
3283 ixgbe->mcast_table[i - 1] =
3284 ixgbe->mcast_table[i];
3285 }
3286 ixgbe->mcast_count--;
3287 break;
3288 }
3289 }
3290
3291 /*
3292 * Update the multicast table in the hardware
3293 */
3294 ixgbe_setup_multicst(ixgbe);
3295
3296 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3297 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3298 return (EIO);
3299 }
3300
3301 return (0);
3302 }
3303
3304 /*
3305 * ixgbe_setup_multicast - Setup multicast data structures.
3306 *
3307 * This routine initializes all of the multicast related structures
3308 * and save them in the hardware registers.
3309 */
3310 static void
3311 ixgbe_setup_multicst(ixgbe_t *ixgbe)
3312 {
3313 uint8_t *mc_addr_list;
3314 uint32_t mc_addr_count;
3315 struct ixgbe_hw *hw = &ixgbe->hw;
3316
3317 ASSERT(mutex_owned(&ixgbe->gen_lock));
3318
3319 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
3320
3321 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
3322 mc_addr_count = ixgbe->mcast_count;
3323
3324 /*
3325 * Update the multicast addresses to the MTA registers
3326 */
3327 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
3328 ixgbe_mc_table_itr, TRUE);
3329 }
3330
3331 /*
3332 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
3333 *
3334 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
3335 * Different chipsets may have different allowed configuration of vmdq and rss.
3336 */
3337 static void
3338 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
3339 {
3340 struct ixgbe_hw *hw = &ixgbe->hw;
3341 uint32_t ring_per_group;
3342
3343 switch (hw->mac.type) {
3344 case ixgbe_mac_82598EB:
3345 /*
3346 * 82598 supports the following combination:
3347 * vmdq no. x rss no.
3348 * [5..16] x 1
3349 * [1..4] x [1..16]
3350 * However 8 rss queue per pool (vmdq) is sufficient for
3351 * most cases.
3352 */
3353 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3354 if (ixgbe->num_rx_groups > 4) {
3355 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3356 } else {
3357 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3358 min(8, ring_per_group);
3359 }
3360
3361 break;
3362
3363 case ixgbe_mac_82599EB:
3364 case ixgbe_mac_X540:
3365 case ixgbe_mac_X550:
3366 case ixgbe_mac_X550EM_x:
3367 case ixgbe_mac_X550EM_a:
3368 /*
3369 * 82599 supports the following combination:
3370 * vmdq no. x rss no.
3371 * [33..64] x [1..2]
3372 * [2..32] x [1..4]
3373 * 1 x [1..16]
3374 * However 8 rss queue per pool (vmdq) is sufficient for
3375 * most cases.
3376 *
3377 * For now, treat X540 and X550 like the 82599.
3378 */
3379 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3380 if (ixgbe->num_rx_groups == 1) {
3381 ixgbe->num_rx_rings = min(8, ring_per_group);
3382 } else if (ixgbe->num_rx_groups <= 32) {
3383 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3384 min(4, ring_per_group);
3385 } else if (ixgbe->num_rx_groups <= 64) {
3386 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3387 min(2, ring_per_group);
3388 }
3389 break;
3390
3391 default:
3392 break;
3393 }
3394
3395 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3396
3397 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3398 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3399 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3400 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3401 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3402 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3403 } else {
3404 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3405 }
3406
3407 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3408 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3409 }
3410
3411 /*
3412 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3413 *
3414 * This routine gets user-configured values out of the configuration
3415 * file ixgbe.conf.
3416 *
3417 * For each configurable value, there is a minimum, a maximum, and a
3418 * default.
3419 * If user does not configure a value, use the default.
3420 * If user configures below the minimum, use the minumum.
3421 * If user configures above the maximum, use the maxumum.
3422 */
3423 static void
3424 ixgbe_get_conf(ixgbe_t *ixgbe)
3425 {
3426 struct ixgbe_hw *hw = &ixgbe->hw;
3427 uint32_t flow_control;
3428
3429 /*
3430 * ixgbe driver supports the following user configurations:
3431 *
3432 * Jumbo frame configuration:
3433 * default_mtu
3434 *
3435 * Ethernet flow control configuration:
3436 * flow_control
3437 *
3438 * Multiple rings configurations:
3439 * tx_queue_number
3440 * tx_ring_size
3441 * rx_queue_number
3442 * rx_ring_size
3443 *
3444 * Call ixgbe_get_prop() to get the value for a specific
3445 * configuration parameter.
3446 */
3447
3448 /*
3449 * Jumbo frame configuration - max_frame_size controls host buffer
3450 * allocation, so includes MTU, ethernet header, vlan tag and
3451 * frame check sequence.
3452 */
3453 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3454 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3455
3456 ixgbe->max_frame_size = ixgbe->default_mtu +
3457 sizeof (struct ether_vlan_header) + ETHERFCSL;
3458
3459 /*
3460 * Ethernet flow control configuration
3461 */
3462 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3463 ixgbe_fc_none, 3, ixgbe_fc_none);
3464 if (flow_control == 3)
3465 flow_control = ixgbe_fc_default;
3466
3467 /*
3468 * fc.requested mode is what the user requests. After autoneg,
3469 * fc.current_mode will be the flow_control mode that was negotiated.
3470 */
3471 hw->fc.requested_mode = flow_control;
3472
3473 /*
3474 * Multiple rings configurations
3475 */
3476 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3477 ixgbe->capab->min_tx_que_num,
3478 ixgbe->capab->max_tx_que_num,
3479 ixgbe->capab->def_tx_que_num);
3480 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3481 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3482
3483 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3484 ixgbe->capab->min_rx_que_num,
3485 ixgbe->capab->max_rx_que_num,
3486 ixgbe->capab->def_rx_que_num);
3487 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3488 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3489
3490 /*
3491 * Multiple groups configuration
3492 */
3493 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3494 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3495 ixgbe->capab->def_rx_grp_num);
3496
3497 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3498 0, 1, DEFAULT_MR_ENABLE);
3499
3500 if (ixgbe->mr_enable == B_FALSE) {
3501 ixgbe->num_tx_rings = 1;
3502 ixgbe->num_rx_rings = 1;
3503 ixgbe->num_rx_groups = 1;
3504 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3505 } else {
3506 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3507 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3508 /*
3509 * The combination of num_rx_rings and num_rx_groups
3510 * may be not supported by h/w. We need to adjust
3511 * them to appropriate values.
3512 */
3513 ixgbe_setup_vmdq_rss_conf(ixgbe);
3514 }
3515
3516 /*
3517 * Tunable used to force an interrupt type. The only use is
3518 * for testing of the lesser interrupt types.
3519 * 0 = don't force interrupt type
3520 * 1 = force interrupt type MSI-X
3521 * 2 = force interrupt type MSI
3522 * 3 = force interrupt type Legacy
3523 */
3524 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3525 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3526
3527 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3528 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3529 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3530 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3531 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3532 0, 1, DEFAULT_LSO_ENABLE);
3533 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3534 0, 1, DEFAULT_LRO_ENABLE);
3535 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3536 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3537 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3538 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3539
3540 /* Head Write Back not recommended for 82599, X540 and X550 */
3541 if (hw->mac.type == ixgbe_mac_82599EB ||
3542 hw->mac.type == ixgbe_mac_X540 ||
3543 hw->mac.type == ixgbe_mac_X550 ||
3544 hw->mac.type == ixgbe_mac_X550EM_x ||
3545 hw->mac.type == ixgbe_mac_X550EM_a) {
3546 ixgbe->tx_head_wb_enable = B_FALSE;
3547 }
3548
3549 /*
3550 * ixgbe LSO needs the tx h/w checksum support.
3551 * LSO will be disabled if tx h/w checksum is not
3552 * enabled.
3553 */
3554 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3555 ixgbe->lso_enable = B_FALSE;
3556 }
3557
3558 /*
3559 * ixgbe LRO needs the rx h/w checksum support.
3560 * LRO will be disabled if rx h/w checksum is not
3561 * enabled.
3562 */
3563 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3564 ixgbe->lro_enable = B_FALSE;
3565 }
3566
3567 /*
3568 * ixgbe LRO only supported by 82599, X540 and X550
3569 */
3570 if (hw->mac.type == ixgbe_mac_82598EB) {
3571 ixgbe->lro_enable = B_FALSE;
3572 }
3573 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3574 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3575 DEFAULT_TX_COPY_THRESHOLD);
3576 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3577 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3578 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3579 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3580 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3581 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3582 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3583 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3584 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3585
3586 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3587 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3588 DEFAULT_RX_COPY_THRESHOLD);
3589 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3590 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3591 DEFAULT_RX_LIMIT_PER_INTR);
3592
3593 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3594 ixgbe->capab->min_intr_throttle,
3595 ixgbe->capab->max_intr_throttle,
3596 ixgbe->capab->def_intr_throttle);
3597 /*
3598 * 82599, X540 and X550 require the interrupt throttling rate is
3599 * a multiple of 8. This is enforced by the register definiton.
3600 */
3601 if (hw->mac.type == ixgbe_mac_82599EB ||
3602 hw->mac.type == ixgbe_mac_X540 ||
3603 hw->mac.type == ixgbe_mac_X550 ||
3604 hw->mac.type == ixgbe_mac_X550EM_x ||
3605 hw->mac.type == ixgbe_mac_X550EM_a)
3606 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3607
3608 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3609 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3610 }
3611
3612 static void
3613 ixgbe_init_params(ixgbe_t *ixgbe)
3614 {
3615 struct ixgbe_hw *hw = &ixgbe->hw;
3616 ixgbe_link_speed speeds_supported = 0;
3617 boolean_t negotiate;
3618
3619 /*
3620 * Get a list of speeds the adapter supports. If the hw struct hasn't
3621 * been populated with this information yet, retrieve it from the
3622 * adapter and save it to our own variable.
3623 *
3624 * On certain adapters, such as ones which use SFPs, the contents of
3625 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not
3626 * updated, so we must rely on calling ixgbe_get_link_capabilities()
3627 * in order to ascertain the speeds which we are capable of supporting,
3628 * and in the case of SFP-equipped adapters, which speed we are
3629 * advertising. If ixgbe_get_link_capabilities() fails for some reason,
3630 * we'll go with a default list of speeds as a last resort.
3631 */
3632 speeds_supported = hw->phy.speeds_supported;
3633
3634 if (speeds_supported == 0) {
3635 if (ixgbe_get_link_capabilities(hw, &speeds_supported,
3636 &negotiate) != IXGBE_SUCCESS) {
3637 if (hw->mac.type == ixgbe_mac_82598EB) {
3638 speeds_supported =
3639 IXGBE_LINK_SPEED_82598_AUTONEG;
3640 } else {
3641 speeds_supported =
3642 IXGBE_LINK_SPEED_82599_AUTONEG;
3643 }
3644 }
3645 }
3646 ixgbe->speeds_supported = speeds_supported;
3647
3648 /*
3649 * By default, all supported speeds are enabled and advertised.
3650 */
3651 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) {
3652 ixgbe->param_en_10000fdx_cap = 1;
3653 ixgbe->param_adv_10000fdx_cap = 1;
3654 } else {
3655 ixgbe->param_en_10000fdx_cap = 0;
3656 ixgbe->param_adv_10000fdx_cap = 0;
3657 }
3658
3659 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) {
3660 ixgbe->param_en_5000fdx_cap = 1;
3661 ixgbe->param_adv_5000fdx_cap = 1;
3662 } else {
3663 ixgbe->param_en_5000fdx_cap = 0;
3664 ixgbe->param_adv_5000fdx_cap = 0;
3665 }
3666
3667 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) {
3668 ixgbe->param_en_2500fdx_cap = 1;
3669 ixgbe->param_adv_2500fdx_cap = 1;
3670 } else {
3671 ixgbe->param_en_2500fdx_cap = 0;
3672 ixgbe->param_adv_2500fdx_cap = 0;
3673 }
3674
3675 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) {
3676 ixgbe->param_en_1000fdx_cap = 1;
3677 ixgbe->param_adv_1000fdx_cap = 1;
3678 } else {
3679 ixgbe->param_en_1000fdx_cap = 0;
3680 ixgbe->param_adv_1000fdx_cap = 0;
3681 }
3682
3683 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) {
3684 ixgbe->param_en_100fdx_cap = 1;
3685 ixgbe->param_adv_100fdx_cap = 1;
3686 } else {
3687 ixgbe->param_en_100fdx_cap = 0;
3688 ixgbe->param_adv_100fdx_cap = 0;
3689 }
3690
3691 ixgbe->param_pause_cap = 1;
3692 ixgbe->param_asym_pause_cap = 1;
3693 ixgbe->param_rem_fault = 0;
3694
3695 ixgbe->param_adv_autoneg_cap = 1;
3696 ixgbe->param_adv_pause_cap = 1;
3697 ixgbe->param_adv_asym_pause_cap = 1;
3698 ixgbe->param_adv_rem_fault = 0;
3699
3700 ixgbe->param_lp_10000fdx_cap = 0;
3701 ixgbe->param_lp_5000fdx_cap = 0;
3702 ixgbe->param_lp_2500fdx_cap = 0;
3703 ixgbe->param_lp_1000fdx_cap = 0;
3704 ixgbe->param_lp_100fdx_cap = 0;
3705 ixgbe->param_lp_autoneg_cap = 0;
3706 ixgbe->param_lp_pause_cap = 0;
3707 ixgbe->param_lp_asym_pause_cap = 0;
3708 ixgbe->param_lp_rem_fault = 0;
3709 }
3710
3711 /*
3712 * ixgbe_get_prop - Get a property value out of the configuration file
3713 * ixgbe.conf.
3714 *
3715 * Caller provides the name of the property, a default value, a minimum
3716 * value, and a maximum value.
3717 *
3718 * Return configured value of the property, with default, minimum and
3719 * maximum properly applied.
3720 */
3721 static int
3722 ixgbe_get_prop(ixgbe_t *ixgbe,
3723 char *propname, /* name of the property */
3724 int minval, /* minimum acceptable value */
3725 int maxval, /* maximim acceptable value */
3726 int defval) /* default value */
3727 {
3728 int value;
3729
3730 /*
3731 * Call ddi_prop_get_int() to read the conf settings
3732 */
3733 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3734 DDI_PROP_DONTPASS, propname, defval);
3735 if (value > maxval)
3736 value = maxval;
3737
3738 if (value < minval)
3739 value = minval;
3740
3741 return (value);
3742 }
3743
3744 /*
3745 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3746 */
3747 int
3748 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3749 {
3750 struct ixgbe_hw *hw = &ixgbe->hw;
3751 ixgbe_link_speed advertised = 0;
3752
3753 /*
3754 * Assemble a list of enabled speeds to auto-negotiate with.
3755 */
3756 if (ixgbe->param_en_10000fdx_cap == 1)
3757 advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3758
3759 if (ixgbe->param_en_5000fdx_cap == 1)
3760 advertised |= IXGBE_LINK_SPEED_5GB_FULL;
3761
3762 if (ixgbe->param_en_2500fdx_cap == 1)
3763 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL;
3764
3765 if (ixgbe->param_en_1000fdx_cap == 1)
3766 advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3767
3768 if (ixgbe->param_en_100fdx_cap == 1)
3769 advertised |= IXGBE_LINK_SPEED_100_FULL;
3770
3771 /*
3772 * As a last resort, autoneg with a default list of speeds.
3773 */
3774 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) {
3775 ixgbe_notice(ixgbe, "Invalid link settings. Setting link "
3776 "to autonegotiate with full capabilities.");
3777
3778 if (hw->mac.type == ixgbe_mac_82598EB)
3779 advertised = IXGBE_LINK_SPEED_82598_AUTONEG;
3780 else
3781 advertised = IXGBE_LINK_SPEED_82599_AUTONEG;
3782 }
3783
3784 if (setup_hw) {
3785 if (ixgbe_setup_link(&ixgbe->hw, advertised,
3786 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3787 ixgbe_notice(ixgbe, "Setup link failed on this "
3788 "device.");
3789 return (IXGBE_FAILURE);
3790 }
3791 }
3792
3793 return (IXGBE_SUCCESS);
3794 }
3795
3796 /*
3797 * ixgbe_driver_link_check - Link status processing.
3798 *
3799 * This function can be called in both kernel context and interrupt context
3800 */
3801 static void
3802 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3803 {
3804 struct ixgbe_hw *hw = &ixgbe->hw;
3805 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3806 boolean_t link_up = B_FALSE;
3807 boolean_t link_changed = B_FALSE;
3808
3809 ASSERT(mutex_owned(&ixgbe->gen_lock));
3810
3811 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3812 if (link_up) {
3813 ixgbe->link_check_complete = B_TRUE;
3814
3815 /* Link is up, enable flow control settings */
3816 (void) ixgbe_fc_enable(hw);
3817
3818 /*
3819 * The Link is up, check whether it was marked as down earlier
3820 */
3821 if (ixgbe->link_state != LINK_STATE_UP) {
3822 switch (speed) {
3823 case IXGBE_LINK_SPEED_10GB_FULL:
3824 ixgbe->link_speed = SPEED_10GB;
3825 break;
3826 case IXGBE_LINK_SPEED_5GB_FULL:
3827 ixgbe->link_speed = SPEED_5GB;
3828 break;
3829 case IXGBE_LINK_SPEED_2_5GB_FULL:
3830 ixgbe->link_speed = SPEED_2_5GB;
3831 break;
3832 case IXGBE_LINK_SPEED_1GB_FULL:
3833 ixgbe->link_speed = SPEED_1GB;
3834 break;
3835 case IXGBE_LINK_SPEED_100_FULL:
3836 ixgbe->link_speed = SPEED_100;
3837 }
3838 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3839 ixgbe->link_state = LINK_STATE_UP;
3840 link_changed = B_TRUE;
3841 }
3842 } else {
3843 if (ixgbe->link_check_complete == B_TRUE ||
3844 (ixgbe->link_check_complete == B_FALSE &&
3845 gethrtime() >= ixgbe->link_check_hrtime)) {
3846 /*
3847 * The link is really down
3848 */
3849 ixgbe->link_check_complete = B_TRUE;
3850
3851 if (ixgbe->link_state != LINK_STATE_DOWN) {
3852 ixgbe->link_speed = 0;
3853 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3854 ixgbe->link_state = LINK_STATE_DOWN;
3855 link_changed = B_TRUE;
3856 }
3857 }
3858 }
3859
3860 /*
3861 * If we are in an interrupt context, need to re-enable the
3862 * interrupt, which was automasked
3863 */
3864 if (servicing_interrupt() != 0) {
3865 ixgbe->eims |= IXGBE_EICR_LSC;
3866 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3867 }
3868
3869 if (link_changed) {
3870 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3871 }
3872 }
3873
3874 /*
3875 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3876 */
3877 static void
3878 ixgbe_sfp_check(void *arg)
3879 {
3880 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3881 uint32_t eicr = ixgbe->eicr;
3882 struct ixgbe_hw *hw = &ixgbe->hw;
3883
3884 mutex_enter(&ixgbe->gen_lock);
3885 (void) hw->phy.ops.identify_sfp(hw);
3886 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3887 /* clear the interrupt */
3888 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3889
3890 /* if link up, do multispeed fiber setup */
3891 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3892 B_TRUE);
3893 ixgbe_driver_link_check(ixgbe);
3894 ixgbe_get_hw_state(ixgbe);
3895 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3896 /* clear the interrupt */
3897 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3898
3899 /* if link up, do sfp module setup */
3900 (void) hw->mac.ops.setup_sfp(hw);
3901
3902 /* do multispeed fiber setup */
3903 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3904 B_TRUE);
3905 ixgbe_driver_link_check(ixgbe);
3906 ixgbe_get_hw_state(ixgbe);
3907 }
3908 mutex_exit(&ixgbe->gen_lock);
3909
3910 /*
3911 * We need to fully re-check the link later.
3912 */
3913 ixgbe->link_check_complete = B_FALSE;
3914 ixgbe->link_check_hrtime = gethrtime() +
3915 (IXGBE_LINK_UP_TIME * 100000000ULL);
3916 }
3917
3918 /*
3919 * ixgbe_overtemp_check - overtemp module processing done in taskq
3920 *
3921 * This routine will only be called on adapters with temperature sensor.
3922 * The indication of over-temperature can be either SDP0 interrupt or the link
3923 * status change interrupt.
3924 */
3925 static void
3926 ixgbe_overtemp_check(void *arg)
3927 {
3928 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3929 struct ixgbe_hw *hw = &ixgbe->hw;
3930 uint32_t eicr = ixgbe->eicr;
3931 ixgbe_link_speed speed;
3932 boolean_t link_up;
3933
3934 mutex_enter(&ixgbe->gen_lock);
3935
3936 /* make sure we know current state of link */
3937 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3938
3939 /* check over-temp condition */
3940 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3941 (eicr & IXGBE_EICR_LSC)) {
3942 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3943 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3944
3945 /*
3946 * Disable the adapter interrupts
3947 */
3948 ixgbe_disable_adapter_interrupts(ixgbe);
3949
3950 /*
3951 * Disable Rx/Tx units
3952 */
3953 (void) ixgbe_stop_adapter(hw);
3954
3955 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3956 ixgbe_error(ixgbe,
3957 "Problem: Network adapter has been stopped "
3958 "because it has overheated");
3959 ixgbe_error(ixgbe,
3960 "Action: Restart the computer. "
3961 "If the problem persists, power off the system "
3962 "and replace the adapter");
3963 }
3964 }
3965
3966 /* write to clear the interrupt */
3967 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3968
3969 mutex_exit(&ixgbe->gen_lock);
3970 }
3971
3972 /*
3973 * ixgbe_phy_check - taskq to process interrupts from an external PHY
3974 *
3975 * This routine will only be called on adapters with external PHYs
3976 * (such as X550) that may be trying to raise our attention to some event.
3977 * Currently, this is limited to claiming PHY overtemperature and link status
3978 * change (LSC) events, however this may expand to include other things in
3979 * future adapters.
3980 */
3981 static void
3982 ixgbe_phy_check(void *arg)
3983 {
3984 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3985 struct ixgbe_hw *hw = &ixgbe->hw;
3986 int rv;
3987
3988 mutex_enter(&ixgbe->gen_lock);
3989
3990 /*
3991 * X550 baseT PHY overtemp and LSC events are handled here.
3992 *
3993 * If an overtemp event occurs, it will be reflected in the
3994 * return value of phy.ops.handle_lasi() and the common code will
3995 * automatically power off the baseT PHY. This is our cue to trigger
3996 * an FMA event.
3997 *
3998 * If a link status change event occurs, phy.ops.handle_lasi() will
3999 * automatically initiate a link setup between the integrated KR PHY
4000 * and the external X557 PHY to ensure that the link speed between
4001 * them matches the link speed of the baseT link.
4002 */
4003 rv = ixgbe_handle_lasi(hw);
4004
4005 if (rv == IXGBE_ERR_OVERTEMP) {
4006 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4007
4008 /*
4009 * Disable the adapter interrupts
4010 */
4011 ixgbe_disable_adapter_interrupts(ixgbe);
4012
4013 /*
4014 * Disable Rx/Tx units
4015 */
4016 (void) ixgbe_stop_adapter(hw);
4017
4018 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4019 ixgbe_error(ixgbe,
4020 "Problem: Network adapter has been stopped due to a "
4021 "overtemperature event being detected.");
4022 ixgbe_error(ixgbe,
4023 "Action: Shut down or restart the computer. If the issue "
4024 "persists, please take action in accordance with the "
4025 "recommendations from your system vendor.");
4026 }
4027
4028 mutex_exit(&ixgbe->gen_lock);
4029 }
4030
4031 /*
4032 * ixgbe_link_timer - timer for link status detection
4033 */
4034 static void
4035 ixgbe_link_timer(void *arg)
4036 {
4037 ixgbe_t *ixgbe = (ixgbe_t *)arg;
4038
4039 mutex_enter(&ixgbe->gen_lock);
4040 ixgbe_driver_link_check(ixgbe);
4041 mutex_exit(&ixgbe->gen_lock);
4042 }
4043
4044 /*
4045 * ixgbe_local_timer - Driver watchdog function.
4046 *
4047 * This function will handle the transmit stall check and other routines.
4048 */
4049 static void
4050 ixgbe_local_timer(void *arg)
4051 {
4052 ixgbe_t *ixgbe = (ixgbe_t *)arg;
4053
4054 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
4055 goto out;
4056
4057 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
4058 ixgbe->reset_count++;
4059 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
4060 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
4061 goto out;
4062 }
4063
4064 if (ixgbe_stall_check(ixgbe)) {
4065 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
4066 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4067
4068 ixgbe->reset_count++;
4069 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
4070 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
4071 }
4072
4073 out:
4074 ixgbe_restart_watchdog_timer(ixgbe);
4075 }
4076
4077 /*
4078 * ixgbe_stall_check - Check for transmit stall.
4079 *
4080 * This function checks if the adapter is stalled (in transmit).
4081 *
4082 * It is called each time the watchdog timeout is invoked.
4083 * If the transmit descriptor reclaim continuously fails,
4084 * the watchdog value will increment by 1. If the watchdog
4085 * value exceeds the threshold, the ixgbe is assumed to
4086 * have stalled and need to be reset.
4087 */
4088 static boolean_t
4089 ixgbe_stall_check(ixgbe_t *ixgbe)
4090 {
4091 ixgbe_tx_ring_t *tx_ring;
4092 boolean_t result;
4093 int i;
4094
4095 if (ixgbe->link_state != LINK_STATE_UP)
4096 return (B_FALSE);
4097
4098 /*
4099 * If any tx ring is stalled, we'll reset the chipset
4100 */
4101 result = B_FALSE;
4102 for (i = 0; i < ixgbe->num_tx_rings; i++) {
4103 tx_ring = &ixgbe->tx_rings[i];
4104 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
4105 tx_ring->tx_recycle(tx_ring);
4106 }
4107
4108 if (tx_ring->recycle_fail > 0)
4109 tx_ring->stall_watchdog++;
4110 else
4111 tx_ring->stall_watchdog = 0;
4112
4113 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
4114 result = B_TRUE;
4115 break;
4116 }
4117 }
4118
4119 if (result) {
4120 tx_ring->stall_watchdog = 0;
4121 tx_ring->recycle_fail = 0;
4122 }
4123
4124 return (result);
4125 }
4126
4127
4128 /*
4129 * is_valid_mac_addr - Check if the mac address is valid.
4130 */
4131 static boolean_t
4132 is_valid_mac_addr(uint8_t *mac_addr)
4133 {
4134 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
4135 const uint8_t addr_test2[6] =
4136 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
4137
4138 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
4139 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
4140 return (B_FALSE);
4141
4142 return (B_TRUE);
4143 }
4144
4145 static boolean_t
4146 ixgbe_find_mac_address(ixgbe_t *ixgbe)
4147 {
4148 #ifdef __sparc
4149 struct ixgbe_hw *hw = &ixgbe->hw;
4150 uchar_t *bytes;
4151 struct ether_addr sysaddr;
4152 uint_t nelts;
4153 int err;
4154 boolean_t found = B_FALSE;
4155
4156 /*
4157 * The "vendor's factory-set address" may already have
4158 * been extracted from the chip, but if the property
4159 * "local-mac-address" is set we use that instead.
4160 *
4161 * We check whether it looks like an array of 6
4162 * bytes (which it should, if OBP set it). If we can't
4163 * make sense of it this way, we'll ignore it.
4164 */
4165 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4166 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
4167 if (err == DDI_PROP_SUCCESS) {
4168 if (nelts == ETHERADDRL) {
4169 while (nelts--)
4170 hw->mac.addr[nelts] = bytes[nelts];
4171 found = B_TRUE;
4172 }
4173 ddi_prop_free(bytes);
4174 }
4175
4176 /*
4177 * Look up the OBP property "local-mac-address?". If the user has set
4178 * 'local-mac-address? = false', use "the system address" instead.
4179 */
4180 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
4181 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
4182 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
4183 if (localetheraddr(NULL, &sysaddr) != 0) {
4184 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
4185 found = B_TRUE;
4186 }
4187 }
4188 ddi_prop_free(bytes);
4189 }
4190
4191 /*
4192 * Finally(!), if there's a valid "mac-address" property (created
4193 * if we netbooted from this interface), we must use this instead
4194 * of any of the above to ensure that the NFS/install server doesn't
4195 * get confused by the address changing as illumos takes over!
4196 */
4197 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4198 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
4199 if (err == DDI_PROP_SUCCESS) {
4200 if (nelts == ETHERADDRL) {
4201 while (nelts--)
4202 hw->mac.addr[nelts] = bytes[nelts];
4203 found = B_TRUE;
4204 }
4205 ddi_prop_free(bytes);
4206 }
4207
4208 if (found) {
4209 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
4210 return (B_TRUE);
4211 }
4212 #else
4213 _NOTE(ARGUNUSED(ixgbe));
4214 #endif
4215
4216 return (B_TRUE);
4217 }
4218
4219 #pragma inline(ixgbe_arm_watchdog_timer)
4220 static void
4221 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
4222 {
4223 /*
4224 * Fire a watchdog timer
4225 */
4226 ixgbe->watchdog_tid =
4227 timeout(ixgbe_local_timer,
4228 (void *)ixgbe, 1 * drv_usectohz(1000000));
4229
4230 }
4231
4232 /*
4233 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
4234 */
4235 void
4236 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
4237 {
4238 mutex_enter(&ixgbe->watchdog_lock);
4239
4240 if (!ixgbe->watchdog_enable) {
4241 ixgbe->watchdog_enable = B_TRUE;
4242 ixgbe->watchdog_start = B_TRUE;
4243 ixgbe_arm_watchdog_timer(ixgbe);
4244 }
4245
4246 mutex_exit(&ixgbe->watchdog_lock);
4247 }
4248
4249 /*
4250 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
4251 */
4252 void
4253 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
4254 {
4255 timeout_id_t tid;
4256
4257 mutex_enter(&ixgbe->watchdog_lock);
4258
4259 ixgbe->watchdog_enable = B_FALSE;
4260 ixgbe->watchdog_start = B_FALSE;
4261 tid = ixgbe->watchdog_tid;
4262 ixgbe->watchdog_tid = 0;
4263
4264 mutex_exit(&ixgbe->watchdog_lock);
4265
4266 if (tid != 0)
4267 (void) untimeout(tid);
4268 }
4269
4270 /*
4271 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
4272 */
4273 void
4274 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
4275 {
4276 mutex_enter(&ixgbe->watchdog_lock);
4277
4278 if (ixgbe->watchdog_enable) {
4279 if (!ixgbe->watchdog_start) {
4280 ixgbe->watchdog_start = B_TRUE;
4281 ixgbe_arm_watchdog_timer(ixgbe);
4282 }
4283 }
4284
4285 mutex_exit(&ixgbe->watchdog_lock);
4286 }
4287
4288 /*
4289 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
4290 */
4291 static void
4292 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
4293 {
4294 mutex_enter(&ixgbe->watchdog_lock);
4295
4296 if (ixgbe->watchdog_start)
4297 ixgbe_arm_watchdog_timer(ixgbe);
4298
4299 mutex_exit(&ixgbe->watchdog_lock);
4300 }
4301
4302 /*
4303 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
4304 */
4305 void
4306 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
4307 {
4308 timeout_id_t tid;
4309
4310 mutex_enter(&ixgbe->watchdog_lock);
4311
4312 ixgbe->watchdog_start = B_FALSE;
4313 tid = ixgbe->watchdog_tid;
4314 ixgbe->watchdog_tid = 0;
4315
4316 mutex_exit(&ixgbe->watchdog_lock);
4317
4318 if (tid != 0)
4319 (void) untimeout(tid);
4320 }
4321
4322 /*
4323 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
4324 */
4325 static void
4326 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
4327 {
4328 struct ixgbe_hw *hw = &ixgbe->hw;
4329
4330 /*
4331 * mask all interrupts off
4332 */
4333 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
4334
4335 /*
4336 * for MSI-X, also disable autoclear
4337 */
4338 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4339 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
4340 }
4341
4342 IXGBE_WRITE_FLUSH(hw);
4343 }
4344
4345 /*
4346 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
4347 */
4348 static void
4349 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
4350 {
4351 struct ixgbe_hw *hw = &ixgbe->hw;
4352 uint32_t eiac, eiam;
4353 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4354
4355 /* interrupt types to enable */
4356 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
4357 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
4358 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
4359
4360 /* enable automask on "other" causes that this adapter can generate */
4361 eiam = ixgbe->capab->other_intr;
4362
4363 /*
4364 * msi-x mode
4365 */
4366 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4367 /* enable autoclear but not on bits 29:20 */
4368 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
4369
4370 /* general purpose interrupt enable */
4371 gpie |= (IXGBE_GPIE_MSIX_MODE
4372 | IXGBE_GPIE_PBA_SUPPORT
4373 | IXGBE_GPIE_OCD
4374 | IXGBE_GPIE_EIAME);
4375 /*
4376 * non-msi-x mode
4377 */
4378 } else {
4379
4380 /* disable autoclear, leave gpie at default */
4381 eiac = 0;
4382
4383 /*
4384 * General purpose interrupt enable.
4385 * For 82599, X540 and X550, extended interrupt
4386 * automask enable only in MSI or MSI-X mode
4387 */
4388 if ((hw->mac.type == ixgbe_mac_82598EB) ||
4389 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
4390 gpie |= IXGBE_GPIE_EIAME;
4391 }
4392 }
4393
4394 /* Enable specific "other" interrupt types */
4395 switch (hw->mac.type) {
4396 case ixgbe_mac_82598EB:
4397 gpie |= ixgbe->capab->other_gpie;
4398 break;
4399
4400 case ixgbe_mac_82599EB:
4401 case ixgbe_mac_X540:
4402 case ixgbe_mac_X550:
4403 case ixgbe_mac_X550EM_x:
4404 case ixgbe_mac_X550EM_a:
4405 gpie |= ixgbe->capab->other_gpie;
4406
4407 /* Enable RSC Delay 8us when LRO enabled */
4408 if (ixgbe->lro_enable) {
4409 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
4410 }
4411 break;
4412
4413 default:
4414 break;
4415 }
4416
4417 /* write to interrupt control registers */
4418 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4419 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
4420 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
4421 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4422 IXGBE_WRITE_FLUSH(hw);
4423 }
4424
4425 /*
4426 * ixgbe_loopback_ioctl - Loopback support.
4427 */
4428 enum ioc_reply
4429 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
4430 {
4431 lb_info_sz_t *lbsp;
4432 lb_property_t *lbpp;
4433 uint32_t *lbmp;
4434 uint32_t size;
4435 uint32_t value;
4436
4437 if (mp->b_cont == NULL)
4438 return (IOC_INVAL);
4439
4440 switch (iocp->ioc_cmd) {
4441 default:
4442 return (IOC_INVAL);
4443
4444 case LB_GET_INFO_SIZE:
4445 size = sizeof (lb_info_sz_t);
4446 if (iocp->ioc_count != size)
4447 return (IOC_INVAL);
4448
4449 value = sizeof (lb_normal);
4450 value += sizeof (lb_mac);
4451 value += sizeof (lb_external);
4452
4453 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
4454 *lbsp = value;
4455 break;
4456
4457 case LB_GET_INFO:
4458 value = sizeof (lb_normal);
4459 value += sizeof (lb_mac);
4460 value += sizeof (lb_external);
4461
4462 size = value;
4463 if (iocp->ioc_count != size)
4464 return (IOC_INVAL);
4465
4466 value = 0;
4467 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
4468
4469 lbpp[value++] = lb_normal;
4470 lbpp[value++] = lb_mac;
4471 lbpp[value++] = lb_external;
4472 break;
4473
4474 case LB_GET_MODE:
4475 size = sizeof (uint32_t);
4476 if (iocp->ioc_count != size)
4477 return (IOC_INVAL);
4478
4479 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4480 *lbmp = ixgbe->loopback_mode;
4481 break;
4482
4483 case LB_SET_MODE:
4484 size = 0;
4485 if (iocp->ioc_count != sizeof (uint32_t))
4486 return (IOC_INVAL);
4487
4488 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4489 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
4490 return (IOC_INVAL);
4491 break;
4492 }
4493
4494 iocp->ioc_count = size;
4495 iocp->ioc_error = 0;
4496
4497 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4498 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4499 return (IOC_INVAL);
4500 }
4501
4502 return (IOC_REPLY);
4503 }
4504
4505 /*
4506 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
4507 */
4508 static boolean_t
4509 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
4510 {
4511 if (mode == ixgbe->loopback_mode)
4512 return (B_TRUE);
4513
4514 ixgbe->loopback_mode = mode;
4515
4516 if (mode == IXGBE_LB_NONE) {
4517 /*
4518 * Reset the chip
4519 */
4520 (void) ixgbe_reset(ixgbe);
4521 return (B_TRUE);
4522 }
4523
4524 mutex_enter(&ixgbe->gen_lock);
4525
4526 switch (mode) {
4527 default:
4528 mutex_exit(&ixgbe->gen_lock);
4529 return (B_FALSE);
4530
4531 case IXGBE_LB_EXTERNAL:
4532 break;
4533
4534 case IXGBE_LB_INTERNAL_MAC:
4535 ixgbe_set_internal_mac_loopback(ixgbe);
4536 break;
4537 }
4538
4539 mutex_exit(&ixgbe->gen_lock);
4540
4541 return (B_TRUE);
4542 }
4543
4544 /*
4545 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4546 */
4547 static void
4548 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4549 {
4550 struct ixgbe_hw *hw;
4551 uint32_t reg;
4552 uint8_t atlas;
4553
4554 hw = &ixgbe->hw;
4555
4556 /*
4557 * Setup MAC loopback
4558 */
4559 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4560 reg |= IXGBE_HLREG0_LPBK;
4561 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4562
4563 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4564 reg &= ~IXGBE_AUTOC_LMS_MASK;
4565 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4566
4567 /*
4568 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4569 */
4570 switch (hw->mac.type) {
4571 case ixgbe_mac_82598EB:
4572 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4573 &atlas);
4574 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4575 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4576 atlas);
4577
4578 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4579 &atlas);
4580 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4581 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4582 atlas);
4583
4584 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4585 &atlas);
4586 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4587 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4588 atlas);
4589
4590 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4591 &atlas);
4592 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4593 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4594 atlas);
4595 break;
4596
4597 case ixgbe_mac_82599EB:
4598 case ixgbe_mac_X540:
4599 case ixgbe_mac_X550:
4600 case ixgbe_mac_X550EM_x:
4601 case ixgbe_mac_X550EM_a:
4602 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4603 reg |= (IXGBE_AUTOC_FLU |
4604 IXGBE_AUTOC_10G_KX4);
4605 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4606
4607 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4608 B_FALSE);
4609 break;
4610
4611 default:
4612 break;
4613 }
4614 }
4615
4616 #pragma inline(ixgbe_intr_rx_work)
4617 /*
4618 * ixgbe_intr_rx_work - RX processing of ISR.
4619 */
4620 static void
4621 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4622 {
4623 mblk_t *mp;
4624
4625 mutex_enter(&rx_ring->rx_lock);
4626
4627 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4628 mutex_exit(&rx_ring->rx_lock);
4629
4630 if (mp != NULL)
4631 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4632 rx_ring->ring_gen_num);
4633 }
4634
4635 #pragma inline(ixgbe_intr_tx_work)
4636 /*
4637 * ixgbe_intr_tx_work - TX processing of ISR.
4638 */
4639 static void
4640 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4641 {
4642 ixgbe_t *ixgbe = tx_ring->ixgbe;
4643
4644 /*
4645 * Recycle the tx descriptors
4646 */
4647 tx_ring->tx_recycle(tx_ring);
4648
4649 /*
4650 * Schedule the re-transmit
4651 */
4652 if (tx_ring->reschedule &&
4653 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4654 tx_ring->reschedule = B_FALSE;
4655 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4656 tx_ring->ring_handle);
4657 tx_ring->stat_reschedule++;
4658 }
4659 }
4660
4661 #pragma inline(ixgbe_intr_other_work)
4662 /*
4663 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4664 */
4665 static void
4666 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4667 {
4668 struct ixgbe_hw *hw = &ixgbe->hw;
4669
4670 ASSERT(mutex_owned(&ixgbe->gen_lock));
4671
4672 /*
4673 * handle link status change
4674 */
4675 if (eicr & IXGBE_EICR_LSC) {
4676 ixgbe_driver_link_check(ixgbe);
4677 ixgbe_get_hw_state(ixgbe);
4678 }
4679
4680 /*
4681 * check for fan failure on adapters with fans
4682 */
4683 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4684 (eicr & IXGBE_EICR_GPI_SDP1)) {
4685 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4686
4687 /*
4688 * Disable the adapter interrupts
4689 */
4690 ixgbe_disable_adapter_interrupts(ixgbe);
4691
4692 /*
4693 * Disable Rx/Tx units
4694 */
4695 (void) ixgbe_stop_adapter(&ixgbe->hw);
4696
4697 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4698 ixgbe_error(ixgbe,
4699 "Problem: Network adapter has been stopped "
4700 "because the fan has stopped.\n");
4701 ixgbe_error(ixgbe,
4702 "Action: Replace the adapter.\n");
4703
4704 /* re-enable the interrupt, which was automasked */
4705 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4706 }
4707
4708 /*
4709 * Do SFP check for adapters with hot-plug capability
4710 */
4711 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4712 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4713 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4714 ixgbe->eicr = eicr;
4715 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4716 ixgbe_sfp_check, (void *)ixgbe,
4717 DDI_NOSLEEP)) != DDI_SUCCESS) {
4718 ixgbe_log(ixgbe, "No memory available to dispatch "
4719 "taskq for SFP check");
4720 }
4721 }
4722
4723 /*
4724 * Do over-temperature check for adapters with temp sensor
4725 */
4726 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4727 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) ||
4728 (eicr & IXGBE_EICR_LSC))) {
4729 ixgbe->eicr = eicr;
4730 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4731 ixgbe_overtemp_check, (void *)ixgbe,
4732 DDI_NOSLEEP)) != DDI_SUCCESS) {
4733 ixgbe_log(ixgbe, "No memory available to dispatch "
4734 "taskq for overtemp check");
4735 }
4736 }
4737
4738 /*
4739 * Process an external PHY interrupt
4740 */
4741 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
4742 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
4743 ixgbe->eicr = eicr;
4744 if ((ddi_taskq_dispatch(ixgbe->phy_taskq,
4745 ixgbe_phy_check, (void *)ixgbe,
4746 DDI_NOSLEEP)) != DDI_SUCCESS) {
4747 ixgbe_log(ixgbe, "No memory available to dispatch "
4748 "taskq for PHY check");
4749 }
4750 }
4751 }
4752
4753 /*
4754 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4755 */
4756 static uint_t
4757 ixgbe_intr_legacy(void *arg1, void *arg2)
4758 {
4759 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4760 struct ixgbe_hw *hw = &ixgbe->hw;
4761 ixgbe_tx_ring_t *tx_ring;
4762 ixgbe_rx_ring_t *rx_ring;
4763 uint32_t eicr;
4764 mblk_t *mp;
4765 boolean_t tx_reschedule;
4766 uint_t result;
4767
4768 _NOTE(ARGUNUSED(arg2));
4769
4770 mutex_enter(&ixgbe->gen_lock);
4771 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4772 mutex_exit(&ixgbe->gen_lock);
4773 return (DDI_INTR_UNCLAIMED);
4774 }
4775
4776 mp = NULL;
4777 tx_reschedule = B_FALSE;
4778
4779 /*
4780 * Any bit set in eicr: claim this interrupt
4781 */
4782 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4783
4784 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4785 mutex_exit(&ixgbe->gen_lock);
4786 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4787 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4788 return (DDI_INTR_CLAIMED);
4789 }
4790
4791 if (eicr) {
4792 /*
4793 * For legacy interrupt, we have only one interrupt,
4794 * so we have only one rx ring and one tx ring enabled.
4795 */
4796 ASSERT(ixgbe->num_rx_rings == 1);
4797 ASSERT(ixgbe->num_tx_rings == 1);
4798
4799 /*
4800 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4801 */
4802 if (eicr & 0x1) {
4803 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4804 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4805 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4806 /*
4807 * Clean the rx descriptors
4808 */
4809 rx_ring = &ixgbe->rx_rings[0];
4810 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4811 }
4812
4813 /*
4814 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4815 */
4816 if (eicr & 0x2) {
4817 /*
4818 * Recycle the tx descriptors
4819 */
4820 tx_ring = &ixgbe->tx_rings[0];
4821 tx_ring->tx_recycle(tx_ring);
4822
4823 /*
4824 * Schedule the re-transmit
4825 */
4826 tx_reschedule = (tx_ring->reschedule &&
4827 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4828 }
4829
4830 /* any interrupt type other than tx/rx */
4831 if (eicr & ixgbe->capab->other_intr) {
4832 switch (hw->mac.type) {
4833 case ixgbe_mac_82598EB:
4834 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4835 break;
4836
4837 case ixgbe_mac_82599EB:
4838 case ixgbe_mac_X540:
4839 case ixgbe_mac_X550:
4840 case ixgbe_mac_X550EM_x:
4841 case ixgbe_mac_X550EM_a:
4842 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4843 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4844 break;
4845
4846 default:
4847 break;
4848 }
4849 ixgbe_intr_other_work(ixgbe, eicr);
4850 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4851 }
4852
4853 mutex_exit(&ixgbe->gen_lock);
4854
4855 result = DDI_INTR_CLAIMED;
4856 } else {
4857 mutex_exit(&ixgbe->gen_lock);
4858
4859 /*
4860 * No interrupt cause bits set: don't claim this interrupt.
4861 */
4862 result = DDI_INTR_UNCLAIMED;
4863 }
4864
4865 /* re-enable the interrupts which were automasked */
4866 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4867
4868 /*
4869 * Do the following work outside of the gen_lock
4870 */
4871 if (mp != NULL) {
4872 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4873 rx_ring->ring_gen_num);
4874 }
4875
4876 if (tx_reschedule) {
4877 tx_ring->reschedule = B_FALSE;
4878 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4879 tx_ring->stat_reschedule++;
4880 }
4881
4882 return (result);
4883 }
4884
4885 /*
4886 * ixgbe_intr_msi - Interrupt handler for MSI.
4887 */
4888 static uint_t
4889 ixgbe_intr_msi(void *arg1, void *arg2)
4890 {
4891 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4892 struct ixgbe_hw *hw = &ixgbe->hw;
4893 uint32_t eicr;
4894
4895 _NOTE(ARGUNUSED(arg2));
4896
4897 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4898
4899 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4900 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4901 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4902 return (DDI_INTR_CLAIMED);
4903 }
4904
4905 /*
4906 * For MSI interrupt, we have only one vector,
4907 * so we have only one rx ring and one tx ring enabled.
4908 */
4909 ASSERT(ixgbe->num_rx_rings == 1);
4910 ASSERT(ixgbe->num_tx_rings == 1);
4911
4912 /*
4913 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4914 */
4915 if (eicr & 0x1) {
4916 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4917 }
4918
4919 /*
4920 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4921 */
4922 if (eicr & 0x2) {
4923 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4924 }
4925
4926 /* any interrupt type other than tx/rx */
4927 if (eicr & ixgbe->capab->other_intr) {
4928 mutex_enter(&ixgbe->gen_lock);
4929 switch (hw->mac.type) {
4930 case ixgbe_mac_82598EB:
4931 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4932 break;
4933
4934 case ixgbe_mac_82599EB:
4935 case ixgbe_mac_X540:
4936 case ixgbe_mac_X550:
4937 case ixgbe_mac_X550EM_x:
4938 case ixgbe_mac_X550EM_a:
4939 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4940 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4941 break;
4942
4943 default:
4944 break;
4945 }
4946 ixgbe_intr_other_work(ixgbe, eicr);
4947 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4948 mutex_exit(&ixgbe->gen_lock);
4949 }
4950
4951 /* re-enable the interrupts which were automasked */
4952 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4953
4954 return (DDI_INTR_CLAIMED);
4955 }
4956
4957 /*
4958 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4959 */
4960 static uint_t
4961 ixgbe_intr_msix(void *arg1, void *arg2)
4962 {
4963 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4964 ixgbe_t *ixgbe = vect->ixgbe;
4965 struct ixgbe_hw *hw = &ixgbe->hw;
4966 uint32_t eicr;
4967 int r_idx = 0;
4968
4969 _NOTE(ARGUNUSED(arg2));
4970
4971 /*
4972 * Clean each rx ring that has its bit set in the map
4973 */
4974 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4975 while (r_idx >= 0) {
4976 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4977 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4978 (ixgbe->num_rx_rings - 1));
4979 }
4980
4981 /*
4982 * Clean each tx ring that has its bit set in the map
4983 */
4984 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4985 while (r_idx >= 0) {
4986 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4987 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4988 (ixgbe->num_tx_rings - 1));
4989 }
4990
4991
4992 /*
4993 * Clean other interrupt (link change) that has its bit set in the map
4994 */
4995 if (BT_TEST(vect->other_map, 0) == 1) {
4996 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4997
4998 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4999 DDI_FM_OK) {
5000 ddi_fm_service_impact(ixgbe->dip,
5001 DDI_SERVICE_DEGRADED);
5002 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
5003 return (DDI_INTR_CLAIMED);
5004 }
5005
5006 /*
5007 * Check "other" cause bits: any interrupt type other than tx/rx
5008 */
5009 if (eicr & ixgbe->capab->other_intr) {
5010 mutex_enter(&ixgbe->gen_lock);
5011 switch (hw->mac.type) {
5012 case ixgbe_mac_82598EB:
5013 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
5014 ixgbe_intr_other_work(ixgbe, eicr);
5015 break;
5016
5017 case ixgbe_mac_82599EB:
5018 case ixgbe_mac_X540:
5019 case ixgbe_mac_X550:
5020 case ixgbe_mac_X550EM_x:
5021 case ixgbe_mac_X550EM_a:
5022 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
5023 ixgbe_intr_other_work(ixgbe, eicr);
5024 break;
5025
5026 default:
5027 break;
5028 }
5029 mutex_exit(&ixgbe->gen_lock);
5030 }
5031
5032 /* re-enable the interrupts which were automasked */
5033 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
5034 }
5035
5036 return (DDI_INTR_CLAIMED);
5037 }
5038
5039 /*
5040 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
5041 *
5042 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
5043 * if not successful, try Legacy.
5044 * ixgbe->intr_force can be used to force sequence to start with
5045 * any of the 3 types.
5046 * If MSI-X is not used, number of tx/rx rings is forced to 1.
5047 */
5048 static int
5049 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
5050 {
5051 dev_info_t *devinfo;
5052 int intr_types;
5053 int rc;
5054
5055 devinfo = ixgbe->dip;
5056
5057 /*
5058 * Get supported interrupt types
5059 */
5060 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
5061
5062 if (rc != DDI_SUCCESS) {
5063 ixgbe_log(ixgbe,
5064 "Get supported interrupt types failed: %d", rc);
5065 return (IXGBE_FAILURE);
5066 }
5067 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
5068
5069 ixgbe->intr_type = 0;
5070
5071 /*
5072 * Install MSI-X interrupts
5073 */
5074 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
5075 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
5076 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
5077 if (rc == IXGBE_SUCCESS)
5078 return (IXGBE_SUCCESS);
5079
5080 ixgbe_log(ixgbe,
5081 "Allocate MSI-X failed, trying MSI interrupts...");
5082 }
5083
5084 /*
5085 * MSI-X not used, force rings and groups to 1
5086 */
5087 ixgbe->num_rx_rings = 1;
5088 ixgbe->num_rx_groups = 1;
5089 ixgbe->num_tx_rings = 1;
5090 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
5091 ixgbe_log(ixgbe,
5092 "MSI-X not used, force rings and groups number to 1");
5093
5094 /*
5095 * Install MSI interrupts
5096 */
5097 if ((intr_types & DDI_INTR_TYPE_MSI) &&
5098 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
5099 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
5100 if (rc == IXGBE_SUCCESS)
5101 return (IXGBE_SUCCESS);
5102
5103 ixgbe_log(ixgbe,
5104 "Allocate MSI failed, trying Legacy interrupts...");
5105 }
5106
5107 /*
5108 * Install legacy interrupts
5109 */
5110 if (intr_types & DDI_INTR_TYPE_FIXED) {
5111 /*
5112 * Disallow legacy interrupts for X550. X550 has a silicon
5113 * bug which prevents Shared Legacy interrupts from working.
5114 * For details, please reference:
5115 *
5116 * Intel Ethernet Controller X550 Specification Update rev. 2.1
5117 * May 2016, erratum 22: PCIe Interrupt Status Bit
5118 */
5119 if (ixgbe->hw.mac.type == ixgbe_mac_X550 ||
5120 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x ||
5121 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a ||
5122 ixgbe->hw.mac.type == ixgbe_mac_X550_vf ||
5123 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf ||
5124 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a_vf) {
5125 ixgbe_log(ixgbe,
5126 "Legacy interrupts are not supported on this "
5127 "adapter. Please use MSI or MSI-X instead.");
5128 return (IXGBE_FAILURE);
5129 }
5130 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
5131 if (rc == IXGBE_SUCCESS)
5132 return (IXGBE_SUCCESS);
5133
5134 ixgbe_log(ixgbe,
5135 "Allocate Legacy interrupts failed");
5136 }
5137
5138 /*
5139 * If none of the 3 types succeeded, return failure
5140 */
5141 return (IXGBE_FAILURE);
5142 }
5143
5144 /*
5145 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
5146 *
5147 * For legacy and MSI, only 1 handle is needed. For MSI-X,
5148 * if fewer than 2 handles are available, return failure.
5149 * Upon success, this maps the vectors to rx and tx rings for
5150 * interrupts.
5151 */
5152 static int
5153 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
5154 {
5155 dev_info_t *devinfo;
5156 int request, count, actual;
5157 int minimum;
5158 int rc;
5159 uint32_t ring_per_group;
5160
5161 devinfo = ixgbe->dip;
5162
5163 switch (intr_type) {
5164 case DDI_INTR_TYPE_FIXED:
5165 request = 1; /* Request 1 legacy interrupt handle */
5166 minimum = 1;
5167 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
5168 break;
5169
5170 case DDI_INTR_TYPE_MSI:
5171 request = 1; /* Request 1 MSI interrupt handle */
5172 minimum = 1;
5173 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
5174 break;
5175
5176 case DDI_INTR_TYPE_MSIX:
5177 /*
5178 * Best number of vectors for the adapter is
5179 * (# rx rings + # tx rings), however we will
5180 * limit the request number.
5181 */
5182 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
5183 if (request > ixgbe->capab->max_ring_vect)
5184 request = ixgbe->capab->max_ring_vect;
5185 minimum = 1;
5186 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
5187 break;
5188
5189 default:
5190 ixgbe_log(ixgbe,
5191 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
5192 intr_type);
5193 return (IXGBE_FAILURE);
5194 }
5195 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
5196 request, minimum);
5197
5198 /*
5199 * Get number of supported interrupts
5200 */
5201 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
5202 if ((rc != DDI_SUCCESS) || (count < minimum)) {
5203 ixgbe_log(ixgbe,
5204 "Get interrupt number failed. Return: %d, count: %d",
5205 rc, count);
5206 return (IXGBE_FAILURE);
5207 }
5208 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
5209
5210 actual = 0;
5211 ixgbe->intr_cnt = 0;
5212 ixgbe->intr_cnt_max = 0;
5213 ixgbe->intr_cnt_min = 0;
5214
5215 /*
5216 * Allocate an array of interrupt handles
5217 */
5218 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
5219 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
5220
5221 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
5222 request, &actual, DDI_INTR_ALLOC_NORMAL);
5223 if (rc != DDI_SUCCESS) {
5224 ixgbe_log(ixgbe, "Allocate interrupts failed. "
5225 "return: %d, request: %d, actual: %d",
5226 rc, request, actual);
5227 goto alloc_handle_fail;
5228 }
5229 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
5230
5231 /*
5232 * upper/lower limit of interrupts
5233 */
5234 ixgbe->intr_cnt = actual;
5235 ixgbe->intr_cnt_max = request;
5236 ixgbe->intr_cnt_min = minimum;
5237
5238 /*
5239 * rss number per group should not exceed the rx interrupt number,
5240 * else need to adjust rx ring number.
5241 */
5242 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5243 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
5244 if (actual < ring_per_group) {
5245 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
5246 ixgbe_setup_vmdq_rss_conf(ixgbe);
5247 }
5248
5249 /*
5250 * Now we know the actual number of vectors. Here we map the vector
5251 * to other, rx rings and tx ring.
5252 */
5253 if (actual < minimum) {
5254 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
5255 actual);
5256 goto alloc_handle_fail;
5257 }
5258
5259 /*
5260 * Get priority for first vector, assume remaining are all the same
5261 */
5262 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
5263 if (rc != DDI_SUCCESS) {
5264 ixgbe_log(ixgbe,
5265 "Get interrupt priority failed: %d", rc);
5266 goto alloc_handle_fail;
5267 }
5268
5269 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
5270 if (rc != DDI_SUCCESS) {
5271 ixgbe_log(ixgbe,
5272 "Get interrupt cap failed: %d", rc);
5273 goto alloc_handle_fail;
5274 }
5275
5276 ixgbe->intr_type = intr_type;
5277
5278 return (IXGBE_SUCCESS);
5279
5280 alloc_handle_fail:
5281 ixgbe_rem_intrs(ixgbe);
5282
5283 return (IXGBE_FAILURE);
5284 }
5285
5286 /*
5287 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
5288 *
5289 * Before adding the interrupt handlers, the interrupt vectors have
5290 * been allocated, and the rx/tx rings have also been allocated.
5291 */
5292 static int
5293 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
5294 {
5295 int vector = 0;
5296 int rc;
5297
5298 switch (ixgbe->intr_type) {
5299 case DDI_INTR_TYPE_MSIX:
5300 /*
5301 * Add interrupt handler for all vectors
5302 */
5303 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
5304 /*
5305 * install pointer to vect_map[vector]
5306 */
5307 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5308 (ddi_intr_handler_t *)ixgbe_intr_msix,
5309 (void *)&ixgbe->vect_map[vector], NULL);
5310
5311 if (rc != DDI_SUCCESS) {
5312 ixgbe_log(ixgbe,
5313 "Add interrupt handler failed. "
5314 "return: %d, vector: %d", rc, vector);
5315 for (vector--; vector >= 0; vector--) {
5316 (void) ddi_intr_remove_handler(
5317 ixgbe->htable[vector]);
5318 }
5319 return (IXGBE_FAILURE);
5320 }
5321 }
5322
5323 break;
5324
5325 case DDI_INTR_TYPE_MSI:
5326 /*
5327 * Add interrupt handlers for the only vector
5328 */
5329 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5330 (ddi_intr_handler_t *)ixgbe_intr_msi,
5331 (void *)ixgbe, NULL);
5332
5333 if (rc != DDI_SUCCESS) {
5334 ixgbe_log(ixgbe,
5335 "Add MSI interrupt handler failed: %d", rc);
5336 return (IXGBE_FAILURE);
5337 }
5338
5339 break;
5340
5341 case DDI_INTR_TYPE_FIXED:
5342 /*
5343 * Add interrupt handlers for the only vector
5344 */
5345 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5346 (ddi_intr_handler_t *)ixgbe_intr_legacy,
5347 (void *)ixgbe, NULL);
5348
5349 if (rc != DDI_SUCCESS) {
5350 ixgbe_log(ixgbe,
5351 "Add legacy interrupt handler failed: %d", rc);
5352 return (IXGBE_FAILURE);
5353 }
5354
5355 break;
5356
5357 default:
5358 return (IXGBE_FAILURE);
5359 }
5360
5361 return (IXGBE_SUCCESS);
5362 }
5363
5364 #pragma inline(ixgbe_map_rxring_to_vector)
5365 /*
5366 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
5367 */
5368 static void
5369 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
5370 {
5371 /*
5372 * Set bit in map
5373 */
5374 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5375
5376 /*
5377 * Count bits set
5378 */
5379 ixgbe->vect_map[v_idx].rxr_cnt++;
5380
5381 /*
5382 * Remember bit position
5383 */
5384 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
5385 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
5386 }
5387
5388 #pragma inline(ixgbe_map_txring_to_vector)
5389 /*
5390 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
5391 */
5392 static void
5393 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
5394 {
5395 /*
5396 * Set bit in map
5397 */
5398 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
5399
5400 /*
5401 * Count bits set
5402 */
5403 ixgbe->vect_map[v_idx].txr_cnt++;
5404
5405 /*
5406 * Remember bit position
5407 */
5408 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
5409 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
5410 }
5411
5412 /*
5413 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
5414 * allocation register (IVAR).
5415 * cause:
5416 * -1 : other cause
5417 * 0 : rx
5418 * 1 : tx
5419 */
5420 static void
5421 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
5422 int8_t cause)
5423 {
5424 struct ixgbe_hw *hw = &ixgbe->hw;
5425 u32 ivar, index;
5426
5427 switch (hw->mac.type) {
5428 case ixgbe_mac_82598EB:
5429 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5430 if (cause == -1) {
5431 cause = 0;
5432 }
5433 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5434 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5435 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
5436 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
5437 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5438 break;
5439
5440 case ixgbe_mac_82599EB:
5441 case ixgbe_mac_X540:
5442 case ixgbe_mac_X550:
5443 case ixgbe_mac_X550EM_x:
5444 case ixgbe_mac_X550EM_a:
5445 if (cause == -1) {
5446 /* other causes */
5447 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5448 index = (intr_alloc_entry & 1) * 8;
5449 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5450 ivar &= ~(0xFF << index);
5451 ivar |= (msix_vector << index);
5452 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5453 } else {
5454 /* tx or rx causes */
5455 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5456 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5457 ivar = IXGBE_READ_REG(hw,
5458 IXGBE_IVAR(intr_alloc_entry >> 1));
5459 ivar &= ~(0xFF << index);
5460 ivar |= (msix_vector << index);
5461 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5462 ivar);
5463 }
5464 break;
5465
5466 default:
5467 break;
5468 }
5469 }
5470
5471 /*
5472 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
5473 * given interrupt vector allocation register (IVAR).
5474 * cause:
5475 * -1 : other cause
5476 * 0 : rx
5477 * 1 : tx
5478 */
5479 static void
5480 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5481 {
5482 struct ixgbe_hw *hw = &ixgbe->hw;
5483 u32 ivar, index;
5484
5485 switch (hw->mac.type) {
5486 case ixgbe_mac_82598EB:
5487 if (cause == -1) {
5488 cause = 0;
5489 }
5490 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5491 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5492 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
5493 (intr_alloc_entry & 0x3)));
5494 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5495 break;
5496
5497 case ixgbe_mac_82599EB:
5498 case ixgbe_mac_X540:
5499 case ixgbe_mac_X550:
5500 case ixgbe_mac_X550EM_x:
5501 case ixgbe_mac_X550EM_a:
5502 if (cause == -1) {
5503 /* other causes */
5504 index = (intr_alloc_entry & 1) * 8;
5505 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5506 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5507 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5508 } else {
5509 /* tx or rx causes */
5510 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5511 ivar = IXGBE_READ_REG(hw,
5512 IXGBE_IVAR(intr_alloc_entry >> 1));
5513 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5514 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5515 ivar);
5516 }
5517 break;
5518
5519 default:
5520 break;
5521 }
5522 }
5523
5524 /*
5525 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
5526 * given interrupt vector allocation register (IVAR).
5527 * cause:
5528 * -1 : other cause
5529 * 0 : rx
5530 * 1 : tx
5531 */
5532 static void
5533 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5534 {
5535 struct ixgbe_hw *hw = &ixgbe->hw;
5536 u32 ivar, index;
5537
5538 switch (hw->mac.type) {
5539 case ixgbe_mac_82598EB:
5540 if (cause == -1) {
5541 cause = 0;
5542 }
5543 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5544 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5545 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5546 (intr_alloc_entry & 0x3)));
5547 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5548 break;
5549
5550 case ixgbe_mac_82599EB:
5551 case ixgbe_mac_X540:
5552 case ixgbe_mac_X550:
5553 case ixgbe_mac_X550EM_x:
5554 case ixgbe_mac_X550EM_a:
5555 if (cause == -1) {
5556 /* other causes */
5557 index = (intr_alloc_entry & 1) * 8;
5558 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5559 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5560 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5561 } else {
5562 /* tx or rx causes */
5563 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5564 ivar = IXGBE_READ_REG(hw,
5565 IXGBE_IVAR(intr_alloc_entry >> 1));
5566 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5567 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5568 ivar);
5569 }
5570 break;
5571
5572 default:
5573 break;
5574 }
5575 }
5576
5577 /*
5578 * Convert the rx ring index driver maintained to the rx ring index
5579 * in h/w.
5580 */
5581 static uint32_t
5582 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5583 {
5584
5585 struct ixgbe_hw *hw = &ixgbe->hw;
5586 uint32_t rx_ring_per_group, hw_rx_index;
5587
5588 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5589 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5590 return (sw_rx_index);
5591 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5592 switch (hw->mac.type) {
5593 case ixgbe_mac_82598EB:
5594 return (sw_rx_index);
5595
5596 case ixgbe_mac_82599EB:
5597 case ixgbe_mac_X540:
5598 case ixgbe_mac_X550:
5599 case ixgbe_mac_X550EM_x:
5600 case ixgbe_mac_X550EM_a:
5601 return (sw_rx_index * 2);
5602
5603 default:
5604 break;
5605 }
5606 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5607 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5608
5609 switch (hw->mac.type) {
5610 case ixgbe_mac_82598EB:
5611 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5612 16 + (sw_rx_index % rx_ring_per_group);
5613 return (hw_rx_index);
5614
5615 case ixgbe_mac_82599EB:
5616 case ixgbe_mac_X540:
5617 case ixgbe_mac_X550:
5618 case ixgbe_mac_X550EM_x:
5619 case ixgbe_mac_X550EM_a:
5620 if (ixgbe->num_rx_groups > 32) {
5621 hw_rx_index = (sw_rx_index /
5622 rx_ring_per_group) * 2 +
5623 (sw_rx_index % rx_ring_per_group);
5624 } else {
5625 hw_rx_index = (sw_rx_index /
5626 rx_ring_per_group) * 4 +
5627 (sw_rx_index % rx_ring_per_group);
5628 }
5629 return (hw_rx_index);
5630
5631 default:
5632 break;
5633 }
5634 }
5635
5636 /*
5637 * Should never reach. Just to make compiler happy.
5638 */
5639 return (sw_rx_index);
5640 }
5641
5642 /*
5643 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5644 *
5645 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5646 * to vector[0 - (intr_cnt -1)].
5647 */
5648 static int
5649 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5650 {
5651 int i, vector = 0;
5652
5653 /* initialize vector map */
5654 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5655 for (i = 0; i < ixgbe->intr_cnt; i++) {
5656 ixgbe->vect_map[i].ixgbe = ixgbe;
5657 }
5658
5659 /*
5660 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5661 * tx rings[0] on RTxQ[1].
5662 */
5663 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5664 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5665 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5666 return (IXGBE_SUCCESS);
5667 }
5668
5669 /*
5670 * Interrupts/vectors mapping for MSI-X
5671 */
5672
5673 /*
5674 * Map other interrupt to vector 0,
5675 * Set bit in map and count the bits set.
5676 */
5677 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5678 ixgbe->vect_map[vector].other_cnt++;
5679
5680 /*
5681 * Map rx ring interrupts to vectors
5682 */
5683 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5684 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5685 vector = (vector +1) % ixgbe->intr_cnt;
5686 }
5687
5688 /*
5689 * Map tx ring interrupts to vectors
5690 */
5691 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5692 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5693 vector = (vector +1) % ixgbe->intr_cnt;
5694 }
5695
5696 return (IXGBE_SUCCESS);
5697 }
5698
5699 /*
5700 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5701 *
5702 * This relies on ring/vector mapping already set up in the
5703 * vect_map[] structures
5704 */
5705 static void
5706 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5707 {
5708 struct ixgbe_hw *hw = &ixgbe->hw;
5709 ixgbe_intr_vector_t *vect; /* vector bitmap */
5710 int r_idx; /* ring index */
5711 int v_idx; /* vector index */
5712 uint32_t hw_index;
5713
5714 /*
5715 * Clear any previous entries
5716 */
5717 switch (hw->mac.type) {
5718 case ixgbe_mac_82598EB:
5719 for (v_idx = 0; v_idx < 25; v_idx++)
5720 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5721 break;
5722
5723 case ixgbe_mac_82599EB:
5724 case ixgbe_mac_X540:
5725 case ixgbe_mac_X550:
5726 case ixgbe_mac_X550EM_x:
5727 case ixgbe_mac_X550EM_a:
5728 for (v_idx = 0; v_idx < 64; v_idx++)
5729 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5730 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5731 break;
5732
5733 default:
5734 break;
5735 }
5736
5737 /*
5738 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5739 * tx rings[0] will use RTxQ[1].
5740 */
5741 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5742 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5743 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5744 return;
5745 }
5746
5747 /*
5748 * For MSI-X interrupt, "Other" is always on vector[0].
5749 */
5750 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5751
5752 /*
5753 * For each interrupt vector, populate the IVAR table
5754 */
5755 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5756 vect = &ixgbe->vect_map[v_idx];
5757
5758 /*
5759 * For each rx ring bit set
5760 */
5761 r_idx = bt_getlowbit(vect->rx_map, 0,
5762 (ixgbe->num_rx_rings - 1));
5763
5764 while (r_idx >= 0) {
5765 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5766 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5767 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5768 (ixgbe->num_rx_rings - 1));
5769 }
5770
5771 /*
5772 * For each tx ring bit set
5773 */
5774 r_idx = bt_getlowbit(vect->tx_map, 0,
5775 (ixgbe->num_tx_rings - 1));
5776
5777 while (r_idx >= 0) {
5778 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5779 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5780 (ixgbe->num_tx_rings - 1));
5781 }
5782 }
5783 }
5784
5785 /*
5786 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5787 */
5788 static void
5789 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5790 {
5791 int i;
5792 int rc;
5793
5794 for (i = 0; i < ixgbe->intr_cnt; i++) {
5795 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5796 if (rc != DDI_SUCCESS) {
5797 IXGBE_DEBUGLOG_1(ixgbe,
5798 "Remove intr handler failed: %d", rc);
5799 }
5800 }
5801 }
5802
5803 /*
5804 * ixgbe_rem_intrs - Remove the allocated interrupts.
5805 */
5806 static void
5807 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5808 {
5809 int i;
5810 int rc;
5811
5812 for (i = 0; i < ixgbe->intr_cnt; i++) {
5813 rc = ddi_intr_free(ixgbe->htable[i]);
5814 if (rc != DDI_SUCCESS) {
5815 IXGBE_DEBUGLOG_1(ixgbe,
5816 "Free intr failed: %d", rc);
5817 }
5818 }
5819
5820 kmem_free(ixgbe->htable, ixgbe->intr_size);
5821 ixgbe->htable = NULL;
5822 }
5823
5824 /*
5825 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5826 */
5827 static int
5828 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5829 {
5830 int i;
5831 int rc;
5832
5833 /*
5834 * Enable interrupts
5835 */
5836 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5837 /*
5838 * Call ddi_intr_block_enable() for MSI
5839 */
5840 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5841 if (rc != DDI_SUCCESS) {
5842 ixgbe_log(ixgbe,
5843 "Enable block intr failed: %d", rc);
5844 return (IXGBE_FAILURE);
5845 }
5846 } else {
5847 /*
5848 * Call ddi_intr_enable() for Legacy/MSI non block enable
5849 */
5850 for (i = 0; i < ixgbe->intr_cnt; i++) {
5851 rc = ddi_intr_enable(ixgbe->htable[i]);
5852 if (rc != DDI_SUCCESS) {
5853 ixgbe_log(ixgbe,
5854 "Enable intr failed: %d", rc);
5855 return (IXGBE_FAILURE);
5856 }
5857 }
5858 }
5859
5860 return (IXGBE_SUCCESS);
5861 }
5862
5863 /*
5864 * ixgbe_disable_intrs - Disable all the interrupts.
5865 */
5866 static int
5867 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5868 {
5869 int i;
5870 int rc;
5871
5872 /*
5873 * Disable all interrupts
5874 */
5875 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5876 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5877 if (rc != DDI_SUCCESS) {
5878 ixgbe_log(ixgbe,
5879 "Disable block intr failed: %d", rc);
5880 return (IXGBE_FAILURE);
5881 }
5882 } else {
5883 for (i = 0; i < ixgbe->intr_cnt; i++) {
5884 rc = ddi_intr_disable(ixgbe->htable[i]);
5885 if (rc != DDI_SUCCESS) {
5886 ixgbe_log(ixgbe,
5887 "Disable intr failed: %d", rc);
5888 return (IXGBE_FAILURE);
5889 }
5890 }
5891 }
5892
5893 return (IXGBE_SUCCESS);
5894 }
5895
5896 /*
5897 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5898 */
5899 static void
5900 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5901 {
5902 struct ixgbe_hw *hw = &ixgbe->hw;
5903 ixgbe_link_speed speed = 0;
5904 boolean_t link_up = B_FALSE;
5905 uint32_t pcs1g_anlp = 0;
5906
5907 ASSERT(mutex_owned(&ixgbe->gen_lock));
5908 ixgbe->param_lp_1000fdx_cap = 0;
5909 ixgbe->param_lp_100fdx_cap = 0;
5910
5911 /* check for link, don't wait */
5912 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
5913
5914 /*
5915 * Update the observed Link Partner's capabilities. Not all adapters
5916 * can provide full information on the LP's capable speeds, so we
5917 * provide what we can.
5918 */
5919 if (link_up) {
5920 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5921
5922 ixgbe->param_lp_1000fdx_cap =
5923 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5924 ixgbe->param_lp_100fdx_cap =
5925 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5926 }
5927
5928 /*
5929 * Update GLD's notion of the adapter's currently advertised speeds.
5930 * Since the common code doesn't always record the current autonegotiate
5931 * settings in the phy struct for all parts (specifically, adapters with
5932 * SFPs) we first test to see if it is 0, and if so, we fall back to
5933 * using the adapter's speed capabilities which we saved during instance
5934 * init in ixgbe_init_params().
5935 *
5936 * Adapters with SFPs will always be shown as advertising all of their
5937 * supported speeds, and adapters with baseT PHYs (where the phy struct
5938 * is maintained by the common code) will always have a factual view of
5939 * their currently-advertised speeds. In the case of SFPs, this is
5940 * acceptable as we default to advertising all speeds that the adapter
5941 * claims to support, and those properties are immutable; unlike on
5942 * baseT (copper) PHYs, where speeds can be enabled or disabled at will.
5943 */
5944 speed = hw->phy.autoneg_advertised;
5945 if (speed == 0)
5946 speed = ixgbe->speeds_supported;
5947
5948 ixgbe->param_adv_10000fdx_cap =
5949 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0;
5950 ixgbe->param_adv_5000fdx_cap =
5951 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0;
5952 ixgbe->param_adv_2500fdx_cap =
5953 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0;
5954 ixgbe->param_adv_1000fdx_cap =
5955 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0;
5956 ixgbe->param_adv_100fdx_cap =
5957 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0;
5958 }
5959
5960 /*
5961 * ixgbe_get_driver_control - Notify that driver is in control of device.
5962 */
5963 static void
5964 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5965 {
5966 uint32_t ctrl_ext;
5967
5968 /*
5969 * Notify firmware that driver is in control of device
5970 */
5971 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5972 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5973 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5974 }
5975
5976 /*
5977 * ixgbe_release_driver_control - Notify that driver is no longer in control
5978 * of device.
5979 */
5980 static void
5981 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5982 {
5983 uint32_t ctrl_ext;
5984
5985 /*
5986 * Notify firmware that driver is no longer in control of device
5987 */
5988 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5989 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5990 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5991 }
5992
5993 /*
5994 * ixgbe_atomic_reserve - Atomic decrease operation.
5995 */
5996 int
5997 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5998 {
5999 uint32_t oldval;
6000 uint32_t newval;
6001
6002 /*
6003 * ATOMICALLY
6004 */
6005 do {
6006 oldval = *count_p;
6007 if (oldval < n)
6008 return (-1);
6009 newval = oldval - n;
6010 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
6011
6012 return (newval);
6013 }
6014
6015 /*
6016 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
6017 */
6018 static uint8_t *
6019 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
6020 {
6021 uint8_t *addr = *upd_ptr;
6022 uint8_t *new_ptr;
6023
6024 _NOTE(ARGUNUSED(hw));
6025 _NOTE(ARGUNUSED(vmdq));
6026
6027 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
6028 *upd_ptr = new_ptr;
6029 return (addr);
6030 }
6031
6032 /*
6033 * FMA support
6034 */
6035 int
6036 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
6037 {
6038 ddi_fm_error_t de;
6039
6040 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
6041 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
6042 return (de.fme_status);
6043 }
6044
6045 int
6046 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
6047 {
6048 ddi_fm_error_t de;
6049
6050 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
6051 return (de.fme_status);
6052 }
6053
6054 /*
6055 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
6056 */
6057 static int
6058 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
6059 {
6060 _NOTE(ARGUNUSED(impl_data));
6061 /*
6062 * as the driver can always deal with an error in any dma or
6063 * access handle, we can just return the fme_status value.
6064 */
6065 pci_ereport_post(dip, err, NULL);
6066 return (err->fme_status);
6067 }
6068
6069 static void
6070 ixgbe_fm_init(ixgbe_t *ixgbe)
6071 {
6072 ddi_iblock_cookie_t iblk;
6073 int fma_dma_flag;
6074
6075 /*
6076 * Only register with IO Fault Services if we have some capability
6077 */
6078 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
6079 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
6080 } else {
6081 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
6082 }
6083
6084 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
6085 fma_dma_flag = 1;
6086 } else {
6087 fma_dma_flag = 0;
6088 }
6089
6090 ixgbe_set_fma_flags(fma_dma_flag);
6091
6092 if (ixgbe->fm_capabilities) {
6093
6094 /*
6095 * Register capabilities with IO Fault Services
6096 */
6097 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
6098
6099 /*
6100 * Initialize pci ereport capabilities if ereport capable
6101 */
6102 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
6103 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6104 pci_ereport_setup(ixgbe->dip);
6105
6106 /*
6107 * Register error callback if error callback capable
6108 */
6109 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6110 ddi_fm_handler_register(ixgbe->dip,
6111 ixgbe_fm_error_cb, (void*) ixgbe);
6112 }
6113 }
6114
6115 static void
6116 ixgbe_fm_fini(ixgbe_t *ixgbe)
6117 {
6118 /*
6119 * Only unregister FMA capabilities if they are registered
6120 */
6121 if (ixgbe->fm_capabilities) {
6122
6123 /*
6124 * Release any resources allocated by pci_ereport_setup()
6125 */
6126 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
6127 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6128 pci_ereport_teardown(ixgbe->dip);
6129
6130 /*
6131 * Un-register error callback if error callback capable
6132 */
6133 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6134 ddi_fm_handler_unregister(ixgbe->dip);
6135
6136 /*
6137 * Unregister from IO Fault Service
6138 */
6139 ddi_fm_fini(ixgbe->dip);
6140 }
6141 }
6142
6143 void
6144 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
6145 {
6146 uint64_t ena;
6147 char buf[FM_MAX_CLASS];
6148
6149 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
6150 ena = fm_ena_generate(0, FM_ENA_FMT1);
6151 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
6152 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
6153 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
6154 }
6155 }
6156
6157 static int
6158 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
6159 {
6160 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
6161
6162 mutex_enter(&rx_ring->rx_lock);
6163 rx_ring->ring_gen_num = mr_gen_num;
6164 mutex_exit(&rx_ring->rx_lock);
6165 return (0);
6166 }
6167
6168 /*
6169 * Get the global ring index by a ring index within a group.
6170 */
6171 static int
6172 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
6173 {
6174 ixgbe_rx_ring_t *rx_ring;
6175 int i;
6176
6177 for (i = 0; i < ixgbe->num_rx_rings; i++) {
6178 rx_ring = &ixgbe->rx_rings[i];
6179 if (rx_ring->group_index == gindex)
6180 rindex--;
6181 if (rindex < 0)
6182 return (i);
6183 }
6184
6185 return (-1);
6186 }
6187
6188 /*
6189 * Callback funtion for MAC layer to register all rings.
6190 */
6191 /* ARGSUSED */
6192 void
6193 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
6194 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
6195 {
6196 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6197 mac_intr_t *mintr = &infop->mri_intr;
6198
6199 switch (rtype) {
6200 case MAC_RING_TYPE_RX: {
6201 /*
6202 * 'index' is the ring index within the group.
6203 * Need to get the global ring index by searching in groups.
6204 */
6205 int global_ring_index = ixgbe_get_rx_ring_index(
6206 ixgbe, group_index, ring_index);
6207
6208 ASSERT(global_ring_index >= 0);
6209
6210 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
6211 rx_ring->ring_handle = rh;
6212
6213 infop->mri_driver = (mac_ring_driver_t)rx_ring;
6214 infop->mri_start = ixgbe_ring_start;
6215 infop->mri_stop = NULL;
6216 infop->mri_poll = ixgbe_ring_rx_poll;
6217 infop->mri_stat = ixgbe_rx_ring_stat;
6218
6219 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
6220 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
6221 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
6222 if (ixgbe->intr_type &
6223 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6224 mintr->mi_ddi_handle =
6225 ixgbe->htable[rx_ring->intr_vector];
6226 }
6227
6228 break;
6229 }
6230 case MAC_RING_TYPE_TX: {
6231 ASSERT(group_index == -1);
6232 ASSERT(ring_index < ixgbe->num_tx_rings);
6233
6234 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
6235 tx_ring->ring_handle = rh;
6236
6237 infop->mri_driver = (mac_ring_driver_t)tx_ring;
6238 infop->mri_start = NULL;
6239 infop->mri_stop = NULL;
6240 infop->mri_tx = ixgbe_ring_tx;
6241 infop->mri_stat = ixgbe_tx_ring_stat;
6242 if (ixgbe->intr_type &
6243 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6244 mintr->mi_ddi_handle =
6245 ixgbe->htable[tx_ring->intr_vector];
6246 }
6247 break;
6248 }
6249 default:
6250 break;
6251 }
6252 }
6253
6254 /*
6255 * Callback funtion for MAC layer to register all groups.
6256 */
6257 void
6258 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
6259 mac_group_info_t *infop, mac_group_handle_t gh)
6260 {
6261 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6262 struct ixgbe_hw *hw = &ixgbe->hw;
6263
6264 switch (rtype) {
6265 case MAC_RING_TYPE_RX: {
6266 ixgbe_rx_group_t *rx_group;
6267
6268 rx_group = &ixgbe->rx_groups[index];
6269 rx_group->group_handle = gh;
6270
6271 infop->mgi_driver = (mac_group_driver_t)rx_group;
6272 infop->mgi_start = NULL;
6273 infop->mgi_stop = NULL;
6274 infop->mgi_addmac = ixgbe_addmac;
6275 infop->mgi_remmac = ixgbe_remmac;
6276
6277 if ((ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ ||
6278 ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) &&
6279 (hw->mac.type == ixgbe_mac_82599EB ||
6280 hw->mac.type == ixgbe_mac_X540 ||
6281 hw->mac.type == ixgbe_mac_X550 ||
6282 hw->mac.type == ixgbe_mac_X550EM_x)) {
6283 infop->mgi_addvlan = ixgbe_addvlan;
6284 infop->mgi_remvlan = ixgbe_remvlan;
6285 } else {
6286 infop->mgi_addvlan = NULL;
6287 infop->mgi_remvlan = NULL;
6288 }
6289
6290 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
6291
6292 break;
6293 }
6294 case MAC_RING_TYPE_TX:
6295 break;
6296 default:
6297 break;
6298 }
6299 }
6300
6301 /*
6302 * Enable interrupt on the specificed rx ring.
6303 */
6304 int
6305 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
6306 {
6307 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6308 ixgbe_t *ixgbe = rx_ring->ixgbe;
6309 int r_idx = rx_ring->index;
6310 int hw_r_idx = rx_ring->hw_index;
6311 int v_idx = rx_ring->intr_vector;
6312
6313 mutex_enter(&ixgbe->gen_lock);
6314 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6315 mutex_exit(&ixgbe->gen_lock);
6316 /*
6317 * Simply return 0.
6318 * Interrupts are being adjusted. ixgbe_intr_adjust()
6319 * will eventually re-enable the interrupt when it's
6320 * done with the adjustment.
6321 */
6322 return (0);
6323 }
6324
6325 /*
6326 * To enable interrupt by setting the VAL bit of given interrupt
6327 * vector allocation register (IVAR).
6328 */
6329 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
6330
6331 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
6332
6333 /*
6334 * Trigger a Rx interrupt on this ring
6335 */
6336 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
6337 IXGBE_WRITE_FLUSH(&ixgbe->hw);
6338
6339 mutex_exit(&ixgbe->gen_lock);
6340
6341 return (0);
6342 }
6343
6344 /*
6345 * Disable interrupt on the specificed rx ring.
6346 */
6347 int
6348 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
6349 {
6350 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6351 ixgbe_t *ixgbe = rx_ring->ixgbe;
6352 int r_idx = rx_ring->index;
6353 int hw_r_idx = rx_ring->hw_index;
6354 int v_idx = rx_ring->intr_vector;
6355
6356 mutex_enter(&ixgbe->gen_lock);
6357 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6358 mutex_exit(&ixgbe->gen_lock);
6359 /*
6360 * Simply return 0.
6361 * In the rare case where an interrupt is being
6362 * disabled while interrupts are being adjusted,
6363 * we don't fail the operation. No interrupts will
6364 * be generated while they are adjusted, and
6365 * ixgbe_intr_adjust() will cause the interrupts
6366 * to be re-enabled once it completes. Note that
6367 * in this case, packets may be delivered to the
6368 * stack via interrupts before xgbe_rx_ring_intr_enable()
6369 * is called again. This is acceptable since interrupt
6370 * adjustment is infrequent, and the stack will be
6371 * able to handle these packets.
6372 */
6373 return (0);
6374 }
6375
6376 /*
6377 * To disable interrupt by clearing the VAL bit of given interrupt
6378 * vector allocation register (IVAR).
6379 */
6380 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
6381
6382 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
6383
6384 mutex_exit(&ixgbe->gen_lock);
6385
6386 return (0);
6387 }
6388
6389 static ixgbe_vlan_t *
6390 ixgbe_find_vlan(ixgbe_rx_group_t *rx_group, uint16_t vid)
6391 {
6392 for (ixgbe_vlan_t *vlp = list_head(&rx_group->vlans); vlp != NULL;
6393 vlp = list_next(&rx_group->vlans, vlp)) {
6394 if (vlp->ixvl_vid == vid)
6395 return (vlp);
6396 }
6397
6398 return (NULL);
6399 }
6400
6401 /*
6402 * Attempt to use a VLAN HW filter for this group. If the group is
6403 * interested in untagged packets then set AUPE only. If the group is
6404 * the default then only set the VFTA. Leave the VLVF slots open for
6405 * reserved groups to guarantee their use of HW filtering.
6406 */
6407 static int
6408 ixgbe_addvlan(mac_group_driver_t gdriver, uint16_t vid)
6409 {
6410 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)gdriver;
6411 ixgbe_t *ixgbe = rx_group->ixgbe;
6412 struct ixgbe_hw *hw = &ixgbe->hw;
6413 ixgbe_vlan_t *vlp;
6414 int ret;
6415 boolean_t is_def_grp;
6416
6417 mutex_enter(&ixgbe->gen_lock);
6418
6419 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6420 mutex_exit(&ixgbe->gen_lock);
6421 return (ECANCELED);
6422 }
6423
6424 /*
6425 * Let's be sure VLAN filtering is enabled.
6426 */
6427 VERIFY3B(ixgbe->vlft_enabled, ==, B_TRUE);
6428 is_def_grp = (rx_group->index == ixgbe->rx_def_group);
6429
6430 /*
6431 * VLAN filtering is enabled but we want to receive untagged
6432 * traffic on this group -- set the AUPE bit on the group and
6433 * leave the VLAN tables alone.
6434 */
6435 if (vid == MAC_VLAN_UNTAGGED) {
6436 /*
6437 * We never enable AUPE on the default group; it is
6438 * redundant. Untagged traffic which passes L2
6439 * filtering is delivered to the default group if no
6440 * other group is interested.
6441 */
6442 if (!is_def_grp) {
6443 uint32_t vml2flt;
6444
6445 vml2flt = IXGBE_READ_REG(hw,
6446 IXGBE_VMOLR(rx_group->index));
6447 vml2flt |= IXGBE_VMOLR_AUPE;
6448 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(rx_group->index),
6449 vml2flt);
6450 rx_group->aupe = B_TRUE;
6451 }
6452
6453 mutex_exit(&ixgbe->gen_lock);
6454 return (0);
6455 }
6456
6457 vlp = ixgbe_find_vlan(rx_group, vid);
6458 if (vlp != NULL) {
6459 /* Only the default group supports multiple clients. */
6460 VERIFY3B(is_def_grp, ==, B_TRUE);
6461 vlp->ixvl_refs++;
6462 mutex_exit(&ixgbe->gen_lock);
6463 return (0);
6464 }
6465
6466 /*
6467 * The default group doesn't require a VLVF entry, only a VFTA
6468 * entry. All traffic passing L2 filtering (MPSAR + VFTA) is
6469 * delivered to the default group if no other group is
6470 * interested. The fourth argument, vlvf_bypass, tells the
6471 * ixgbe common code to avoid using a VLVF slot if one isn't
6472 * already allocated to this VLAN.
6473 *
6474 * This logic is meant to reserve VLVF slots for use by
6475 * reserved groups: guaranteeing their use of HW filtering.
6476 */
6477 ret = ixgbe_set_vfta(hw, vid, rx_group->index, B_TRUE, is_def_grp);
6478
6479 if (ret == IXGBE_SUCCESS) {
6480 vlp = kmem_zalloc(sizeof (ixgbe_vlan_t), KM_SLEEP);
6481 vlp->ixvl_vid = vid;
6482 vlp->ixvl_refs = 1;
6483 list_insert_tail(&rx_group->vlans, vlp);
6484 mutex_exit(&ixgbe->gen_lock);
6485 return (0);
6486 }
6487
6488 /*
6489 * We should actually never return ENOSPC because we've set
6490 * things up so that every reserved group is guaranteed to
6491 * have a VLVF slot.
6492 */
6493 if (ret == IXGBE_ERR_PARAM)
6494 ret = EINVAL;
6495 else if (ret == IXGBE_ERR_NO_SPACE)
6496 ret = ENOSPC;
6497 else
6498 ret = EIO;
6499
6500 mutex_exit(&ixgbe->gen_lock);
6501 return (ret);
6502 }
6503
6504 /*
6505 * Attempt to remove the VLAN HW filter associated with this group. If
6506 * we are removing a HW filter for the default group then we know only
6507 * the VFTA was set (VLVF is reserved for non-default/reserved
6508 * groups). If the group wishes to stop receiving untagged traffic
6509 * then clear the AUPE but leave the VLAN filters alone.
6510 */
6511 static int
6512 ixgbe_remvlan(mac_group_driver_t gdriver, uint16_t vid)
6513 {
6514 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)gdriver;
6515 ixgbe_t *ixgbe = rx_group->ixgbe;
6516 struct ixgbe_hw *hw = &ixgbe->hw;
6517 int ret;
6518 ixgbe_vlan_t *vlp;
6519 boolean_t is_def_grp;
6520
6521 mutex_enter(&ixgbe->gen_lock);
6522
6523 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6524 mutex_exit(&ixgbe->gen_lock);
6525 return (ECANCELED);
6526 }
6527
6528 is_def_grp = (rx_group->index == ixgbe->rx_def_group);
6529
6530 /* See the AUPE comment in ixgbe_addvlan(). */
6531 if (vid == MAC_VLAN_UNTAGGED) {
6532 if (!is_def_grp) {
6533 uint32_t vml2flt;
6534
6535 vml2flt = IXGBE_READ_REG(hw,
6536 IXGBE_VMOLR(rx_group->index));
6537 vml2flt &= ~IXGBE_VMOLR_AUPE;
6538 IXGBE_WRITE_REG(hw,
6539 IXGBE_VMOLR(rx_group->index), vml2flt);
6540 rx_group->aupe = B_FALSE;
6541 }
6542 mutex_exit(&ixgbe->gen_lock);
6543 return (0);
6544 }
6545
6546 vlp = ixgbe_find_vlan(rx_group, vid);
6547 if (vlp == NULL)
6548 return (ENOENT);
6549
6550 /*
6551 * See the comment in ixgbe_addvlan() about is_def_grp and
6552 * vlvf_bypass.
6553 */
6554 if (vlp->ixvl_refs == 1) {
6555 ret = ixgbe_set_vfta(hw, vid, rx_group->index, B_FALSE,
6556 is_def_grp);
6557 } else {
6558 /*
6559 * Only the default group can have multiple clients.
6560 * If there is more than one client, leave the
6561 * VFTA[vid] bit alone.
6562 */
6563 VERIFY3B(is_def_grp, ==, B_TRUE);
6564 VERIFY3U(vlp->ixvl_refs, >, 1);
6565 vlp->ixvl_refs--;
6566 mutex_exit(&ixgbe->gen_lock);
6567 return (0);
6568 }
6569
6570 if (ret != IXGBE_SUCCESS) {
6571 mutex_exit(&ixgbe->gen_lock);
6572 /* IXGBE_ERR_PARAM should be the only possible error here. */
6573 if (ret == IXGBE_ERR_PARAM)
6574 return (EINVAL);
6575 else
6576 return (EIO);
6577 }
6578
6579 VERIFY3U(vlp->ixvl_refs, ==, 1);
6580 vlp->ixvl_refs = 0;
6581 list_remove(&rx_group->vlans, vlp);
6582 kmem_free(vlp, sizeof (ixgbe_vlan_t));
6583
6584 /*
6585 * Calling ixgbe_set_vfta() on a non-default group may have
6586 * cleared the VFTA[vid] bit even though the default group
6587 * still has clients using the vid. This happens because the
6588 * ixgbe common code doesn't ref count the use of VLANs. Check
6589 * for any use of vid on the default group and make sure the
6590 * VFTA[vid] bit is set. This operation is idempotent: setting
6591 * VFTA[vid] to true if already true won't hurt anything.
6592 */
6593 if (!is_def_grp) {
6594 ixgbe_rx_group_t *defgrp;
6595
6596 defgrp = &ixgbe->rx_groups[ixgbe->rx_def_group];
6597 vlp = ixgbe_find_vlan(defgrp, vid);
6598 if (vlp != NULL) {
6599 /* This shouldn't fail, but if it does return EIO. */
6600 ret = ixgbe_set_vfta(hw, vid, rx_group->index, B_TRUE,
6601 B_TRUE);
6602 if (ret != IXGBE_SUCCESS)
6603 return (EIO);
6604 }
6605 }
6606
6607 mutex_exit(&ixgbe->gen_lock);
6608 return (0);
6609 }
6610
6611 /*
6612 * Add a mac address.
6613 */
6614 static int
6615 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
6616 {
6617 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6618 ixgbe_t *ixgbe = rx_group->ixgbe;
6619 struct ixgbe_hw *hw = &ixgbe->hw;
6620 int slot, i;
6621
6622 mutex_enter(&ixgbe->gen_lock);
6623
6624 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6625 mutex_exit(&ixgbe->gen_lock);
6626 return (ECANCELED);
6627 }
6628
6629 if (ixgbe->unicst_avail == 0) {
6630 /* no slots available */
6631 mutex_exit(&ixgbe->gen_lock);
6632 return (ENOSPC);
6633 }
6634
6635 /*
6636 * The first ixgbe->num_rx_groups slots are reserved for each respective
6637 * group. The rest slots are shared by all groups. While adding a
6638 * MAC address, reserved slots are firstly checked then the shared
6639 * slots are searched.
6640 */
6641 slot = -1;
6642 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
6643 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
6644 if (ixgbe->unicst_addr[i].mac.set == 0) {
6645 slot = i;
6646 break;
6647 }
6648 }
6649 } else {
6650 slot = rx_group->index;
6651 }
6652
6653 if (slot == -1) {
6654 /* no slots available */
6655 mutex_exit(&ixgbe->gen_lock);
6656 return (ENOSPC);
6657 }
6658
6659 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6660 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
6661 rx_group->index, IXGBE_RAH_AV);
6662 ixgbe->unicst_addr[slot].mac.set = 1;
6663 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
6664 ixgbe->unicst_avail--;
6665
6666 mutex_exit(&ixgbe->gen_lock);
6667
6668 return (0);
6669 }
6670
6671 /*
6672 * Remove a mac address.
6673 */
6674 static int
6675 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
6676 {
6677 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6678 ixgbe_t *ixgbe = rx_group->ixgbe;
6679 struct ixgbe_hw *hw = &ixgbe->hw;
6680 int slot;
6681
6682 mutex_enter(&ixgbe->gen_lock);
6683
6684 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6685 mutex_exit(&ixgbe->gen_lock);
6686 return (ECANCELED);
6687 }
6688
6689 slot = ixgbe_unicst_find(ixgbe, mac_addr);
6690 if (slot == -1) {
6691 mutex_exit(&ixgbe->gen_lock);
6692 return (EINVAL);
6693 }
6694
6695 if (ixgbe->unicst_addr[slot].mac.set == 0) {
6696 mutex_exit(&ixgbe->gen_lock);
6697 return (EINVAL);
6698 }
6699
6700 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6701 (void) ixgbe_clear_rar(hw, slot);
6702 ixgbe->unicst_addr[slot].mac.set = 0;
6703 ixgbe->unicst_avail++;
6704
6705 mutex_exit(&ixgbe->gen_lock);
6706
6707 return (0);
6708 }