1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2017, Joyent, Inc.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
33 */
34
35 #include "ixgbe_sw.h"
36
37 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
38
39 /*
40 * Local function protoypes
41 */
42 static int ixgbe_register_mac(ixgbe_t *);
43 static int ixgbe_identify_hardware(ixgbe_t *);
44 static int ixgbe_regs_map(ixgbe_t *);
45 static void ixgbe_init_properties(ixgbe_t *);
46 static int ixgbe_init_driver_settings(ixgbe_t *);
47 static void ixgbe_init_locks(ixgbe_t *);
48 static void ixgbe_destroy_locks(ixgbe_t *);
49 static int ixgbe_init(ixgbe_t *);
50 static int ixgbe_chip_start(ixgbe_t *);
51 static void ixgbe_chip_stop(ixgbe_t *);
52 static int ixgbe_reset(ixgbe_t *);
53 static void ixgbe_tx_clean(ixgbe_t *);
54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
56 static int ixgbe_alloc_rings(ixgbe_t *);
57 static void ixgbe_free_rings(ixgbe_t *);
58 static int ixgbe_alloc_rx_data(ixgbe_t *);
59 static void ixgbe_free_rx_data(ixgbe_t *);
60 static void ixgbe_setup_rings(ixgbe_t *);
61 static void ixgbe_setup_rx(ixgbe_t *);
62 static void ixgbe_setup_tx(ixgbe_t *);
63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
65 static void ixgbe_setup_rss(ixgbe_t *);
66 static void ixgbe_setup_vmdq(ixgbe_t *);
67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
68 static void ixgbe_setup_rss_table(ixgbe_t *);
69 static void ixgbe_init_unicst(ixgbe_t *);
70 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
71 static void ixgbe_setup_multicst(ixgbe_t *);
72 static void ixgbe_get_hw_state(ixgbe_t *);
73 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
74 static void ixgbe_get_conf(ixgbe_t *);
75 static void ixgbe_init_params(ixgbe_t *);
76 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
77 static void ixgbe_driver_link_check(ixgbe_t *);
78 static void ixgbe_sfp_check(void *);
79 static void ixgbe_overtemp_check(void *);
80 static void ixgbe_phy_check(void *);
81 static void ixgbe_link_timer(void *);
82 static void ixgbe_local_timer(void *);
83 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
84 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
87 static boolean_t is_valid_mac_addr(uint8_t *);
88 static boolean_t ixgbe_stall_check(ixgbe_t *);
89 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
91 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
92 static int ixgbe_alloc_intrs(ixgbe_t *);
93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
94 static int ixgbe_add_intr_handlers(ixgbe_t *);
95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
101 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
102 static void ixgbe_setup_adapter_vector(ixgbe_t *);
103 static void ixgbe_rem_intr_handlers(ixgbe_t *);
104 static void ixgbe_rem_intrs(ixgbe_t *);
105 static int ixgbe_enable_intrs(ixgbe_t *);
106 static int ixgbe_disable_intrs(ixgbe_t *);
107 static uint_t ixgbe_intr_legacy(void *, void *);
108 static uint_t ixgbe_intr_msi(void *, void *);
109 static uint_t ixgbe_intr_msix(void *, void *);
110 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
111 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
112 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
113 static void ixgbe_get_driver_control(struct ixgbe_hw *);
114 static int ixgbe_addmac(void *, const uint8_t *);
115 static int ixgbe_remmac(void *, const uint8_t *);
116 static void ixgbe_release_driver_control(struct ixgbe_hw *);
117
118 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
119 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
120 static int ixgbe_resume(dev_info_t *);
121 static int ixgbe_suspend(dev_info_t *);
122 static int ixgbe_quiesce(dev_info_t *);
123 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
124 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
125 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
126 static int ixgbe_intr_cb_register(ixgbe_t *);
127 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
128
129 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
130 const void *impl_data);
131 static void ixgbe_fm_init(ixgbe_t *);
132 static void ixgbe_fm_fini(ixgbe_t *);
133
134 char *ixgbe_priv_props[] = {
135 "_tx_copy_thresh",
136 "_tx_recycle_thresh",
137 "_tx_overload_thresh",
138 "_tx_resched_thresh",
139 "_rx_copy_thresh",
140 "_rx_limit_per_intr",
141 "_intr_throttling",
142 "_adv_pause_cap",
143 "_adv_asym_pause_cap",
144 NULL
145 };
146
147 #define IXGBE_MAX_PRIV_PROPS \
148 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
149
150 static struct cb_ops ixgbe_cb_ops = {
151 nulldev, /* cb_open */
152 nulldev, /* cb_close */
153 nodev, /* cb_strategy */
154 nodev, /* cb_print */
155 nodev, /* cb_dump */
156 nodev, /* cb_read */
157 nodev, /* cb_write */
158 nodev, /* cb_ioctl */
159 nodev, /* cb_devmap */
160 nodev, /* cb_mmap */
161 nodev, /* cb_segmap */
162 nochpoll, /* cb_chpoll */
163 ddi_prop_op, /* cb_prop_op */
164 NULL, /* cb_stream */
165 D_MP | D_HOTPLUG, /* cb_flag */
166 CB_REV, /* cb_rev */
167 nodev, /* cb_aread */
168 nodev /* cb_awrite */
169 };
170
171 static struct dev_ops ixgbe_dev_ops = {
172 DEVO_REV, /* devo_rev */
173 0, /* devo_refcnt */
174 NULL, /* devo_getinfo */
175 nulldev, /* devo_identify */
176 nulldev, /* devo_probe */
177 ixgbe_attach, /* devo_attach */
178 ixgbe_detach, /* devo_detach */
179 nodev, /* devo_reset */
180 &ixgbe_cb_ops, /* devo_cb_ops */
181 NULL, /* devo_bus_ops */
182 ddi_power, /* devo_power */
183 ixgbe_quiesce, /* devo_quiesce */
184 };
185
186 static struct modldrv ixgbe_modldrv = {
187 &mod_driverops, /* Type of module. This one is a driver */
188 ixgbe_ident, /* Discription string */
189 &ixgbe_dev_ops /* driver ops */
190 };
191
192 static struct modlinkage ixgbe_modlinkage = {
193 MODREV_1, &ixgbe_modldrv, NULL
194 };
195
196 /*
197 * Access attributes for register mapping
198 */
199 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
200 DDI_DEVICE_ATTR_V1,
201 DDI_STRUCTURE_LE_ACC,
202 DDI_STRICTORDER_ACC,
203 DDI_FLAGERR_ACC
204 };
205
206 /*
207 * Loopback property
208 */
209 static lb_property_t lb_normal = {
210 normal, "normal", IXGBE_LB_NONE
211 };
212
213 static lb_property_t lb_mac = {
214 internal, "MAC", IXGBE_LB_INTERNAL_MAC
215 };
216
217 static lb_property_t lb_external = {
218 external, "External", IXGBE_LB_EXTERNAL
219 };
220
221 #define IXGBE_M_CALLBACK_FLAGS \
222 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
223
224 static mac_callbacks_t ixgbe_m_callbacks = {
225 IXGBE_M_CALLBACK_FLAGS,
226 ixgbe_m_stat,
227 ixgbe_m_start,
228 ixgbe_m_stop,
229 ixgbe_m_promisc,
230 ixgbe_m_multicst,
231 NULL,
232 NULL,
233 NULL,
234 ixgbe_m_ioctl,
235 ixgbe_m_getcapab,
236 NULL,
237 NULL,
238 ixgbe_m_setprop,
239 ixgbe_m_getprop,
240 ixgbe_m_propinfo
241 };
242
243 /*
244 * Initialize capabilities of each supported adapter type
245 */
246 static adapter_info_t ixgbe_82598eb_cap = {
247 64, /* maximum number of rx queues */
248 1, /* minimum number of rx queues */
249 64, /* default number of rx queues */
250 16, /* maximum number of rx groups */
251 1, /* minimum number of rx groups */
252 1, /* default number of rx groups */
253 32, /* maximum number of tx queues */
254 1, /* minimum number of tx queues */
255 8, /* default number of tx queues */
256 16366, /* maximum MTU size */
257 0xFFFF, /* maximum interrupt throttle rate */
258 0, /* minimum interrupt throttle rate */
259 200, /* default interrupt throttle rate */
260 18, /* maximum total msix vectors */
261 16, /* maximum number of ring vectors */
262 2, /* maximum number of other vectors */
263 IXGBE_EICR_LSC, /* "other" interrupt types handled */
264 0, /* "other" interrupt types enable mask */
265 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
266 | IXGBE_FLAG_RSS_CAPABLE
267 | IXGBE_FLAG_VMDQ_CAPABLE)
268 };
269
270 static adapter_info_t ixgbe_82599eb_cap = {
271 128, /* maximum number of rx queues */
272 1, /* minimum number of rx queues */
273 128, /* default number of rx queues */
274 64, /* maximum number of rx groups */
275 1, /* minimum number of rx groups */
276 1, /* default number of rx groups */
277 128, /* maximum number of tx queues */
278 1, /* minimum number of tx queues */
279 8, /* default number of tx queues */
280 15500, /* maximum MTU size */
281 0xFF8, /* maximum interrupt throttle rate */
282 0, /* minimum interrupt throttle rate */
283 200, /* default interrupt throttle rate */
284 64, /* maximum total msix vectors */
285 16, /* maximum number of ring vectors */
286 2, /* maximum number of other vectors */
287 (IXGBE_EICR_LSC
288 | IXGBE_EICR_GPI_SDP1
289 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
290
291 (IXGBE_SDP1_GPIEN
292 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
293
294 (IXGBE_FLAG_DCA_CAPABLE
295 | IXGBE_FLAG_RSS_CAPABLE
296 | IXGBE_FLAG_VMDQ_CAPABLE
297 | IXGBE_FLAG_RSC_CAPABLE
298 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
299 };
300
301 static adapter_info_t ixgbe_X540_cap = {
302 128, /* maximum number of rx queues */
303 1, /* minimum number of rx queues */
304 128, /* default number of rx queues */
305 64, /* maximum number of rx groups */
306 1, /* minimum number of rx groups */
307 1, /* default number of rx groups */
308 128, /* maximum number of tx queues */
309 1, /* minimum number of tx queues */
310 8, /* default number of tx queues */
311 15500, /* maximum MTU size */
312 0xFF8, /* maximum interrupt throttle rate */
313 0, /* minimum interrupt throttle rate */
314 200, /* default interrupt throttle rate */
315 64, /* maximum total msix vectors */
316 16, /* maximum number of ring vectors */
317 2, /* maximum number of other vectors */
318 (IXGBE_EICR_LSC
319 | IXGBE_EICR_GPI_SDP1_X540
320 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
321
322 (IXGBE_SDP1_GPIEN_X540
323 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
324
325 (IXGBE_FLAG_DCA_CAPABLE
326 | IXGBE_FLAG_RSS_CAPABLE
327 | IXGBE_FLAG_VMDQ_CAPABLE
328 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
329 };
330
331 static adapter_info_t ixgbe_X550_cap = {
332 128, /* maximum number of rx queues */
333 1, /* minimum number of rx queues */
334 128, /* default number of rx queues */
335 64, /* maximum number of rx groups */
336 1, /* minimum number of rx groups */
337 1, /* default number of rx groups */
338 128, /* maximum number of tx queues */
339 1, /* minimum number of tx queues */
340 8, /* default number of tx queues */
341 15500, /* maximum MTU size */
342 0xFF8, /* maximum interrupt throttle rate */
343 0, /* minimum interrupt throttle rate */
344 0x200, /* default interrupt throttle rate */
345 64, /* maximum total msix vectors */
346 16, /* maximum number of ring vectors */
347 2, /* maximum number of other vectors */
348 IXGBE_EICR_LSC, /* "other" interrupt types handled */
349 0, /* "other" interrupt types enable mask */
350 (IXGBE_FLAG_RSS_CAPABLE
351 | IXGBE_FLAG_VMDQ_CAPABLE
352 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
353 };
354
355 /*
356 * Module Initialization Functions.
357 */
358
359 int
360 _init(void)
361 {
362 int status;
363
364 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
365
366 status = mod_install(&ixgbe_modlinkage);
367
368 if (status != DDI_SUCCESS) {
369 mac_fini_ops(&ixgbe_dev_ops);
370 }
371
372 return (status);
373 }
374
375 int
376 _fini(void)
377 {
378 int status;
379
380 status = mod_remove(&ixgbe_modlinkage);
381
382 if (status == DDI_SUCCESS) {
383 mac_fini_ops(&ixgbe_dev_ops);
384 }
385
386 return (status);
387 }
388
389 int
390 _info(struct modinfo *modinfop)
391 {
392 int status;
393
394 status = mod_info(&ixgbe_modlinkage, modinfop);
395
396 return (status);
397 }
398
399 /*
400 * ixgbe_attach - Driver attach.
401 *
402 * This function is the device specific initialization entry
403 * point. This entry point is required and must be written.
404 * The DDI_ATTACH command must be provided in the attach entry
405 * point. When attach() is called with cmd set to DDI_ATTACH,
406 * all normal kernel services (such as kmem_alloc(9F)) are
407 * available for use by the driver.
408 *
409 * The attach() function will be called once for each instance
410 * of the device on the system with cmd set to DDI_ATTACH.
411 * Until attach() succeeds, the only driver entry points which
412 * may be called are open(9E) and getinfo(9E).
413 */
414 static int
415 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
416 {
417 ixgbe_t *ixgbe;
418 struct ixgbe_osdep *osdep;
419 struct ixgbe_hw *hw;
420 int instance;
421 char taskqname[32];
422
423 /*
424 * Check the command and perform corresponding operations
425 */
426 switch (cmd) {
427 default:
428 return (DDI_FAILURE);
429
430 case DDI_RESUME:
431 return (ixgbe_resume(devinfo));
432
433 case DDI_ATTACH:
434 break;
435 }
436
437 /* Get the device instance */
438 instance = ddi_get_instance(devinfo);
439
440 /* Allocate memory for the instance data structure */
441 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
442
443 ixgbe->dip = devinfo;
444 ixgbe->instance = instance;
445
446 hw = &ixgbe->hw;
447 osdep = &ixgbe->osdep;
448 hw->back = osdep;
449 osdep->ixgbe = ixgbe;
450
451 /* Attach the instance pointer to the dev_info data structure */
452 ddi_set_driver_private(devinfo, ixgbe);
453
454 /*
455 * Initialize for FMA support
456 */
457 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
458 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
459 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
460 ixgbe_fm_init(ixgbe);
461 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
462
463 /*
464 * Map PCI config space registers
465 */
466 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
467 ixgbe_error(ixgbe, "Failed to map PCI configurations");
468 goto attach_fail;
469 }
470 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
471
472 /*
473 * Identify the chipset family
474 */
475 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
476 ixgbe_error(ixgbe, "Failed to identify hardware");
477 goto attach_fail;
478 }
479
480 /*
481 * Map device registers
482 */
483 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
484 ixgbe_error(ixgbe, "Failed to map device registers");
485 goto attach_fail;
486 }
487 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
488
489 /*
490 * Initialize driver parameters
491 */
492 ixgbe_init_properties(ixgbe);
493 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
494
495 /*
496 * Register interrupt callback
497 */
498 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
499 ixgbe_error(ixgbe, "Failed to register interrupt callback");
500 goto attach_fail;
501 }
502
503 /*
504 * Allocate interrupts
505 */
506 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
507 ixgbe_error(ixgbe, "Failed to allocate interrupts");
508 goto attach_fail;
509 }
510 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
511
512 /*
513 * Allocate rx/tx rings based on the ring numbers.
514 * The actual numbers of rx/tx rings are decided by the number of
515 * allocated interrupt vectors, so we should allocate the rings after
516 * interrupts are allocated.
517 */
518 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
519 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
520 goto attach_fail;
521 }
522 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
523
524 /*
525 * Map rings to interrupt vectors
526 */
527 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
528 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
529 goto attach_fail;
530 }
531
532 /*
533 * Add interrupt handlers
534 */
535 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
536 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
537 goto attach_fail;
538 }
539 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
540
541 /*
542 * Create a taskq for sfp-change
543 */
544 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
545 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
546 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
547 ixgbe_error(ixgbe, "sfp_taskq create failed");
548 goto attach_fail;
549 }
550 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
551
552 /*
553 * Create a taskq for over-temp
554 */
555 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
556 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
557 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
558 ixgbe_error(ixgbe, "overtemp_taskq create failed");
559 goto attach_fail;
560 }
561 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
562
563 /*
564 * Create a taskq for processing external PHY interrupts
565 */
566 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance);
567 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname,
568 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
569 ixgbe_error(ixgbe, "phy_taskq create failed");
570 goto attach_fail;
571 }
572 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ;
573
574 /*
575 * Initialize driver parameters
576 */
577 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
578 ixgbe_error(ixgbe, "Failed to initialize driver settings");
579 goto attach_fail;
580 }
581
582 /*
583 * Initialize mutexes for this device.
584 * Do this before enabling the interrupt handler and
585 * register the softint to avoid the condition where
586 * interrupt handler can try using uninitialized mutex.
587 */
588 ixgbe_init_locks(ixgbe);
589 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
590
591 /*
592 * Initialize chipset hardware
593 */
594 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
595 ixgbe_error(ixgbe, "Failed to initialize adapter");
596 goto attach_fail;
597 }
598 ixgbe->link_check_complete = B_FALSE;
599 ixgbe->link_check_hrtime = gethrtime() +
600 (IXGBE_LINK_UP_TIME * 100000000ULL);
601 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
602
603 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
604 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
605 goto attach_fail;
606 }
607
608 /*
609 * Initialize adapter capabilities
610 */
611 ixgbe_init_params(ixgbe);
612
613 /*
614 * Initialize statistics
615 */
616 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
617 ixgbe_error(ixgbe, "Failed to initialize statistics");
618 goto attach_fail;
619 }
620 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
621
622 /*
623 * Register the driver to the MAC
624 */
625 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
626 ixgbe_error(ixgbe, "Failed to register MAC");
627 goto attach_fail;
628 }
629 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
630 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
631
632 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
633 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
634 if (ixgbe->periodic_id == 0) {
635 ixgbe_error(ixgbe, "Failed to add the link check timer");
636 goto attach_fail;
637 }
638 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
639
640 /*
641 * Now that mutex locks are initialized, and the chip is also
642 * initialized, enable interrupts.
643 */
644 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
645 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
646 goto attach_fail;
647 }
648 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
649
650 ixgbe_log(ixgbe, "%s", ixgbe_ident);
651 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
652
653 return (DDI_SUCCESS);
654
655 attach_fail:
656 ixgbe_unconfigure(devinfo, ixgbe);
657 return (DDI_FAILURE);
658 }
659
660 /*
661 * ixgbe_detach - Driver detach.
662 *
663 * The detach() function is the complement of the attach routine.
664 * If cmd is set to DDI_DETACH, detach() is used to remove the
665 * state associated with a given instance of a device node
666 * prior to the removal of that instance from the system.
667 *
668 * The detach() function will be called once for each instance
669 * of the device for which there has been a successful attach()
670 * once there are no longer any opens on the device.
671 *
672 * Interrupts routine are disabled, All memory allocated by this
673 * driver are freed.
674 */
675 static int
676 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
677 {
678 ixgbe_t *ixgbe;
679
680 /*
681 * Check detach command
682 */
683 switch (cmd) {
684 default:
685 return (DDI_FAILURE);
686
687 case DDI_SUSPEND:
688 return (ixgbe_suspend(devinfo));
689
690 case DDI_DETACH:
691 break;
692 }
693
694 /*
695 * Get the pointer to the driver private data structure
696 */
697 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
698 if (ixgbe == NULL)
699 return (DDI_FAILURE);
700
701 /*
702 * If the device is still running, it needs to be stopped first.
703 * This check is necessary because under some specific circumstances,
704 * the detach routine can be called without stopping the interface
705 * first.
706 */
707 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
708 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
709 mutex_enter(&ixgbe->gen_lock);
710 ixgbe_stop(ixgbe, B_TRUE);
711 mutex_exit(&ixgbe->gen_lock);
712 /* Disable and stop the watchdog timer */
713 ixgbe_disable_watchdog_timer(ixgbe);
714 }
715
716 /*
717 * Check if there are still rx buffers held by the upper layer.
718 * If so, fail the detach.
719 */
720 if (!ixgbe_rx_drain(ixgbe))
721 return (DDI_FAILURE);
722
723 /*
724 * Do the remaining unconfigure routines
725 */
726 ixgbe_unconfigure(devinfo, ixgbe);
727
728 return (DDI_SUCCESS);
729 }
730
731 /*
732 * quiesce(9E) entry point.
733 *
734 * This function is called when the system is single-threaded at high
735 * PIL with preemption disabled. Therefore, this function must not be
736 * blocked.
737 *
738 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
739 * DDI_FAILURE indicates an error condition and should almost never happen.
740 */
741 static int
742 ixgbe_quiesce(dev_info_t *devinfo)
743 {
744 ixgbe_t *ixgbe;
745 struct ixgbe_hw *hw;
746
747 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
748
749 if (ixgbe == NULL)
750 return (DDI_FAILURE);
751
752 hw = &ixgbe->hw;
753
754 /*
755 * Disable the adapter interrupts
756 */
757 ixgbe_disable_adapter_interrupts(ixgbe);
758
759 /*
760 * Tell firmware driver is no longer in control
761 */
762 ixgbe_release_driver_control(hw);
763
764 /*
765 * Reset the chipset
766 */
767 (void) ixgbe_reset_hw(hw);
768
769 /*
770 * Reset PHY
771 */
772 (void) ixgbe_reset_phy(hw);
773
774 return (DDI_SUCCESS);
775 }
776
777 static void
778 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
779 {
780 /*
781 * Disable interrupt
782 */
783 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
784 (void) ixgbe_disable_intrs(ixgbe);
785 }
786
787 /*
788 * remove the link check timer
789 */
790 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
791 if (ixgbe->periodic_id != NULL) {
792 ddi_periodic_delete(ixgbe->periodic_id);
793 ixgbe->periodic_id = NULL;
794 }
795 }
796
797 /*
798 * Unregister MAC
799 */
800 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
801 (void) mac_unregister(ixgbe->mac_hdl);
802 }
803
804 /*
805 * Free statistics
806 */
807 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
808 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
809 }
810
811 /*
812 * Remove interrupt handlers
813 */
814 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
815 ixgbe_rem_intr_handlers(ixgbe);
816 }
817
818 /*
819 * Remove taskq for sfp-status-change
820 */
821 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
822 ddi_taskq_destroy(ixgbe->sfp_taskq);
823 }
824
825 /*
826 * Remove taskq for over-temp
827 */
828 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
829 ddi_taskq_destroy(ixgbe->overtemp_taskq);
830 }
831
832 /*
833 * Remove taskq for external PHYs
834 */
835 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) {
836 ddi_taskq_destroy(ixgbe->phy_taskq);
837 }
838
839 /*
840 * Remove interrupts
841 */
842 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
843 ixgbe_rem_intrs(ixgbe);
844 }
845
846 /*
847 * Unregister interrupt callback handler
848 */
849 if (ixgbe->cb_hdl != NULL) {
850 (void) ddi_cb_unregister(ixgbe->cb_hdl);
851 }
852
853 /*
854 * Remove driver properties
855 */
856 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
857 (void) ddi_prop_remove_all(devinfo);
858 }
859
860 /*
861 * Stop the chipset
862 */
863 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
864 mutex_enter(&ixgbe->gen_lock);
865 ixgbe_chip_stop(ixgbe);
866 mutex_exit(&ixgbe->gen_lock);
867 }
868
869 /*
870 * Free register handle
871 */
872 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
873 if (ixgbe->osdep.reg_handle != NULL)
874 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
875 }
876
877 /*
878 * Free PCI config handle
879 */
880 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
881 if (ixgbe->osdep.cfg_handle != NULL)
882 pci_config_teardown(&ixgbe->osdep.cfg_handle);
883 }
884
885 /*
886 * Free locks
887 */
888 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
889 ixgbe_destroy_locks(ixgbe);
890 }
891
892 /*
893 * Free the rx/tx rings
894 */
895 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
896 ixgbe_free_rings(ixgbe);
897 }
898
899 /*
900 * Unregister FMA capabilities
901 */
902 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
903 ixgbe_fm_fini(ixgbe);
904 }
905
906 /*
907 * Free the driver data structure
908 */
909 kmem_free(ixgbe, sizeof (ixgbe_t));
910
911 ddi_set_driver_private(devinfo, NULL);
912 }
913
914 /*
915 * ixgbe_register_mac - Register the driver and its function pointers with
916 * the GLD interface.
917 */
918 static int
919 ixgbe_register_mac(ixgbe_t *ixgbe)
920 {
921 struct ixgbe_hw *hw = &ixgbe->hw;
922 mac_register_t *mac;
923 int status;
924
925 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
926 return (IXGBE_FAILURE);
927
928 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
929 mac->m_driver = ixgbe;
930 mac->m_dip = ixgbe->dip;
931 mac->m_src_addr = hw->mac.addr;
932 mac->m_callbacks = &ixgbe_m_callbacks;
933 mac->m_min_sdu = 0;
934 mac->m_max_sdu = ixgbe->default_mtu;
935 mac->m_margin = VLAN_TAGSZ;
936 mac->m_priv_props = ixgbe_priv_props;
937 mac->m_v12n = MAC_VIRT_LEVEL1;
938
939 status = mac_register(mac, &ixgbe->mac_hdl);
940
941 mac_free(mac);
942
943 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
944 }
945
946 /*
947 * ixgbe_identify_hardware - Identify the type of the chipset.
948 */
949 static int
950 ixgbe_identify_hardware(ixgbe_t *ixgbe)
951 {
952 struct ixgbe_hw *hw = &ixgbe->hw;
953 struct ixgbe_osdep *osdep = &ixgbe->osdep;
954
955 /*
956 * Get the device id
957 */
958 hw->vendor_id =
959 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
960 hw->device_id =
961 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
962 hw->revision_id =
963 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
964 hw->subsystem_device_id =
965 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
966 hw->subsystem_vendor_id =
967 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
968
969 /*
970 * Set the mac type of the adapter based on the device id
971 */
972 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
973 return (IXGBE_FAILURE);
974 }
975
976 /*
977 * Install adapter capabilities
978 */
979 switch (hw->mac.type) {
980 case ixgbe_mac_82598EB:
981 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
982 ixgbe->capab = &ixgbe_82598eb_cap;
983
984 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
985 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
986 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
987 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
988 }
989 break;
990
991 case ixgbe_mac_82599EB:
992 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
993 ixgbe->capab = &ixgbe_82599eb_cap;
994
995 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
996 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
997 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
998 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
999 }
1000 break;
1001
1002 case ixgbe_mac_X540:
1003 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
1004 ixgbe->capab = &ixgbe_X540_cap;
1005 /*
1006 * For now, X540 is all set in its capab structure.
1007 * As other X540 variants show up, things can change here.
1008 */
1009 break;
1010
1011 case ixgbe_mac_X550:
1012 case ixgbe_mac_X550EM_x:
1013 case ixgbe_mac_X550EM_a:
1014 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
1015 ixgbe->capab = &ixgbe_X550_cap;
1016
1017 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1018 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
1019 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
1020 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP ||
1021 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP_N) {
1022 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
1023 }
1024
1025 /*
1026 * Link detection on X552 SFP+ and X552/X557-AT
1027 */
1028 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1029 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
1030 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
1031 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
1032 ixgbe->capab->other_intr |=
1033 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
1034 }
1035 if (hw->phy.type == ixgbe_phy_x550em_ext_t) {
1036 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540;
1037 }
1038 break;
1039
1040 default:
1041 IXGBE_DEBUGLOG_1(ixgbe,
1042 "adapter not supported in ixgbe_identify_hardware(): %d\n",
1043 hw->mac.type);
1044 return (IXGBE_FAILURE);
1045 }
1046
1047 return (IXGBE_SUCCESS);
1048 }
1049
1050 /*
1051 * ixgbe_regs_map - Map the device registers.
1052 *
1053 */
1054 static int
1055 ixgbe_regs_map(ixgbe_t *ixgbe)
1056 {
1057 dev_info_t *devinfo = ixgbe->dip;
1058 struct ixgbe_hw *hw = &ixgbe->hw;
1059 struct ixgbe_osdep *osdep = &ixgbe->osdep;
1060 off_t mem_size;
1061
1062 /*
1063 * First get the size of device registers to be mapped.
1064 */
1065 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
1066 != DDI_SUCCESS) {
1067 return (IXGBE_FAILURE);
1068 }
1069
1070 /*
1071 * Call ddi_regs_map_setup() to map registers
1072 */
1073 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
1074 (caddr_t *)&hw->hw_addr, 0,
1075 mem_size, &ixgbe_regs_acc_attr,
1076 &osdep->reg_handle)) != DDI_SUCCESS) {
1077 return (IXGBE_FAILURE);
1078 }
1079
1080 return (IXGBE_SUCCESS);
1081 }
1082
1083 /*
1084 * ixgbe_init_properties - Initialize driver properties.
1085 */
1086 static void
1087 ixgbe_init_properties(ixgbe_t *ixgbe)
1088 {
1089 /*
1090 * Get conf file properties, including link settings
1091 * jumbo frames, ring number, descriptor number, etc.
1092 */
1093 ixgbe_get_conf(ixgbe);
1094 }
1095
1096 /*
1097 * ixgbe_init_driver_settings - Initialize driver settings.
1098 *
1099 * The settings include hardware function pointers, bus information,
1100 * rx/tx rings settings, link state, and any other parameters that
1101 * need to be setup during driver initialization.
1102 */
1103 static int
1104 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1105 {
1106 struct ixgbe_hw *hw = &ixgbe->hw;
1107 dev_info_t *devinfo = ixgbe->dip;
1108 ixgbe_rx_ring_t *rx_ring;
1109 ixgbe_rx_group_t *rx_group;
1110 ixgbe_tx_ring_t *tx_ring;
1111 uint32_t rx_size;
1112 uint32_t tx_size;
1113 uint32_t ring_per_group;
1114 int i;
1115
1116 /*
1117 * Initialize chipset specific hardware function pointers
1118 */
1119 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1120 return (IXGBE_FAILURE);
1121 }
1122
1123 /*
1124 * Get the system page size
1125 */
1126 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1127
1128 /*
1129 * Set rx buffer size
1130 *
1131 * The IP header alignment room is counted in the calculation.
1132 * The rx buffer size is in unit of 1K that is required by the
1133 * chipset hardware.
1134 */
1135 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1136 ixgbe->rx_buf_size = ((rx_size >> 10) +
1137 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1138
1139 /*
1140 * Set tx buffer size
1141 */
1142 tx_size = ixgbe->max_frame_size;
1143 ixgbe->tx_buf_size = ((tx_size >> 10) +
1144 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1145
1146 /*
1147 * Initialize rx/tx rings/groups parameters
1148 */
1149 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1150 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1151 rx_ring = &ixgbe->rx_rings[i];
1152 rx_ring->index = i;
1153 rx_ring->ixgbe = ixgbe;
1154 rx_ring->group_index = i / ring_per_group;
1155 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1156 }
1157
1158 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1159 rx_group = &ixgbe->rx_groups[i];
1160 rx_group->index = i;
1161 rx_group->ixgbe = ixgbe;
1162 }
1163
1164 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1165 tx_ring = &ixgbe->tx_rings[i];
1166 tx_ring->index = i;
1167 tx_ring->ixgbe = ixgbe;
1168 if (ixgbe->tx_head_wb_enable)
1169 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1170 else
1171 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1172
1173 tx_ring->ring_size = ixgbe->tx_ring_size;
1174 tx_ring->free_list_size = ixgbe->tx_ring_size +
1175 (ixgbe->tx_ring_size >> 1);
1176 }
1177
1178 /*
1179 * Initialize values of interrupt throttling rate
1180 */
1181 for (i = 1; i < MAX_INTR_VECTOR; i++)
1182 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1183
1184 /*
1185 * The initial link state should be "unknown"
1186 */
1187 ixgbe->link_state = LINK_STATE_UNKNOWN;
1188
1189 return (IXGBE_SUCCESS);
1190 }
1191
1192 /*
1193 * ixgbe_init_locks - Initialize locks.
1194 */
1195 static void
1196 ixgbe_init_locks(ixgbe_t *ixgbe)
1197 {
1198 ixgbe_rx_ring_t *rx_ring;
1199 ixgbe_tx_ring_t *tx_ring;
1200 int i;
1201
1202 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1203 rx_ring = &ixgbe->rx_rings[i];
1204 mutex_init(&rx_ring->rx_lock, NULL,
1205 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1206 }
1207
1208 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1209 tx_ring = &ixgbe->tx_rings[i];
1210 mutex_init(&tx_ring->tx_lock, NULL,
1211 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1212 mutex_init(&tx_ring->recycle_lock, NULL,
1213 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1214 mutex_init(&tx_ring->tcb_head_lock, NULL,
1215 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1216 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1217 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1218 }
1219
1220 mutex_init(&ixgbe->gen_lock, NULL,
1221 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1222
1223 mutex_init(&ixgbe->watchdog_lock, NULL,
1224 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1225 }
1226
1227 /*
1228 * ixgbe_destroy_locks - Destroy locks.
1229 */
1230 static void
1231 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1232 {
1233 ixgbe_rx_ring_t *rx_ring;
1234 ixgbe_tx_ring_t *tx_ring;
1235 int i;
1236
1237 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1238 rx_ring = &ixgbe->rx_rings[i];
1239 mutex_destroy(&rx_ring->rx_lock);
1240 }
1241
1242 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1243 tx_ring = &ixgbe->tx_rings[i];
1244 mutex_destroy(&tx_ring->tx_lock);
1245 mutex_destroy(&tx_ring->recycle_lock);
1246 mutex_destroy(&tx_ring->tcb_head_lock);
1247 mutex_destroy(&tx_ring->tcb_tail_lock);
1248 }
1249
1250 mutex_destroy(&ixgbe->gen_lock);
1251 mutex_destroy(&ixgbe->watchdog_lock);
1252 }
1253
1254 /*
1255 * We need to try and determine which LED index in hardware corresponds to the
1256 * link/activity LED. This is the one that'll be overwritten when we perform
1257 * GLDv3 LED activity.
1258 */
1259 static void
1260 ixgbe_led_init(ixgbe_t *ixgbe)
1261 {
1262 uint32_t reg, i;
1263 struct ixgbe_hw *hw = &ixgbe->hw;
1264
1265 reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL);
1266 for (i = 0; i < 4; i++) {
1267 if (((reg >> IXGBE_LED_MODE_SHIFT(i)) &
1268 IXGBE_LED_MODE_MASK_BASE) == IXGBE_LED_LINK_ACTIVE) {
1269 ixgbe->ixgbe_led_index = i;
1270 return;
1271 }
1272 }
1273
1274 /*
1275 * If we couldn't determine this, we use the default for various MACs
1276 * based on information Intel has inserted into other drivers over the
1277 * years.
1278 */
1279 switch (hw->mac.type) {
1280 case ixgbe_mac_X550EM_a:
1281 ixgbe->ixgbe_led_index = 0;
1282 break;
1283 case ixgbe_mac_X550EM_x:
1284 ixgbe->ixgbe_led_index = 1;
1285 break;
1286 default:
1287 ixgbe->ixgbe_led_index = 2;
1288 break;
1289 }
1290 }
1291
1292 static int
1293 ixgbe_resume(dev_info_t *devinfo)
1294 {
1295 ixgbe_t *ixgbe;
1296 int i;
1297
1298 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1299 if (ixgbe == NULL)
1300 return (DDI_FAILURE);
1301
1302 mutex_enter(&ixgbe->gen_lock);
1303
1304 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1305 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1306 mutex_exit(&ixgbe->gen_lock);
1307 return (DDI_FAILURE);
1308 }
1309
1310 /*
1311 * Enable and start the watchdog timer
1312 */
1313 ixgbe_enable_watchdog_timer(ixgbe);
1314 }
1315
1316 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1317
1318 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1319 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1320 mac_tx_ring_update(ixgbe->mac_hdl,
1321 ixgbe->tx_rings[i].ring_handle);
1322 }
1323 }
1324
1325 mutex_exit(&ixgbe->gen_lock);
1326
1327 return (DDI_SUCCESS);
1328 }
1329
1330 static int
1331 ixgbe_suspend(dev_info_t *devinfo)
1332 {
1333 ixgbe_t *ixgbe;
1334
1335 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1336 if (ixgbe == NULL)
1337 return (DDI_FAILURE);
1338
1339 mutex_enter(&ixgbe->gen_lock);
1340
1341 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1342 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1343 mutex_exit(&ixgbe->gen_lock);
1344 return (DDI_SUCCESS);
1345 }
1346 ixgbe_stop(ixgbe, B_FALSE);
1347
1348 mutex_exit(&ixgbe->gen_lock);
1349
1350 /*
1351 * Disable and stop the watchdog timer
1352 */
1353 ixgbe_disable_watchdog_timer(ixgbe);
1354
1355 return (DDI_SUCCESS);
1356 }
1357
1358 /*
1359 * ixgbe_init - Initialize the device.
1360 */
1361 static int
1362 ixgbe_init(ixgbe_t *ixgbe)
1363 {
1364 struct ixgbe_hw *hw = &ixgbe->hw;
1365 u8 pbanum[IXGBE_PBANUM_LENGTH];
1366 int rv;
1367
1368 mutex_enter(&ixgbe->gen_lock);
1369
1370 /*
1371 * Configure/Initialize hardware
1372 */
1373 rv = ixgbe_init_hw(hw);
1374 if (rv != IXGBE_SUCCESS) {
1375 switch (rv) {
1376
1377 /*
1378 * The first three errors are not prohibitive to us progressing
1379 * further, and are maily advisory in nature. In the case of a
1380 * SFP module not being present or not deemed supported by the
1381 * common code, we adivse the operator of this fact but carry on
1382 * instead of failing hard, as SFPs can be inserted or replaced
1383 * while the driver is running. In the case of a unknown error,
1384 * we fail-hard, logging the reason and emitting a FMA event.
1385 */
1386 case IXGBE_ERR_EEPROM_VERSION:
1387 ixgbe_error(ixgbe,
1388 "This Intel 10Gb Ethernet device is pre-release and"
1389 " contains outdated firmware. Please contact your"
1390 " hardware vendor for a replacement.");
1391 break;
1392 case IXGBE_ERR_SFP_NOT_PRESENT:
1393 ixgbe_error(ixgbe,
1394 "No SFP+ module detected on this interface. Please "
1395 "install a supported SFP+ module for this "
1396 "interface to become operational.");
1397 break;
1398 case IXGBE_ERR_SFP_NOT_SUPPORTED:
1399 ixgbe_error(ixgbe,
1400 "Unsupported SFP+ module detected. Please replace "
1401 "it with a supported SFP+ module per Intel "
1402 "documentation, or bypass this check with "
1403 "allow_unsupported_sfp=1 in ixgbe.conf.");
1404 break;
1405 default:
1406 ixgbe_error(ixgbe,
1407 "Failed to initialize hardware. ixgbe_init_hw "
1408 "returned %d", rv);
1409 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1410 goto init_fail;
1411 }
1412 }
1413
1414 /*
1415 * Need to init eeprom before validating the checksum.
1416 */
1417 if (ixgbe_init_eeprom_params(hw) < 0) {
1418 ixgbe_error(ixgbe,
1419 "Unable to intitialize the eeprom interface.");
1420 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1421 goto init_fail;
1422 }
1423
1424 /*
1425 * NVM validation
1426 */
1427 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1428 /*
1429 * Some PCI-E parts fail the first check due to
1430 * the link being in sleep state. Call it again,
1431 * if it fails a second time it's a real issue.
1432 */
1433 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1434 ixgbe_error(ixgbe,
1435 "Invalid NVM checksum. Please contact "
1436 "the vendor to update the NVM.");
1437 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1438 goto init_fail;
1439 }
1440 }
1441
1442 /*
1443 * Setup default flow control thresholds - enable/disable
1444 * & flow control type is controlled by ixgbe.conf
1445 */
1446 hw->fc.high_water[0] = DEFAULT_FCRTH;
1447 hw->fc.low_water[0] = DEFAULT_FCRTL;
1448 hw->fc.pause_time = DEFAULT_FCPAUSE;
1449 hw->fc.send_xon = B_TRUE;
1450
1451 /*
1452 * Initialize flow control
1453 */
1454 (void) ixgbe_start_hw(hw);
1455
1456 /*
1457 * Initialize link settings
1458 */
1459 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1460
1461 /*
1462 * Initialize the chipset hardware
1463 */
1464 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1465 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1466 goto init_fail;
1467 }
1468
1469 /*
1470 * Read identifying information and place in devinfo.
1471 */
1472 pbanum[0] = '\0';
1473 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1474 if (*pbanum != '\0') {
1475 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1476 "printed-board-assembly", (char *)pbanum);
1477 }
1478
1479 /*
1480 * Determine LED index.
1481 */
1482 ixgbe_led_init(ixgbe);
1483
1484 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1485 goto init_fail;
1486 }
1487
1488 mutex_exit(&ixgbe->gen_lock);
1489 return (IXGBE_SUCCESS);
1490
1491 init_fail:
1492 /*
1493 * Reset PHY
1494 */
1495 (void) ixgbe_reset_phy(hw);
1496
1497 mutex_exit(&ixgbe->gen_lock);
1498 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1499 return (IXGBE_FAILURE);
1500 }
1501
1502 /*
1503 * ixgbe_chip_start - Initialize and start the chipset hardware.
1504 */
1505 static int
1506 ixgbe_chip_start(ixgbe_t *ixgbe)
1507 {
1508 struct ixgbe_hw *hw = &ixgbe->hw;
1509 int i;
1510
1511 ASSERT(mutex_owned(&ixgbe->gen_lock));
1512
1513 /*
1514 * Get the mac address
1515 * This function should handle SPARC case correctly.
1516 */
1517 if (!ixgbe_find_mac_address(ixgbe)) {
1518 ixgbe_error(ixgbe, "Failed to get the mac address");
1519 return (IXGBE_FAILURE);
1520 }
1521
1522 /*
1523 * Validate the mac address
1524 */
1525 (void) ixgbe_init_rx_addrs(hw);
1526 if (!is_valid_mac_addr(hw->mac.addr)) {
1527 ixgbe_error(ixgbe, "Invalid mac address");
1528 return (IXGBE_FAILURE);
1529 }
1530
1531 /*
1532 * Re-enable relaxed ordering for performance. It is disabled
1533 * by default in the hardware init.
1534 */
1535 if (ixgbe->relax_order_enable == B_TRUE)
1536 ixgbe_enable_relaxed_ordering(hw);
1537
1538 /*
1539 * Setup adapter interrupt vectors
1540 */
1541 ixgbe_setup_adapter_vector(ixgbe);
1542
1543 /*
1544 * Initialize unicast addresses.
1545 */
1546 ixgbe_init_unicst(ixgbe);
1547
1548 /*
1549 * Setup and initialize the mctable structures.
1550 */
1551 ixgbe_setup_multicst(ixgbe);
1552
1553 /*
1554 * Set interrupt throttling rate
1555 */
1556 for (i = 0; i < ixgbe->intr_cnt; i++) {
1557 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1558 }
1559
1560 /*
1561 * Disable Wake-on-LAN
1562 */
1563 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
1564
1565 /*
1566 * Some adapters offer Energy Efficient Ethernet (EEE) support.
1567 * Due to issues with EEE in e1000g/igb, we disable this by default
1568 * as a precautionary measure.
1569 *
1570 * Currently, this is present on a number of the X550 family parts.
1571 */
1572 (void) ixgbe_setup_eee(hw, B_FALSE);
1573
1574 /*
1575 * Turn on any present SFP Tx laser
1576 */
1577 ixgbe_enable_tx_laser(hw);
1578
1579 /*
1580 * Power on the PHY
1581 */
1582 (void) ixgbe_set_phy_power(hw, B_TRUE);
1583
1584 /*
1585 * Save the state of the PHY
1586 */
1587 ixgbe_get_hw_state(ixgbe);
1588
1589 /*
1590 * Make sure driver has control
1591 */
1592 ixgbe_get_driver_control(hw);
1593
1594 return (IXGBE_SUCCESS);
1595 }
1596
1597 /*
1598 * ixgbe_chip_stop - Stop the chipset hardware
1599 */
1600 static void
1601 ixgbe_chip_stop(ixgbe_t *ixgbe)
1602 {
1603 struct ixgbe_hw *hw = &ixgbe->hw;
1604 int rv;
1605
1606 ASSERT(mutex_owned(&ixgbe->gen_lock));
1607
1608 /*
1609 * Stop interupt generation and disable Tx unit
1610 */
1611 hw->adapter_stopped = B_FALSE;
1612 (void) ixgbe_stop_adapter(hw);
1613
1614 /*
1615 * Reset the chipset
1616 */
1617 (void) ixgbe_reset_hw(hw);
1618
1619 /*
1620 * Reset PHY
1621 */
1622 (void) ixgbe_reset_phy(hw);
1623
1624 /*
1625 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting
1626 * the PHY while doing so. Else, just power down the PHY.
1627 */
1628 if (hw->phy.ops.enter_lplu != NULL) {
1629 hw->phy.reset_disable = B_TRUE;
1630 rv = hw->phy.ops.enter_lplu(hw);
1631 if (rv != IXGBE_SUCCESS)
1632 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv);
1633 hw->phy.reset_disable = B_FALSE;
1634 } else {
1635 (void) ixgbe_set_phy_power(hw, B_FALSE);
1636 }
1637
1638 /*
1639 * Turn off any present SFP Tx laser
1640 * Expected for health and safety reasons
1641 */
1642 ixgbe_disable_tx_laser(hw);
1643
1644 /*
1645 * Tell firmware driver is no longer in control
1646 */
1647 ixgbe_release_driver_control(hw);
1648
1649 }
1650
1651 /*
1652 * ixgbe_reset - Reset the chipset and re-start the driver.
1653 *
1654 * It involves stopping and re-starting the chipset,
1655 * and re-configuring the rx/tx rings.
1656 */
1657 static int
1658 ixgbe_reset(ixgbe_t *ixgbe)
1659 {
1660 int i;
1661
1662 /*
1663 * Disable and stop the watchdog timer
1664 */
1665 ixgbe_disable_watchdog_timer(ixgbe);
1666
1667 mutex_enter(&ixgbe->gen_lock);
1668
1669 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1670 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1671
1672 ixgbe_stop(ixgbe, B_FALSE);
1673
1674 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1675 mutex_exit(&ixgbe->gen_lock);
1676 return (IXGBE_FAILURE);
1677 }
1678
1679 /*
1680 * After resetting, need to recheck the link status.
1681 */
1682 ixgbe->link_check_complete = B_FALSE;
1683 ixgbe->link_check_hrtime = gethrtime() +
1684 (IXGBE_LINK_UP_TIME * 100000000ULL);
1685
1686 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1687
1688 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1689 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1690 mac_tx_ring_update(ixgbe->mac_hdl,
1691 ixgbe->tx_rings[i].ring_handle);
1692 }
1693 }
1694
1695 mutex_exit(&ixgbe->gen_lock);
1696
1697 /*
1698 * Enable and start the watchdog timer
1699 */
1700 ixgbe_enable_watchdog_timer(ixgbe);
1701
1702 return (IXGBE_SUCCESS);
1703 }
1704
1705 /*
1706 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1707 */
1708 static void
1709 ixgbe_tx_clean(ixgbe_t *ixgbe)
1710 {
1711 ixgbe_tx_ring_t *tx_ring;
1712 tx_control_block_t *tcb;
1713 link_list_t pending_list;
1714 uint32_t desc_num;
1715 int i, j;
1716
1717 LINK_LIST_INIT(&pending_list);
1718
1719 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1720 tx_ring = &ixgbe->tx_rings[i];
1721
1722 mutex_enter(&tx_ring->recycle_lock);
1723
1724 /*
1725 * Clean the pending tx data - the pending packets in the
1726 * work_list that have no chances to be transmitted again.
1727 *
1728 * We must ensure the chipset is stopped or the link is down
1729 * before cleaning the transmit packets.
1730 */
1731 desc_num = 0;
1732 for (j = 0; j < tx_ring->ring_size; j++) {
1733 tcb = tx_ring->work_list[j];
1734 if (tcb != NULL) {
1735 desc_num += tcb->desc_num;
1736
1737 tx_ring->work_list[j] = NULL;
1738
1739 ixgbe_free_tcb(tcb);
1740
1741 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1742 }
1743 }
1744
1745 if (desc_num > 0) {
1746 atomic_add_32(&tx_ring->tbd_free, desc_num);
1747 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1748
1749 /*
1750 * Reset the head and tail pointers of the tbd ring;
1751 * Reset the writeback head if it's enable.
1752 */
1753 tx_ring->tbd_head = 0;
1754 tx_ring->tbd_tail = 0;
1755 if (ixgbe->tx_head_wb_enable)
1756 *tx_ring->tbd_head_wb = 0;
1757
1758 IXGBE_WRITE_REG(&ixgbe->hw,
1759 IXGBE_TDH(tx_ring->index), 0);
1760 IXGBE_WRITE_REG(&ixgbe->hw,
1761 IXGBE_TDT(tx_ring->index), 0);
1762 }
1763
1764 mutex_exit(&tx_ring->recycle_lock);
1765
1766 /*
1767 * Add the tx control blocks in the pending list to
1768 * the free list.
1769 */
1770 ixgbe_put_free_list(tx_ring, &pending_list);
1771 }
1772 }
1773
1774 /*
1775 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1776 * transmitted.
1777 */
1778 static boolean_t
1779 ixgbe_tx_drain(ixgbe_t *ixgbe)
1780 {
1781 ixgbe_tx_ring_t *tx_ring;
1782 boolean_t done;
1783 int i, j;
1784
1785 /*
1786 * Wait for a specific time to allow pending tx packets
1787 * to be transmitted.
1788 *
1789 * Check the counter tbd_free to see if transmission is done.
1790 * No lock protection is needed here.
1791 *
1792 * Return B_TRUE if all pending packets have been transmitted;
1793 * Otherwise return B_FALSE;
1794 */
1795 for (i = 0; i < TX_DRAIN_TIME; i++) {
1796
1797 done = B_TRUE;
1798 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1799 tx_ring = &ixgbe->tx_rings[j];
1800 done = done &&
1801 (tx_ring->tbd_free == tx_ring->ring_size);
1802 }
1803
1804 if (done)
1805 break;
1806
1807 msec_delay(1);
1808 }
1809
1810 return (done);
1811 }
1812
1813 /*
1814 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1815 */
1816 static boolean_t
1817 ixgbe_rx_drain(ixgbe_t *ixgbe)
1818 {
1819 boolean_t done = B_TRUE;
1820 int i;
1821
1822 /*
1823 * Polling the rx free list to check if those rx buffers held by
1824 * the upper layer are released.
1825 *
1826 * Check the counter rcb_free to see if all pending buffers are
1827 * released. No lock protection is needed here.
1828 *
1829 * Return B_TRUE if all pending buffers have been released;
1830 * Otherwise return B_FALSE;
1831 */
1832 for (i = 0; i < RX_DRAIN_TIME; i++) {
1833 done = (ixgbe->rcb_pending == 0);
1834
1835 if (done)
1836 break;
1837
1838 msec_delay(1);
1839 }
1840
1841 return (done);
1842 }
1843
1844 /*
1845 * ixgbe_start - Start the driver/chipset.
1846 */
1847 int
1848 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1849 {
1850 struct ixgbe_hw *hw = &ixgbe->hw;
1851 int i;
1852
1853 ASSERT(mutex_owned(&ixgbe->gen_lock));
1854
1855 if (alloc_buffer) {
1856 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1857 ixgbe_error(ixgbe,
1858 "Failed to allocate software receive rings");
1859 return (IXGBE_FAILURE);
1860 }
1861
1862 /* Allocate buffers for all the rx/tx rings */
1863 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1864 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1865 return (IXGBE_FAILURE);
1866 }
1867
1868 ixgbe->tx_ring_init = B_TRUE;
1869 } else {
1870 ixgbe->tx_ring_init = B_FALSE;
1871 }
1872
1873 for (i = 0; i < ixgbe->num_rx_rings; i++)
1874 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1875 for (i = 0; i < ixgbe->num_tx_rings; i++)
1876 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1877
1878 /*
1879 * Start the chipset hardware
1880 */
1881 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1882 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1883 goto start_failure;
1884 }
1885
1886 /*
1887 * Configure link now for X550
1888 *
1889 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the
1890 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550,
1891 * the resting state of the link would be the maximum speed that
1892 * autonegotiation will allow (usually 10Gb, infrastructure allowing)
1893 * so we never bothered with explicitly setting the link to 10Gb as it
1894 * would already be at that state on driver attach. With X550, we must
1895 * trigger a re-negotiation of the link in order to switch from a LPLU
1896 * 1Gb link to 10Gb (cable and link partner permitting.)
1897 */
1898 if (hw->mac.type == ixgbe_mac_X550 ||
1899 hw->mac.type == ixgbe_mac_X550EM_a ||
1900 hw->mac.type == ixgbe_mac_X550EM_x) {
1901 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE);
1902 ixgbe_get_hw_state(ixgbe);
1903 }
1904
1905 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1906 goto start_failure;
1907 }
1908
1909 /*
1910 * Setup the rx/tx rings
1911 */
1912 ixgbe_setup_rings(ixgbe);
1913
1914 /*
1915 * ixgbe_start() will be called when resetting, however if reset
1916 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1917 * before enabling the interrupts.
1918 */
1919 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1920 | IXGBE_STALL| IXGBE_OVERTEMP));
1921
1922 /*
1923 * Enable adapter interrupts
1924 * The interrupts must be enabled after the driver state is START
1925 */
1926 ixgbe_enable_adapter_interrupts(ixgbe);
1927
1928 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1929 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1930 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1931 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1932
1933 return (IXGBE_SUCCESS);
1934
1935 start_failure:
1936 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1937 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1938 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1939 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1940
1941 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1942
1943 return (IXGBE_FAILURE);
1944 }
1945
1946 /*
1947 * ixgbe_stop - Stop the driver/chipset.
1948 */
1949 void
1950 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1951 {
1952 int i;
1953
1954 ASSERT(mutex_owned(&ixgbe->gen_lock));
1955
1956 /*
1957 * Disable the adapter interrupts
1958 */
1959 ixgbe_disable_adapter_interrupts(ixgbe);
1960
1961 /*
1962 * Drain the pending tx packets
1963 */
1964 (void) ixgbe_tx_drain(ixgbe);
1965
1966 for (i = 0; i < ixgbe->num_rx_rings; i++)
1967 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1968 for (i = 0; i < ixgbe->num_tx_rings; i++)
1969 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1970
1971 /*
1972 * Stop the chipset hardware
1973 */
1974 ixgbe_chip_stop(ixgbe);
1975
1976 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1977 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1978 }
1979
1980 /*
1981 * Clean the pending tx data/resources
1982 */
1983 ixgbe_tx_clean(ixgbe);
1984
1985 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1986 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1987 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1988 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1989
1990 if (ixgbe->link_state == LINK_STATE_UP) {
1991 ixgbe->link_state = LINK_STATE_UNKNOWN;
1992 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1993 }
1994
1995 if (free_buffer) {
1996 /*
1997 * Release the DMA/memory resources of rx/tx rings
1998 */
1999 ixgbe_free_dma(ixgbe);
2000 ixgbe_free_rx_data(ixgbe);
2001 }
2002 }
2003
2004 /*
2005 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
2006 */
2007 /* ARGSUSED */
2008 static int
2009 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
2010 void *arg1, void *arg2)
2011 {
2012 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
2013
2014 switch (cbaction) {
2015 /* IRM callback */
2016 int count;
2017 case DDI_CB_INTR_ADD:
2018 case DDI_CB_INTR_REMOVE:
2019 count = (int)(uintptr_t)cbarg;
2020 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
2021 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
2022 int, ixgbe->intr_cnt);
2023 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
2024 DDI_SUCCESS) {
2025 ixgbe_error(ixgbe,
2026 "IRM CB: Failed to adjust interrupts");
2027 goto cb_fail;
2028 }
2029 break;
2030 default:
2031 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
2032 cbaction);
2033 return (DDI_ENOTSUP);
2034 }
2035 return (DDI_SUCCESS);
2036 cb_fail:
2037 return (DDI_FAILURE);
2038 }
2039
2040 /*
2041 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
2042 */
2043 static int
2044 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
2045 {
2046 int i, rc, actual;
2047
2048 if (count == 0)
2049 return (DDI_SUCCESS);
2050
2051 if ((cbaction == DDI_CB_INTR_ADD &&
2052 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
2053 (cbaction == DDI_CB_INTR_REMOVE &&
2054 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
2055 return (DDI_FAILURE);
2056
2057 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
2058 return (DDI_FAILURE);
2059 }
2060
2061 for (i = 0; i < ixgbe->num_rx_rings; i++)
2062 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
2063 for (i = 0; i < ixgbe->num_tx_rings; i++)
2064 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
2065
2066 mutex_enter(&ixgbe->gen_lock);
2067 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
2068 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
2069 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
2070 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
2071
2072 ixgbe_stop(ixgbe, B_FALSE);
2073 /*
2074 * Disable interrupts
2075 */
2076 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
2077 rc = ixgbe_disable_intrs(ixgbe);
2078 ASSERT(rc == IXGBE_SUCCESS);
2079 }
2080 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
2081
2082 /*
2083 * Remove interrupt handlers
2084 */
2085 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
2086 ixgbe_rem_intr_handlers(ixgbe);
2087 }
2088 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
2089
2090 /*
2091 * Clear vect_map
2092 */
2093 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
2094 switch (cbaction) {
2095 case DDI_CB_INTR_ADD:
2096 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
2097 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
2098 DDI_INTR_ALLOC_NORMAL);
2099 if (rc != DDI_SUCCESS || actual != count) {
2100 ixgbe_log(ixgbe, "Adjust interrupts failed."
2101 "return: %d, irm cb size: %d, actual: %d",
2102 rc, count, actual);
2103 goto intr_adjust_fail;
2104 }
2105 ixgbe->intr_cnt += count;
2106 break;
2107
2108 case DDI_CB_INTR_REMOVE:
2109 for (i = ixgbe->intr_cnt - count;
2110 i < ixgbe->intr_cnt; i ++) {
2111 rc = ddi_intr_free(ixgbe->htable[i]);
2112 ixgbe->htable[i] = NULL;
2113 if (rc != DDI_SUCCESS) {
2114 ixgbe_log(ixgbe, "Adjust interrupts failed."
2115 "return: %d, irm cb size: %d, actual: %d",
2116 rc, count, actual);
2117 goto intr_adjust_fail;
2118 }
2119 }
2120 ixgbe->intr_cnt -= count;
2121 break;
2122 }
2123
2124 /*
2125 * Get priority for first vector, assume remaining are all the same
2126 */
2127 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
2128 if (rc != DDI_SUCCESS) {
2129 ixgbe_log(ixgbe,
2130 "Get interrupt priority failed: %d", rc);
2131 goto intr_adjust_fail;
2132 }
2133 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
2134 if (rc != DDI_SUCCESS) {
2135 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
2136 goto intr_adjust_fail;
2137 }
2138 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
2139
2140 /*
2141 * Map rings to interrupt vectors
2142 */
2143 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
2144 ixgbe_error(ixgbe,
2145 "IRM CB: Failed to map interrupts to vectors");
2146 goto intr_adjust_fail;
2147 }
2148
2149 /*
2150 * Add interrupt handlers
2151 */
2152 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
2153 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
2154 goto intr_adjust_fail;
2155 }
2156 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
2157
2158 /*
2159 * Now that mutex locks are initialized, and the chip is also
2160 * initialized, enable interrupts.
2161 */
2162 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
2163 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
2164 goto intr_adjust_fail;
2165 }
2166 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
2167 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
2168 ixgbe_error(ixgbe, "IRM CB: Failed to start");
2169 goto intr_adjust_fail;
2170 }
2171 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
2172 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
2173 ixgbe->ixgbe_state |= IXGBE_STARTED;
2174 mutex_exit(&ixgbe->gen_lock);
2175
2176 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2177 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
2178 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
2179 }
2180 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2181 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
2182 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
2183 }
2184
2185 /* Wakeup all Tx rings */
2186 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2187 mac_tx_ring_update(ixgbe->mac_hdl,
2188 ixgbe->tx_rings[i].ring_handle);
2189 }
2190
2191 IXGBE_DEBUGLOG_3(ixgbe,
2192 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
2193 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
2194 return (DDI_SUCCESS);
2195
2196 intr_adjust_fail:
2197 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2198 mutex_exit(&ixgbe->gen_lock);
2199 return (DDI_FAILURE);
2200 }
2201
2202 /*
2203 * ixgbe_intr_cb_register - Register interrupt callback function.
2204 */
2205 static int
2206 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
2207 {
2208 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
2209 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
2210 return (IXGBE_FAILURE);
2211 }
2212 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
2213 return (IXGBE_SUCCESS);
2214 }
2215
2216 /*
2217 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2218 */
2219 static int
2220 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2221 {
2222 /*
2223 * Allocate memory space for rx rings
2224 */
2225 ixgbe->rx_rings = kmem_zalloc(
2226 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2227 KM_NOSLEEP);
2228
2229 if (ixgbe->rx_rings == NULL) {
2230 return (IXGBE_FAILURE);
2231 }
2232
2233 /*
2234 * Allocate memory space for tx rings
2235 */
2236 ixgbe->tx_rings = kmem_zalloc(
2237 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2238 KM_NOSLEEP);
2239
2240 if (ixgbe->tx_rings == NULL) {
2241 kmem_free(ixgbe->rx_rings,
2242 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2243 ixgbe->rx_rings = NULL;
2244 return (IXGBE_FAILURE);
2245 }
2246
2247 /*
2248 * Allocate memory space for rx ring groups
2249 */
2250 ixgbe->rx_groups = kmem_zalloc(
2251 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2252 KM_NOSLEEP);
2253
2254 if (ixgbe->rx_groups == NULL) {
2255 kmem_free(ixgbe->rx_rings,
2256 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2257 kmem_free(ixgbe->tx_rings,
2258 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2259 ixgbe->rx_rings = NULL;
2260 ixgbe->tx_rings = NULL;
2261 return (IXGBE_FAILURE);
2262 }
2263
2264 return (IXGBE_SUCCESS);
2265 }
2266
2267 /*
2268 * ixgbe_free_rings - Free the memory space of rx/tx rings.
2269 */
2270 static void
2271 ixgbe_free_rings(ixgbe_t *ixgbe)
2272 {
2273 if (ixgbe->rx_rings != NULL) {
2274 kmem_free(ixgbe->rx_rings,
2275 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2276 ixgbe->rx_rings = NULL;
2277 }
2278
2279 if (ixgbe->tx_rings != NULL) {
2280 kmem_free(ixgbe->tx_rings,
2281 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2282 ixgbe->tx_rings = NULL;
2283 }
2284
2285 if (ixgbe->rx_groups != NULL) {
2286 kmem_free(ixgbe->rx_groups,
2287 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2288 ixgbe->rx_groups = NULL;
2289 }
2290 }
2291
2292 static int
2293 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2294 {
2295 ixgbe_rx_ring_t *rx_ring;
2296 int i;
2297
2298 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2299 rx_ring = &ixgbe->rx_rings[i];
2300 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2301 goto alloc_rx_rings_failure;
2302 }
2303 return (IXGBE_SUCCESS);
2304
2305 alloc_rx_rings_failure:
2306 ixgbe_free_rx_data(ixgbe);
2307 return (IXGBE_FAILURE);
2308 }
2309
2310 static void
2311 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2312 {
2313 ixgbe_rx_ring_t *rx_ring;
2314 ixgbe_rx_data_t *rx_data;
2315 int i;
2316
2317 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2318 rx_ring = &ixgbe->rx_rings[i];
2319
2320 mutex_enter(&ixgbe->rx_pending_lock);
2321 rx_data = rx_ring->rx_data;
2322
2323 if (rx_data != NULL) {
2324 rx_data->flag |= IXGBE_RX_STOPPED;
2325
2326 if (rx_data->rcb_pending == 0) {
2327 ixgbe_free_rx_ring_data(rx_data);
2328 rx_ring->rx_data = NULL;
2329 }
2330 }
2331
2332 mutex_exit(&ixgbe->rx_pending_lock);
2333 }
2334 }
2335
2336 /*
2337 * ixgbe_setup_rings - Setup rx/tx rings.
2338 */
2339 static void
2340 ixgbe_setup_rings(ixgbe_t *ixgbe)
2341 {
2342 /*
2343 * Setup the rx/tx rings, including the following:
2344 *
2345 * 1. Setup the descriptor ring and the control block buffers;
2346 * 2. Initialize necessary registers for receive/transmit;
2347 * 3. Initialize software pointers/parameters for receive/transmit;
2348 */
2349 ixgbe_setup_rx(ixgbe);
2350
2351 ixgbe_setup_tx(ixgbe);
2352 }
2353
2354 static void
2355 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2356 {
2357 ixgbe_t *ixgbe = rx_ring->ixgbe;
2358 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2359 struct ixgbe_hw *hw = &ixgbe->hw;
2360 rx_control_block_t *rcb;
2361 union ixgbe_adv_rx_desc *rbd;
2362 uint32_t size;
2363 uint32_t buf_low;
2364 uint32_t buf_high;
2365 uint32_t reg_val;
2366 int i;
2367
2368 ASSERT(mutex_owned(&rx_ring->rx_lock));
2369 ASSERT(mutex_owned(&ixgbe->gen_lock));
2370
2371 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2372 rcb = rx_data->work_list[i];
2373 rbd = &rx_data->rbd_ring[i];
2374
2375 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2376 rbd->read.hdr_addr = 0;
2377 }
2378
2379 /*
2380 * Initialize the length register
2381 */
2382 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2383 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2384
2385 /*
2386 * Initialize the base address registers
2387 */
2388 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2389 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2390 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2391 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2392
2393 /*
2394 * Setup head & tail pointers
2395 */
2396 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2397 rx_data->ring_size - 1);
2398 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2399
2400 rx_data->rbd_next = 0;
2401 rx_data->lro_first = 0;
2402
2403 /*
2404 * Setup the Receive Descriptor Control Register (RXDCTL)
2405 * PTHRESH=32 descriptors (half the internal cache)
2406 * HTHRESH=0 descriptors (to minimize latency on fetch)
2407 * WTHRESH defaults to 1 (writeback each descriptor)
2408 */
2409 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2410 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2411
2412 /* Not a valid value for 82599, X540 or X550 */
2413 if (hw->mac.type == ixgbe_mac_82598EB) {
2414 reg_val |= 0x0020; /* pthresh */
2415 }
2416 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2417
2418 if (hw->mac.type == ixgbe_mac_82599EB ||
2419 hw->mac.type == ixgbe_mac_X540 ||
2420 hw->mac.type == ixgbe_mac_X550 ||
2421 hw->mac.type == ixgbe_mac_X550EM_x ||
2422 hw->mac.type == ixgbe_mac_X550EM_a) {
2423 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2424 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2425 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2426 }
2427
2428 /*
2429 * Setup the Split and Replication Receive Control Register.
2430 * Set the rx buffer size and the advanced descriptor type.
2431 */
2432 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2433 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2434 reg_val |= IXGBE_SRRCTL_DROP_EN;
2435 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2436 }
2437
2438 static void
2439 ixgbe_setup_rx(ixgbe_t *ixgbe)
2440 {
2441 ixgbe_rx_ring_t *rx_ring;
2442 struct ixgbe_hw *hw = &ixgbe->hw;
2443 uint32_t reg_val;
2444 uint32_t ring_mapping;
2445 uint32_t i, index;
2446 uint32_t psrtype_rss_bit;
2447
2448 /*
2449 * Ensure that Rx is disabled while setting up
2450 * the Rx unit and Rx descriptor ring(s)
2451 */
2452 ixgbe_disable_rx(hw);
2453
2454 /* PSRTYPE must be configured for 82599 */
2455 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2456 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2457 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2458 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2459 reg_val |= IXGBE_PSRTYPE_L2HDR;
2460 reg_val |= 0x80000000;
2461 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2462 } else {
2463 if (ixgbe->num_rx_groups > 32) {
2464 psrtype_rss_bit = 0x20000000;
2465 } else {
2466 psrtype_rss_bit = 0x40000000;
2467 }
2468 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2469 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2470 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2471 reg_val |= IXGBE_PSRTYPE_L2HDR;
2472 reg_val |= psrtype_rss_bit;
2473 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2474 }
2475 }
2476
2477 /*
2478 * Set filter control in FCTRL to determine types of packets are passed
2479 * up to the driver.
2480 * - Pass broadcast packets.
2481 * - Do not pass flow control pause frames (82598-specific)
2482 */
2483 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2484 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */
2485 if (hw->mac.type == ixgbe_mac_82598EB) {
2486 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */
2487 }
2488 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2489
2490 /*
2491 * Hardware checksum settings
2492 */
2493 if (ixgbe->rx_hcksum_enable) {
2494 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2495 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2496 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2497 }
2498
2499 /*
2500 * Setup VMDq and RSS for multiple receive queues
2501 */
2502 switch (ixgbe->classify_mode) {
2503 case IXGBE_CLASSIFY_RSS:
2504 /*
2505 * One group, only RSS is needed when more than
2506 * one ring enabled.
2507 */
2508 ixgbe_setup_rss(ixgbe);
2509 break;
2510
2511 case IXGBE_CLASSIFY_VMDQ:
2512 /*
2513 * Multiple groups, each group has one ring,
2514 * only VMDq is needed.
2515 */
2516 ixgbe_setup_vmdq(ixgbe);
2517 break;
2518
2519 case IXGBE_CLASSIFY_VMDQ_RSS:
2520 /*
2521 * Multiple groups and multiple rings, both
2522 * VMDq and RSS are needed.
2523 */
2524 ixgbe_setup_vmdq_rss(ixgbe);
2525 break;
2526
2527 default:
2528 break;
2529 }
2530
2531 /*
2532 * Enable the receive unit. This must be done after filter
2533 * control is set in FCTRL. On 82598, we disable the descriptor monitor.
2534 * 82598 is the only adapter which defines this RXCTRL option.
2535 */
2536 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
2537 if (hw->mac.type == ixgbe_mac_82598EB)
2538 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */
2539 reg_val |= IXGBE_RXCTRL_RXEN;
2540 (void) ixgbe_enable_rx_dma(hw, reg_val);
2541
2542 /*
2543 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2544 */
2545 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2546 rx_ring = &ixgbe->rx_rings[i];
2547 ixgbe_setup_rx_ring(rx_ring);
2548 }
2549
2550 /*
2551 * Setup the per-ring statistics mapping.
2552 */
2553 ring_mapping = 0;
2554 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2555 index = ixgbe->rx_rings[i].hw_index;
2556 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2557 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2558 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2559 }
2560
2561 /*
2562 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2563 * by four bytes if the packet has a VLAN field, so includes MTU,
2564 * ethernet header and frame check sequence.
2565 * Register is MAXFRS in 82599.
2566 */
2567 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD);
2568 reg_val &= ~IXGBE_MHADD_MFS_MASK;
2569 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header)
2570 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2571 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2572
2573 /*
2574 * Setup Jumbo Frame enable bit
2575 */
2576 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2577 if (ixgbe->default_mtu > ETHERMTU)
2578 reg_val |= IXGBE_HLREG0_JUMBOEN;
2579 else
2580 reg_val &= ~IXGBE_HLREG0_JUMBOEN;
2581 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2582
2583 /*
2584 * Setup RSC for multiple receive queues.
2585 */
2586 if (ixgbe->lro_enable) {
2587 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2588 /*
2589 * Make sure rx_buf_size * MAXDESC not greater
2590 * than 65535.
2591 * Intel recommends 4 for MAXDESC field value.
2592 */
2593 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2594 reg_val |= IXGBE_RSCCTL_RSCEN;
2595 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2596 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2597 else
2598 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2599 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2600 }
2601
2602 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2603 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2604 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2605
2606 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2607 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2608 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2609 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2610
2611 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2612 }
2613 }
2614
2615 static void
2616 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2617 {
2618 ixgbe_t *ixgbe = tx_ring->ixgbe;
2619 struct ixgbe_hw *hw = &ixgbe->hw;
2620 uint32_t size;
2621 uint32_t buf_low;
2622 uint32_t buf_high;
2623 uint32_t reg_val;
2624
2625 ASSERT(mutex_owned(&tx_ring->tx_lock));
2626 ASSERT(mutex_owned(&ixgbe->gen_lock));
2627
2628 /*
2629 * Initialize the length register
2630 */
2631 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2632 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2633
2634 /*
2635 * Initialize the base address registers
2636 */
2637 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2638 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2639 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2640 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2641
2642 /*
2643 * Setup head & tail pointers
2644 */
2645 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2646 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2647
2648 /*
2649 * Setup head write-back
2650 */
2651 if (ixgbe->tx_head_wb_enable) {
2652 /*
2653 * The memory of the head write-back is allocated using
2654 * the extra tbd beyond the tail of the tbd ring.
2655 */
2656 tx_ring->tbd_head_wb = (uint32_t *)
2657 ((uintptr_t)tx_ring->tbd_area.address + size);
2658 *tx_ring->tbd_head_wb = 0;
2659
2660 buf_low = (uint32_t)
2661 (tx_ring->tbd_area.dma_address + size);
2662 buf_high = (uint32_t)
2663 ((tx_ring->tbd_area.dma_address + size) >> 32);
2664
2665 /* Set the head write-back enable bit */
2666 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2667
2668 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2669 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2670
2671 /*
2672 * Turn off relaxed ordering for head write back or it will
2673 * cause problems with the tx recycling
2674 */
2675
2676 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2677 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2678 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2679 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2680 if (hw->mac.type == ixgbe_mac_82598EB) {
2681 IXGBE_WRITE_REG(hw,
2682 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2683 } else {
2684 IXGBE_WRITE_REG(hw,
2685 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2686 }
2687 } else {
2688 tx_ring->tbd_head_wb = NULL;
2689 }
2690
2691 tx_ring->tbd_head = 0;
2692 tx_ring->tbd_tail = 0;
2693 tx_ring->tbd_free = tx_ring->ring_size;
2694
2695 if (ixgbe->tx_ring_init == B_TRUE) {
2696 tx_ring->tcb_head = 0;
2697 tx_ring->tcb_tail = 0;
2698 tx_ring->tcb_free = tx_ring->free_list_size;
2699 }
2700
2701 /*
2702 * Initialize the s/w context structure
2703 */
2704 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2705 }
2706
2707 static void
2708 ixgbe_setup_tx(ixgbe_t *ixgbe)
2709 {
2710 struct ixgbe_hw *hw = &ixgbe->hw;
2711 ixgbe_tx_ring_t *tx_ring;
2712 uint32_t reg_val;
2713 uint32_t ring_mapping;
2714 int i;
2715
2716 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2717 tx_ring = &ixgbe->tx_rings[i];
2718 ixgbe_setup_tx_ring(tx_ring);
2719 }
2720
2721 /*
2722 * Setup the per-ring statistics mapping.
2723 */
2724 ring_mapping = 0;
2725 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2726 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2727 if ((i & 0x3) == 0x3) {
2728 switch (hw->mac.type) {
2729 case ixgbe_mac_82598EB:
2730 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2731 ring_mapping);
2732 break;
2733
2734 case ixgbe_mac_82599EB:
2735 case ixgbe_mac_X540:
2736 case ixgbe_mac_X550:
2737 case ixgbe_mac_X550EM_x:
2738 case ixgbe_mac_X550EM_a:
2739 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2740 ring_mapping);
2741 break;
2742
2743 default:
2744 break;
2745 }
2746
2747 ring_mapping = 0;
2748 }
2749 }
2750 if (i & 0x3) {
2751 switch (hw->mac.type) {
2752 case ixgbe_mac_82598EB:
2753 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2754 break;
2755
2756 case ixgbe_mac_82599EB:
2757 case ixgbe_mac_X540:
2758 case ixgbe_mac_X550:
2759 case ixgbe_mac_X550EM_x:
2760 case ixgbe_mac_X550EM_a:
2761 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2762 break;
2763
2764 default:
2765 break;
2766 }
2767 }
2768
2769 /*
2770 * Enable CRC appending and TX padding (for short tx frames)
2771 */
2772 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2773 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2774 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2775
2776 /*
2777 * enable DMA for 82599, X540 and X550 parts
2778 */
2779 if (hw->mac.type == ixgbe_mac_82599EB ||
2780 hw->mac.type == ixgbe_mac_X540 ||
2781 hw->mac.type == ixgbe_mac_X550 ||
2782 hw->mac.type == ixgbe_mac_X550EM_x ||
2783 hw->mac.type == ixgbe_mac_X550EM_a) {
2784 /* DMATXCTL.TE must be set after all Tx config is complete */
2785 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2786 reg_val |= IXGBE_DMATXCTL_TE;
2787 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2788
2789 /* Disable arbiter to set MTQC */
2790 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2791 reg_val |= IXGBE_RTTDCS_ARBDIS;
2792 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2793 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2794 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2795 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2796 }
2797
2798 /*
2799 * Enabling tx queues ..
2800 * For 82599 must be done after DMATXCTL.TE is set
2801 */
2802 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2803 tx_ring = &ixgbe->tx_rings[i];
2804 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2805 reg_val |= IXGBE_TXDCTL_ENABLE;
2806 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2807 }
2808 }
2809
2810 /*
2811 * ixgbe_setup_rss - Setup receive-side scaling feature.
2812 */
2813 static void
2814 ixgbe_setup_rss(ixgbe_t *ixgbe)
2815 {
2816 struct ixgbe_hw *hw = &ixgbe->hw;
2817 uint32_t mrqc;
2818
2819 /*
2820 * Initialize RETA/ERETA table
2821 */
2822 ixgbe_setup_rss_table(ixgbe);
2823
2824 /*
2825 * Enable RSS & perform hash on these packet types
2826 */
2827 mrqc = IXGBE_MRQC_RSSEN |
2828 IXGBE_MRQC_RSS_FIELD_IPV4 |
2829 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2830 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2831 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2832 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2833 IXGBE_MRQC_RSS_FIELD_IPV6 |
2834 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2835 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2836 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2837 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2838 }
2839
2840 /*
2841 * ixgbe_setup_vmdq - Setup MAC classification feature
2842 */
2843 static void
2844 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2845 {
2846 struct ixgbe_hw *hw = &ixgbe->hw;
2847 uint32_t vmdctl, i, vtctl;
2848
2849 /*
2850 * Setup the VMDq Control register, enable VMDq based on
2851 * packet destination MAC address:
2852 */
2853 switch (hw->mac.type) {
2854 case ixgbe_mac_82598EB:
2855 /*
2856 * VMDq Enable = 1;
2857 * VMDq Filter = 0; MAC filtering
2858 * Default VMDq output index = 0;
2859 */
2860 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2861 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2862 break;
2863
2864 case ixgbe_mac_82599EB:
2865 case ixgbe_mac_X540:
2866 case ixgbe_mac_X550:
2867 case ixgbe_mac_X550EM_x:
2868 case ixgbe_mac_X550EM_a:
2869 /*
2870 * Enable VMDq-only.
2871 */
2872 vmdctl = IXGBE_MRQC_VMDQEN;
2873 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2874
2875 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2876 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2877 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2878 }
2879
2880 /*
2881 * Enable Virtualization and Replication.
2882 */
2883 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2884 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2885
2886 /*
2887 * Enable receiving packets to all VFs
2888 */
2889 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2890 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2891 break;
2892
2893 default:
2894 break;
2895 }
2896 }
2897
2898 /*
2899 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2900 */
2901 static void
2902 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2903 {
2904 struct ixgbe_hw *hw = &ixgbe->hw;
2905 uint32_t i, mrqc;
2906 uint32_t vtctl, vmdctl;
2907
2908 /*
2909 * Initialize RETA/ERETA table
2910 */
2911 ixgbe_setup_rss_table(ixgbe);
2912
2913 /*
2914 * Enable and setup RSS and VMDq
2915 */
2916 switch (hw->mac.type) {
2917 case ixgbe_mac_82598EB:
2918 /*
2919 * Enable RSS & Setup RSS Hash functions
2920 */
2921 mrqc = IXGBE_MRQC_RSSEN |
2922 IXGBE_MRQC_RSS_FIELD_IPV4 |
2923 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2924 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2925 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2926 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2927 IXGBE_MRQC_RSS_FIELD_IPV6 |
2928 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2929 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2930 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2931 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2932
2933 /*
2934 * Enable and Setup VMDq
2935 * VMDq Filter = 0; MAC filtering
2936 * Default VMDq output index = 0;
2937 */
2938 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2939 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2940 break;
2941
2942 case ixgbe_mac_82599EB:
2943 case ixgbe_mac_X540:
2944 case ixgbe_mac_X550:
2945 case ixgbe_mac_X550EM_x:
2946 case ixgbe_mac_X550EM_a:
2947 /*
2948 * Enable RSS & Setup RSS Hash functions
2949 */
2950 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2951 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2952 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2953 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2954 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2955 IXGBE_MRQC_RSS_FIELD_IPV6 |
2956 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2957 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2958 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2959
2960 /*
2961 * Enable VMDq+RSS.
2962 */
2963 if (ixgbe->num_rx_groups > 32) {
2964 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2965 } else {
2966 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2967 }
2968
2969 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2970
2971 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2972 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2973 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2974 }
2975 break;
2976
2977 default:
2978 break;
2979
2980 }
2981
2982 if (hw->mac.type == ixgbe_mac_82599EB ||
2983 hw->mac.type == ixgbe_mac_X540 ||
2984 hw->mac.type == ixgbe_mac_X550 ||
2985 hw->mac.type == ixgbe_mac_X550EM_x ||
2986 hw->mac.type == ixgbe_mac_X550EM_a) {
2987 /*
2988 * Enable Virtualization and Replication.
2989 */
2990 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2991 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2992
2993 /*
2994 * Enable receiving packets to all VFs
2995 */
2996 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2997 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2998 }
2999 }
3000
3001 /*
3002 * ixgbe_setup_rss_table - Setup RSS table
3003 */
3004 static void
3005 ixgbe_setup_rss_table(ixgbe_t *ixgbe)
3006 {
3007 struct ixgbe_hw *hw = &ixgbe->hw;
3008 uint32_t i, j;
3009 uint32_t random;
3010 uint32_t reta;
3011 uint32_t ring_per_group;
3012 uint32_t ring;
3013 uint32_t table_size;
3014 uint32_t index_mult;
3015 uint32_t rxcsum;
3016
3017 /*
3018 * Set multiplier for RETA setup and table size based on MAC type.
3019 * RETA table sizes vary by model:
3020 *
3021 * 82598, 82599, X540: 128 table entries.
3022 * X550: 512 table entries.
3023 */
3024 index_mult = 0x1;
3025 table_size = 128;
3026 switch (ixgbe->hw.mac.type) {
3027 case ixgbe_mac_82598EB:
3028 index_mult = 0x11;
3029 break;
3030 case ixgbe_mac_X550:
3031 case ixgbe_mac_X550EM_x:
3032 case ixgbe_mac_X550EM_a:
3033 table_size = 512;
3034 break;
3035 default:
3036 break;
3037 }
3038
3039 /*
3040 * Fill out RSS redirection table. The configuation of the indices is
3041 * hardware-dependent.
3042 *
3043 * 82598: 8 bits wide containing two 4 bit RSS indices
3044 * 82599, X540: 8 bits wide containing one 4 bit RSS index
3045 * X550: 8 bits wide containing one 6 bit RSS index
3046 */
3047 reta = 0;
3048 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3049
3050 for (i = 0, j = 0; i < table_size; i++, j++) {
3051 if (j == ring_per_group) j = 0;
3052
3053 /*
3054 * The low 8 bits are for hash value (n+0);
3055 * The next 8 bits are for hash value (n+1), etc.
3056 */
3057 ring = (j * index_mult);
3058 reta = reta >> 8;
3059 reta = reta | (((uint32_t)ring) << 24);
3060
3061 if ((i & 3) == 3) {
3062 /*
3063 * The first 128 table entries are programmed into the
3064 * RETA register, with any beyond that (eg; on X550)
3065 * into ERETA.
3066 */
3067 if (i < 128)
3068 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3069 else
3070 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3071 reta);
3072 reta = 0;
3073 }
3074 }
3075
3076 /*
3077 * Fill out hash function seeds with a random constant
3078 */
3079 for (i = 0; i < 10; i++) {
3080 (void) random_get_pseudo_bytes((uint8_t *)&random,
3081 sizeof (uint32_t));
3082 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
3083 }
3084
3085 /*
3086 * Disable Packet Checksum to enable RSS for multiple receive queues.
3087 * It is an adapter hardware limitation that Packet Checksum is
3088 * mutually exclusive with RSS.
3089 */
3090 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3091 rxcsum |= IXGBE_RXCSUM_PCSD;
3092 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3093 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3094 }
3095
3096 /*
3097 * ixgbe_init_unicst - Initialize the unicast addresses.
3098 */
3099 static void
3100 ixgbe_init_unicst(ixgbe_t *ixgbe)
3101 {
3102 struct ixgbe_hw *hw = &ixgbe->hw;
3103 uint8_t *mac_addr;
3104 int slot;
3105 /*
3106 * Here we should consider two situations:
3107 *
3108 * 1. Chipset is initialized at the first time,
3109 * Clear all the multiple unicast addresses.
3110 *
3111 * 2. Chipset is reset
3112 * Recover the multiple unicast addresses from the
3113 * software data structure to the RAR registers.
3114 */
3115 if (!ixgbe->unicst_init) {
3116 /*
3117 * Initialize the multiple unicast addresses
3118 */
3119 ixgbe->unicst_total = hw->mac.num_rar_entries;
3120 ixgbe->unicst_avail = ixgbe->unicst_total;
3121 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3122 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3123 bzero(mac_addr, ETHERADDRL);
3124 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3125 ixgbe->unicst_addr[slot].mac.set = 0;
3126 }
3127 ixgbe->unicst_init = B_TRUE;
3128 } else {
3129 /* Re-configure the RAR registers */
3130 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3131 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3132 if (ixgbe->unicst_addr[slot].mac.set == 1) {
3133 (void) ixgbe_set_rar(hw, slot, mac_addr,
3134 ixgbe->unicst_addr[slot].mac.group_index,
3135 IXGBE_RAH_AV);
3136 } else {
3137 bzero(mac_addr, ETHERADDRL);
3138 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3139 }
3140 }
3141 }
3142 }
3143
3144 /*
3145 * ixgbe_unicst_find - Find the slot for the specified unicast address
3146 */
3147 int
3148 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
3149 {
3150 int slot;
3151
3152 ASSERT(mutex_owned(&ixgbe->gen_lock));
3153
3154 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3155 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
3156 mac_addr, ETHERADDRL) == 0)
3157 return (slot);
3158 }
3159
3160 return (-1);
3161 }
3162
3163 /*
3164 * ixgbe_multicst_add - Add a multicst address.
3165 */
3166 int
3167 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3168 {
3169 ASSERT(mutex_owned(&ixgbe->gen_lock));
3170
3171 if ((multiaddr[0] & 01) == 0) {
3172 return (EINVAL);
3173 }
3174
3175 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
3176 return (ENOENT);
3177 }
3178
3179 bcopy(multiaddr,
3180 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
3181 ixgbe->mcast_count++;
3182
3183 /*
3184 * Update the multicast table in the hardware
3185 */
3186 ixgbe_setup_multicst(ixgbe);
3187
3188 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3189 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3190 return (EIO);
3191 }
3192
3193 return (0);
3194 }
3195
3196 /*
3197 * ixgbe_multicst_remove - Remove a multicst address.
3198 */
3199 int
3200 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3201 {
3202 int i;
3203
3204 ASSERT(mutex_owned(&ixgbe->gen_lock));
3205
3206 for (i = 0; i < ixgbe->mcast_count; i++) {
3207 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
3208 ETHERADDRL) == 0) {
3209 for (i++; i < ixgbe->mcast_count; i++) {
3210 ixgbe->mcast_table[i - 1] =
3211 ixgbe->mcast_table[i];
3212 }
3213 ixgbe->mcast_count--;
3214 break;
3215 }
3216 }
3217
3218 /*
3219 * Update the multicast table in the hardware
3220 */
3221 ixgbe_setup_multicst(ixgbe);
3222
3223 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3224 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3225 return (EIO);
3226 }
3227
3228 return (0);
3229 }
3230
3231 /*
3232 * ixgbe_setup_multicast - Setup multicast data structures.
3233 *
3234 * This routine initializes all of the multicast related structures
3235 * and save them in the hardware registers.
3236 */
3237 static void
3238 ixgbe_setup_multicst(ixgbe_t *ixgbe)
3239 {
3240 uint8_t *mc_addr_list;
3241 uint32_t mc_addr_count;
3242 struct ixgbe_hw *hw = &ixgbe->hw;
3243
3244 ASSERT(mutex_owned(&ixgbe->gen_lock));
3245
3246 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
3247
3248 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
3249 mc_addr_count = ixgbe->mcast_count;
3250
3251 /*
3252 * Update the multicast addresses to the MTA registers
3253 */
3254 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
3255 ixgbe_mc_table_itr, TRUE);
3256 }
3257
3258 /*
3259 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
3260 *
3261 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
3262 * Different chipsets may have different allowed configuration of vmdq and rss.
3263 */
3264 static void
3265 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
3266 {
3267 struct ixgbe_hw *hw = &ixgbe->hw;
3268 uint32_t ring_per_group;
3269
3270 switch (hw->mac.type) {
3271 case ixgbe_mac_82598EB:
3272 /*
3273 * 82598 supports the following combination:
3274 * vmdq no. x rss no.
3275 * [5..16] x 1
3276 * [1..4] x [1..16]
3277 * However 8 rss queue per pool (vmdq) is sufficient for
3278 * most cases.
3279 */
3280 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3281 if (ixgbe->num_rx_groups > 4) {
3282 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3283 } else {
3284 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3285 min(8, ring_per_group);
3286 }
3287
3288 break;
3289
3290 case ixgbe_mac_82599EB:
3291 case ixgbe_mac_X540:
3292 case ixgbe_mac_X550:
3293 case ixgbe_mac_X550EM_x:
3294 case ixgbe_mac_X550EM_a:
3295 /*
3296 * 82599 supports the following combination:
3297 * vmdq no. x rss no.
3298 * [33..64] x [1..2]
3299 * [2..32] x [1..4]
3300 * 1 x [1..16]
3301 * However 8 rss queue per pool (vmdq) is sufficient for
3302 * most cases.
3303 *
3304 * For now, treat X540 and X550 like the 82599.
3305 */
3306 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3307 if (ixgbe->num_rx_groups == 1) {
3308 ixgbe->num_rx_rings = min(8, ring_per_group);
3309 } else if (ixgbe->num_rx_groups <= 32) {
3310 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3311 min(4, ring_per_group);
3312 } else if (ixgbe->num_rx_groups <= 64) {
3313 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3314 min(2, ring_per_group);
3315 }
3316 break;
3317
3318 default:
3319 break;
3320 }
3321
3322 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3323
3324 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3325 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3326 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3327 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3328 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3329 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3330 } else {
3331 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3332 }
3333
3334 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3335 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3336 }
3337
3338 /*
3339 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3340 *
3341 * This routine gets user-configured values out of the configuration
3342 * file ixgbe.conf.
3343 *
3344 * For each configurable value, there is a minimum, a maximum, and a
3345 * default.
3346 * If user does not configure a value, use the default.
3347 * If user configures below the minimum, use the minumum.
3348 * If user configures above the maximum, use the maxumum.
3349 */
3350 static void
3351 ixgbe_get_conf(ixgbe_t *ixgbe)
3352 {
3353 struct ixgbe_hw *hw = &ixgbe->hw;
3354 uint32_t flow_control;
3355
3356 /*
3357 * ixgbe driver supports the following user configurations:
3358 *
3359 * Jumbo frame configuration:
3360 * default_mtu
3361 *
3362 * Ethernet flow control configuration:
3363 * flow_control
3364 *
3365 * Multiple rings configurations:
3366 * tx_queue_number
3367 * tx_ring_size
3368 * rx_queue_number
3369 * rx_ring_size
3370 *
3371 * Call ixgbe_get_prop() to get the value for a specific
3372 * configuration parameter.
3373 */
3374
3375 /*
3376 * Jumbo frame configuration - max_frame_size controls host buffer
3377 * allocation, so includes MTU, ethernet header, vlan tag and
3378 * frame check sequence.
3379 */
3380 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3381 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3382
3383 ixgbe->max_frame_size = ixgbe->default_mtu +
3384 sizeof (struct ether_vlan_header) + ETHERFCSL;
3385
3386 /*
3387 * Ethernet flow control configuration
3388 */
3389 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3390 ixgbe_fc_none, 3, ixgbe_fc_none);
3391 if (flow_control == 3)
3392 flow_control = ixgbe_fc_default;
3393
3394 /*
3395 * fc.requested mode is what the user requests. After autoneg,
3396 * fc.current_mode will be the flow_control mode that was negotiated.
3397 */
3398 hw->fc.requested_mode = flow_control;
3399
3400 /*
3401 * Multiple rings configurations
3402 */
3403 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3404 ixgbe->capab->min_tx_que_num,
3405 ixgbe->capab->max_tx_que_num,
3406 ixgbe->capab->def_tx_que_num);
3407 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3408 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3409
3410 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3411 ixgbe->capab->min_rx_que_num,
3412 ixgbe->capab->max_rx_que_num,
3413 ixgbe->capab->def_rx_que_num);
3414 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3415 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3416
3417 /*
3418 * Multiple groups configuration
3419 */
3420 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3421 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3422 ixgbe->capab->def_rx_grp_num);
3423
3424 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3425 0, 1, DEFAULT_MR_ENABLE);
3426
3427 if (ixgbe->mr_enable == B_FALSE) {
3428 ixgbe->num_tx_rings = 1;
3429 ixgbe->num_rx_rings = 1;
3430 ixgbe->num_rx_groups = 1;
3431 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3432 } else {
3433 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3434 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3435 /*
3436 * The combination of num_rx_rings and num_rx_groups
3437 * may be not supported by h/w. We need to adjust
3438 * them to appropriate values.
3439 */
3440 ixgbe_setup_vmdq_rss_conf(ixgbe);
3441 }
3442
3443 /*
3444 * Tunable used to force an interrupt type. The only use is
3445 * for testing of the lesser interrupt types.
3446 * 0 = don't force interrupt type
3447 * 1 = force interrupt type MSI-X
3448 * 2 = force interrupt type MSI
3449 * 3 = force interrupt type Legacy
3450 */
3451 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3452 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3453
3454 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3455 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3456 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3457 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3458 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3459 0, 1, DEFAULT_LSO_ENABLE);
3460 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3461 0, 1, DEFAULT_LRO_ENABLE);
3462 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3463 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3464 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3465 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3466
3467 /* Head Write Back not recommended for 82599, X540 and X550 */
3468 if (hw->mac.type == ixgbe_mac_82599EB ||
3469 hw->mac.type == ixgbe_mac_X540 ||
3470 hw->mac.type == ixgbe_mac_X550 ||
3471 hw->mac.type == ixgbe_mac_X550EM_x ||
3472 hw->mac.type == ixgbe_mac_X550EM_a) {
3473 ixgbe->tx_head_wb_enable = B_FALSE;
3474 }
3475
3476 /*
3477 * ixgbe LSO needs the tx h/w checksum support.
3478 * LSO will be disabled if tx h/w checksum is not
3479 * enabled.
3480 */
3481 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3482 ixgbe->lso_enable = B_FALSE;
3483 }
3484
3485 /*
3486 * ixgbe LRO needs the rx h/w checksum support.
3487 * LRO will be disabled if rx h/w checksum is not
3488 * enabled.
3489 */
3490 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3491 ixgbe->lro_enable = B_FALSE;
3492 }
3493
3494 /*
3495 * ixgbe LRO only supported by 82599, X540 and X550
3496 */
3497 if (hw->mac.type == ixgbe_mac_82598EB) {
3498 ixgbe->lro_enable = B_FALSE;
3499 }
3500 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3501 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3502 DEFAULT_TX_COPY_THRESHOLD);
3503 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3504 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3505 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3506 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3507 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3508 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3509 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3510 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3511 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3512
3513 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3514 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3515 DEFAULT_RX_COPY_THRESHOLD);
3516 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3517 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3518 DEFAULT_RX_LIMIT_PER_INTR);
3519
3520 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3521 ixgbe->capab->min_intr_throttle,
3522 ixgbe->capab->max_intr_throttle,
3523 ixgbe->capab->def_intr_throttle);
3524 /*
3525 * 82599, X540 and X550 require the interrupt throttling rate is
3526 * a multiple of 8. This is enforced by the register definiton.
3527 */
3528 if (hw->mac.type == ixgbe_mac_82599EB ||
3529 hw->mac.type == ixgbe_mac_X540 ||
3530 hw->mac.type == ixgbe_mac_X550 ||
3531 hw->mac.type == ixgbe_mac_X550EM_x ||
3532 hw->mac.type == ixgbe_mac_X550EM_a)
3533 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3534
3535 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3536 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3537 }
3538
3539 static void
3540 ixgbe_init_params(ixgbe_t *ixgbe)
3541 {
3542 struct ixgbe_hw *hw = &ixgbe->hw;
3543 ixgbe_link_speed speeds_supported = 0;
3544 boolean_t negotiate;
3545
3546 /*
3547 * Get a list of speeds the adapter supports. If the hw struct hasn't
3548 * been populated with this information yet, retrieve it from the
3549 * adapter and save it to our own variable.
3550 *
3551 * On certain adapters, such as ones which use SFPs, the contents of
3552 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not
3553 * updated, so we must rely on calling ixgbe_get_link_capabilities()
3554 * in order to ascertain the speeds which we are capable of supporting,
3555 * and in the case of SFP-equipped adapters, which speed we are
3556 * advertising. If ixgbe_get_link_capabilities() fails for some reason,
3557 * we'll go with a default list of speeds as a last resort.
3558 */
3559 speeds_supported = hw->phy.speeds_supported;
3560
3561 if (speeds_supported == 0) {
3562 if (ixgbe_get_link_capabilities(hw, &speeds_supported,
3563 &negotiate) != IXGBE_SUCCESS) {
3564 if (hw->mac.type == ixgbe_mac_82598EB) {
3565 speeds_supported =
3566 IXGBE_LINK_SPEED_82598_AUTONEG;
3567 } else {
3568 speeds_supported =
3569 IXGBE_LINK_SPEED_82599_AUTONEG;
3570 }
3571 }
3572 }
3573 ixgbe->speeds_supported = speeds_supported;
3574
3575 /*
3576 * By default, all supported speeds are enabled and advertised.
3577 */
3578 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) {
3579 ixgbe->param_en_10000fdx_cap = 1;
3580 ixgbe->param_adv_10000fdx_cap = 1;
3581 } else {
3582 ixgbe->param_en_10000fdx_cap = 0;
3583 ixgbe->param_adv_10000fdx_cap = 0;
3584 }
3585
3586 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) {
3587 ixgbe->param_en_5000fdx_cap = 1;
3588 ixgbe->param_adv_5000fdx_cap = 1;
3589 } else {
3590 ixgbe->param_en_5000fdx_cap = 0;
3591 ixgbe->param_adv_5000fdx_cap = 0;
3592 }
3593
3594 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) {
3595 ixgbe->param_en_2500fdx_cap = 1;
3596 ixgbe->param_adv_2500fdx_cap = 1;
3597 } else {
3598 ixgbe->param_en_2500fdx_cap = 0;
3599 ixgbe->param_adv_2500fdx_cap = 0;
3600 }
3601
3602 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) {
3603 ixgbe->param_en_1000fdx_cap = 1;
3604 ixgbe->param_adv_1000fdx_cap = 1;
3605 } else {
3606 ixgbe->param_en_1000fdx_cap = 0;
3607 ixgbe->param_adv_1000fdx_cap = 0;
3608 }
3609
3610 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) {
3611 ixgbe->param_en_100fdx_cap = 1;
3612 ixgbe->param_adv_100fdx_cap = 1;
3613 } else {
3614 ixgbe->param_en_100fdx_cap = 0;
3615 ixgbe->param_adv_100fdx_cap = 0;
3616 }
3617
3618 ixgbe->param_pause_cap = 1;
3619 ixgbe->param_asym_pause_cap = 1;
3620 ixgbe->param_rem_fault = 0;
3621
3622 ixgbe->param_adv_autoneg_cap = 1;
3623 ixgbe->param_adv_pause_cap = 1;
3624 ixgbe->param_adv_asym_pause_cap = 1;
3625 ixgbe->param_adv_rem_fault = 0;
3626
3627 ixgbe->param_lp_10000fdx_cap = 0;
3628 ixgbe->param_lp_5000fdx_cap = 0;
3629 ixgbe->param_lp_2500fdx_cap = 0;
3630 ixgbe->param_lp_1000fdx_cap = 0;
3631 ixgbe->param_lp_100fdx_cap = 0;
3632 ixgbe->param_lp_autoneg_cap = 0;
3633 ixgbe->param_lp_pause_cap = 0;
3634 ixgbe->param_lp_asym_pause_cap = 0;
3635 ixgbe->param_lp_rem_fault = 0;
3636 }
3637
3638 /*
3639 * ixgbe_get_prop - Get a property value out of the configuration file
3640 * ixgbe.conf.
3641 *
3642 * Caller provides the name of the property, a default value, a minimum
3643 * value, and a maximum value.
3644 *
3645 * Return configured value of the property, with default, minimum and
3646 * maximum properly applied.
3647 */
3648 static int
3649 ixgbe_get_prop(ixgbe_t *ixgbe,
3650 char *propname, /* name of the property */
3651 int minval, /* minimum acceptable value */
3652 int maxval, /* maximim acceptable value */
3653 int defval) /* default value */
3654 {
3655 int value;
3656
3657 /*
3658 * Call ddi_prop_get_int() to read the conf settings
3659 */
3660 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3661 DDI_PROP_DONTPASS, propname, defval);
3662 if (value > maxval)
3663 value = maxval;
3664
3665 if (value < minval)
3666 value = minval;
3667
3668 return (value);
3669 }
3670
3671 /*
3672 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3673 */
3674 int
3675 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3676 {
3677 struct ixgbe_hw *hw = &ixgbe->hw;
3678 ixgbe_link_speed advertised = 0;
3679
3680 /*
3681 * Assemble a list of enabled speeds to auto-negotiate with.
3682 */
3683 if (ixgbe->param_en_10000fdx_cap == 1)
3684 advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3685
3686 if (ixgbe->param_en_5000fdx_cap == 1)
3687 advertised |= IXGBE_LINK_SPEED_5GB_FULL;
3688
3689 if (ixgbe->param_en_2500fdx_cap == 1)
3690 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL;
3691
3692 if (ixgbe->param_en_1000fdx_cap == 1)
3693 advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3694
3695 if (ixgbe->param_en_100fdx_cap == 1)
3696 advertised |= IXGBE_LINK_SPEED_100_FULL;
3697
3698 /*
3699 * As a last resort, autoneg with a default list of speeds.
3700 */
3701 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) {
3702 ixgbe_notice(ixgbe, "Invalid link settings. Setting link "
3703 "to autonegotiate with full capabilities.");
3704
3705 if (hw->mac.type == ixgbe_mac_82598EB)
3706 advertised = IXGBE_LINK_SPEED_82598_AUTONEG;
3707 else
3708 advertised = IXGBE_LINK_SPEED_82599_AUTONEG;
3709 }
3710
3711 if (setup_hw) {
3712 if (ixgbe_setup_link(&ixgbe->hw, advertised,
3713 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3714 ixgbe_notice(ixgbe, "Setup link failed on this "
3715 "device.");
3716 return (IXGBE_FAILURE);
3717 }
3718 }
3719
3720 return (IXGBE_SUCCESS);
3721 }
3722
3723 /*
3724 * ixgbe_driver_link_check - Link status processing.
3725 *
3726 * This function can be called in both kernel context and interrupt context
3727 */
3728 static void
3729 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3730 {
3731 struct ixgbe_hw *hw = &ixgbe->hw;
3732 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3733 boolean_t link_up = B_FALSE;
3734 boolean_t link_changed = B_FALSE;
3735
3736 ASSERT(mutex_owned(&ixgbe->gen_lock));
3737
3738 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3739 if (link_up) {
3740 ixgbe->link_check_complete = B_TRUE;
3741
3742 /* Link is up, enable flow control settings */
3743 (void) ixgbe_fc_enable(hw);
3744
3745 /*
3746 * The Link is up, check whether it was marked as down earlier
3747 */
3748 if (ixgbe->link_state != LINK_STATE_UP) {
3749 switch (speed) {
3750 case IXGBE_LINK_SPEED_10GB_FULL:
3751 ixgbe->link_speed = SPEED_10GB;
3752 break;
3753 case IXGBE_LINK_SPEED_5GB_FULL:
3754 ixgbe->link_speed = SPEED_5GB;
3755 break;
3756 case IXGBE_LINK_SPEED_2_5GB_FULL:
3757 ixgbe->link_speed = SPEED_2_5GB;
3758 break;
3759 case IXGBE_LINK_SPEED_1GB_FULL:
3760 ixgbe->link_speed = SPEED_1GB;
3761 break;
3762 case IXGBE_LINK_SPEED_100_FULL:
3763 ixgbe->link_speed = SPEED_100;
3764 }
3765 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3766 ixgbe->link_state = LINK_STATE_UP;
3767 link_changed = B_TRUE;
3768 }
3769 } else {
3770 if (ixgbe->link_check_complete == B_TRUE ||
3771 (ixgbe->link_check_complete == B_FALSE &&
3772 gethrtime() >= ixgbe->link_check_hrtime)) {
3773 /*
3774 * The link is really down
3775 */
3776 ixgbe->link_check_complete = B_TRUE;
3777
3778 if (ixgbe->link_state != LINK_STATE_DOWN) {
3779 ixgbe->link_speed = 0;
3780 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3781 ixgbe->link_state = LINK_STATE_DOWN;
3782 link_changed = B_TRUE;
3783 }
3784 }
3785 }
3786
3787 /*
3788 * If we are in an interrupt context, need to re-enable the
3789 * interrupt, which was automasked
3790 */
3791 if (servicing_interrupt() != 0) {
3792 ixgbe->eims |= IXGBE_EICR_LSC;
3793 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3794 }
3795
3796 if (link_changed) {
3797 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3798 }
3799 }
3800
3801 /*
3802 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3803 */
3804 static void
3805 ixgbe_sfp_check(void *arg)
3806 {
3807 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3808 uint32_t eicr = ixgbe->eicr;
3809 struct ixgbe_hw *hw = &ixgbe->hw;
3810
3811 mutex_enter(&ixgbe->gen_lock);
3812 (void) hw->phy.ops.identify_sfp(hw);
3813 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3814 /* clear the interrupt */
3815 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3816
3817 /* if link up, do multispeed fiber setup */
3818 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3819 B_TRUE);
3820 ixgbe_driver_link_check(ixgbe);
3821 ixgbe_get_hw_state(ixgbe);
3822 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3823 /* clear the interrupt */
3824 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3825
3826 /* if link up, do sfp module setup */
3827 (void) hw->mac.ops.setup_sfp(hw);
3828
3829 /* do multispeed fiber setup */
3830 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3831 B_TRUE);
3832 ixgbe_driver_link_check(ixgbe);
3833 ixgbe_get_hw_state(ixgbe);
3834 }
3835 mutex_exit(&ixgbe->gen_lock);
3836
3837 /*
3838 * We need to fully re-check the link later.
3839 */
3840 ixgbe->link_check_complete = B_FALSE;
3841 ixgbe->link_check_hrtime = gethrtime() +
3842 (IXGBE_LINK_UP_TIME * 100000000ULL);
3843 }
3844
3845 /*
3846 * ixgbe_overtemp_check - overtemp module processing done in taskq
3847 *
3848 * This routine will only be called on adapters with temperature sensor.
3849 * The indication of over-temperature can be either SDP0 interrupt or the link
3850 * status change interrupt.
3851 */
3852 static void
3853 ixgbe_overtemp_check(void *arg)
3854 {
3855 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3856 struct ixgbe_hw *hw = &ixgbe->hw;
3857 uint32_t eicr = ixgbe->eicr;
3858 ixgbe_link_speed speed;
3859 boolean_t link_up;
3860
3861 mutex_enter(&ixgbe->gen_lock);
3862
3863 /* make sure we know current state of link */
3864 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3865
3866 /* check over-temp condition */
3867 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3868 (eicr & IXGBE_EICR_LSC)) {
3869 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3870 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3871
3872 /*
3873 * Disable the adapter interrupts
3874 */
3875 ixgbe_disable_adapter_interrupts(ixgbe);
3876
3877 /*
3878 * Disable Rx/Tx units
3879 */
3880 (void) ixgbe_stop_adapter(hw);
3881
3882 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3883 ixgbe_error(ixgbe,
3884 "Problem: Network adapter has been stopped "
3885 "because it has overheated");
3886 ixgbe_error(ixgbe,
3887 "Action: Restart the computer. "
3888 "If the problem persists, power off the system "
3889 "and replace the adapter");
3890 }
3891 }
3892
3893 /* write to clear the interrupt */
3894 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3895
3896 mutex_exit(&ixgbe->gen_lock);
3897 }
3898
3899 /*
3900 * ixgbe_phy_check - taskq to process interrupts from an external PHY
3901 *
3902 * This routine will only be called on adapters with external PHYs
3903 * (such as X550) that may be trying to raise our attention to some event.
3904 * Currently, this is limited to claiming PHY overtemperature and link status
3905 * change (LSC) events, however this may expand to include other things in
3906 * future adapters.
3907 */
3908 static void
3909 ixgbe_phy_check(void *arg)
3910 {
3911 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3912 struct ixgbe_hw *hw = &ixgbe->hw;
3913 int rv;
3914
3915 mutex_enter(&ixgbe->gen_lock);
3916
3917 /*
3918 * X550 baseT PHY overtemp and LSC events are handled here.
3919 *
3920 * If an overtemp event occurs, it will be reflected in the
3921 * return value of phy.ops.handle_lasi() and the common code will
3922 * automatically power off the baseT PHY. This is our cue to trigger
3923 * an FMA event.
3924 *
3925 * If a link status change event occurs, phy.ops.handle_lasi() will
3926 * automatically initiate a link setup between the integrated KR PHY
3927 * and the external X557 PHY to ensure that the link speed between
3928 * them matches the link speed of the baseT link.
3929 */
3930 rv = ixgbe_handle_lasi(hw);
3931
3932 if (rv == IXGBE_ERR_OVERTEMP) {
3933 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3934
3935 /*
3936 * Disable the adapter interrupts
3937 */
3938 ixgbe_disable_adapter_interrupts(ixgbe);
3939
3940 /*
3941 * Disable Rx/Tx units
3942 */
3943 (void) ixgbe_stop_adapter(hw);
3944
3945 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3946 ixgbe_error(ixgbe,
3947 "Problem: Network adapter has been stopped due to a "
3948 "overtemperature event being detected.");
3949 ixgbe_error(ixgbe,
3950 "Action: Shut down or restart the computer. If the issue "
3951 "persists, please take action in accordance with the "
3952 "recommendations from your system vendor.");
3953 }
3954
3955 mutex_exit(&ixgbe->gen_lock);
3956 }
3957
3958 /*
3959 * ixgbe_link_timer - timer for link status detection
3960 */
3961 static void
3962 ixgbe_link_timer(void *arg)
3963 {
3964 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3965
3966 mutex_enter(&ixgbe->gen_lock);
3967 ixgbe_driver_link_check(ixgbe);
3968 mutex_exit(&ixgbe->gen_lock);
3969 }
3970
3971 /*
3972 * ixgbe_local_timer - Driver watchdog function.
3973 *
3974 * This function will handle the transmit stall check and other routines.
3975 */
3976 static void
3977 ixgbe_local_timer(void *arg)
3978 {
3979 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3980
3981 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3982 goto out;
3983
3984 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3985 ixgbe->reset_count++;
3986 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3987 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3988 goto out;
3989 }
3990
3991 if (ixgbe_stall_check(ixgbe)) {
3992 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3993 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3994
3995 ixgbe->reset_count++;
3996 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3997 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3998 }
3999
4000 out:
4001 ixgbe_restart_watchdog_timer(ixgbe);
4002 }
4003
4004 /*
4005 * ixgbe_stall_check - Check for transmit stall.
4006 *
4007 * This function checks if the adapter is stalled (in transmit).
4008 *
4009 * It is called each time the watchdog timeout is invoked.
4010 * If the transmit descriptor reclaim continuously fails,
4011 * the watchdog value will increment by 1. If the watchdog
4012 * value exceeds the threshold, the ixgbe is assumed to
4013 * have stalled and need to be reset.
4014 */
4015 static boolean_t
4016 ixgbe_stall_check(ixgbe_t *ixgbe)
4017 {
4018 ixgbe_tx_ring_t *tx_ring;
4019 boolean_t result;
4020 int i;
4021
4022 if (ixgbe->link_state != LINK_STATE_UP)
4023 return (B_FALSE);
4024
4025 /*
4026 * If any tx ring is stalled, we'll reset the chipset
4027 */
4028 result = B_FALSE;
4029 for (i = 0; i < ixgbe->num_tx_rings; i++) {
4030 tx_ring = &ixgbe->tx_rings[i];
4031 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
4032 tx_ring->tx_recycle(tx_ring);
4033 }
4034
4035 if (tx_ring->recycle_fail > 0)
4036 tx_ring->stall_watchdog++;
4037 else
4038 tx_ring->stall_watchdog = 0;
4039
4040 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
4041 result = B_TRUE;
4042 break;
4043 }
4044 }
4045
4046 if (result) {
4047 tx_ring->stall_watchdog = 0;
4048 tx_ring->recycle_fail = 0;
4049 }
4050
4051 return (result);
4052 }
4053
4054
4055 /*
4056 * is_valid_mac_addr - Check if the mac address is valid.
4057 */
4058 static boolean_t
4059 is_valid_mac_addr(uint8_t *mac_addr)
4060 {
4061 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
4062 const uint8_t addr_test2[6] =
4063 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
4064
4065 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
4066 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
4067 return (B_FALSE);
4068
4069 return (B_TRUE);
4070 }
4071
4072 static boolean_t
4073 ixgbe_find_mac_address(ixgbe_t *ixgbe)
4074 {
4075 #ifdef __sparc
4076 struct ixgbe_hw *hw = &ixgbe->hw;
4077 uchar_t *bytes;
4078 struct ether_addr sysaddr;
4079 uint_t nelts;
4080 int err;
4081 boolean_t found = B_FALSE;
4082
4083 /*
4084 * The "vendor's factory-set address" may already have
4085 * been extracted from the chip, but if the property
4086 * "local-mac-address" is set we use that instead.
4087 *
4088 * We check whether it looks like an array of 6
4089 * bytes (which it should, if OBP set it). If we can't
4090 * make sense of it this way, we'll ignore it.
4091 */
4092 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4093 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
4094 if (err == DDI_PROP_SUCCESS) {
4095 if (nelts == ETHERADDRL) {
4096 while (nelts--)
4097 hw->mac.addr[nelts] = bytes[nelts];
4098 found = B_TRUE;
4099 }
4100 ddi_prop_free(bytes);
4101 }
4102
4103 /*
4104 * Look up the OBP property "local-mac-address?". If the user has set
4105 * 'local-mac-address? = false', use "the system address" instead.
4106 */
4107 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
4108 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
4109 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
4110 if (localetheraddr(NULL, &sysaddr) != 0) {
4111 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
4112 found = B_TRUE;
4113 }
4114 }
4115 ddi_prop_free(bytes);
4116 }
4117
4118 /*
4119 * Finally(!), if there's a valid "mac-address" property (created
4120 * if we netbooted from this interface), we must use this instead
4121 * of any of the above to ensure that the NFS/install server doesn't
4122 * get confused by the address changing as illumos takes over!
4123 */
4124 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4125 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
4126 if (err == DDI_PROP_SUCCESS) {
4127 if (nelts == ETHERADDRL) {
4128 while (nelts--)
4129 hw->mac.addr[nelts] = bytes[nelts];
4130 found = B_TRUE;
4131 }
4132 ddi_prop_free(bytes);
4133 }
4134
4135 if (found) {
4136 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
4137 return (B_TRUE);
4138 }
4139 #else
4140 _NOTE(ARGUNUSED(ixgbe));
4141 #endif
4142
4143 return (B_TRUE);
4144 }
4145
4146 #pragma inline(ixgbe_arm_watchdog_timer)
4147 static void
4148 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
4149 {
4150 /*
4151 * Fire a watchdog timer
4152 */
4153 ixgbe->watchdog_tid =
4154 timeout(ixgbe_local_timer,
4155 (void *)ixgbe, 1 * drv_usectohz(1000000));
4156
4157 }
4158
4159 /*
4160 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
4161 */
4162 void
4163 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
4164 {
4165 mutex_enter(&ixgbe->watchdog_lock);
4166
4167 if (!ixgbe->watchdog_enable) {
4168 ixgbe->watchdog_enable = B_TRUE;
4169 ixgbe->watchdog_start = B_TRUE;
4170 ixgbe_arm_watchdog_timer(ixgbe);
4171 }
4172
4173 mutex_exit(&ixgbe->watchdog_lock);
4174 }
4175
4176 /*
4177 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
4178 */
4179 void
4180 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
4181 {
4182 timeout_id_t tid;
4183
4184 mutex_enter(&ixgbe->watchdog_lock);
4185
4186 ixgbe->watchdog_enable = B_FALSE;
4187 ixgbe->watchdog_start = B_FALSE;
4188 tid = ixgbe->watchdog_tid;
4189 ixgbe->watchdog_tid = 0;
4190
4191 mutex_exit(&ixgbe->watchdog_lock);
4192
4193 if (tid != 0)
4194 (void) untimeout(tid);
4195 }
4196
4197 /*
4198 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
4199 */
4200 void
4201 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
4202 {
4203 mutex_enter(&ixgbe->watchdog_lock);
4204
4205 if (ixgbe->watchdog_enable) {
4206 if (!ixgbe->watchdog_start) {
4207 ixgbe->watchdog_start = B_TRUE;
4208 ixgbe_arm_watchdog_timer(ixgbe);
4209 }
4210 }
4211
4212 mutex_exit(&ixgbe->watchdog_lock);
4213 }
4214
4215 /*
4216 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
4217 */
4218 static void
4219 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
4220 {
4221 mutex_enter(&ixgbe->watchdog_lock);
4222
4223 if (ixgbe->watchdog_start)
4224 ixgbe_arm_watchdog_timer(ixgbe);
4225
4226 mutex_exit(&ixgbe->watchdog_lock);
4227 }
4228
4229 /*
4230 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
4231 */
4232 void
4233 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
4234 {
4235 timeout_id_t tid;
4236
4237 mutex_enter(&ixgbe->watchdog_lock);
4238
4239 ixgbe->watchdog_start = B_FALSE;
4240 tid = ixgbe->watchdog_tid;
4241 ixgbe->watchdog_tid = 0;
4242
4243 mutex_exit(&ixgbe->watchdog_lock);
4244
4245 if (tid != 0)
4246 (void) untimeout(tid);
4247 }
4248
4249 /*
4250 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
4251 */
4252 static void
4253 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
4254 {
4255 struct ixgbe_hw *hw = &ixgbe->hw;
4256
4257 /*
4258 * mask all interrupts off
4259 */
4260 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
4261
4262 /*
4263 * for MSI-X, also disable autoclear
4264 */
4265 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4266 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
4267 }
4268
4269 IXGBE_WRITE_FLUSH(hw);
4270 }
4271
4272 /*
4273 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
4274 */
4275 static void
4276 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
4277 {
4278 struct ixgbe_hw *hw = &ixgbe->hw;
4279 uint32_t eiac, eiam;
4280 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4281
4282 /* interrupt types to enable */
4283 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
4284 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
4285 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
4286
4287 /* enable automask on "other" causes that this adapter can generate */
4288 eiam = ixgbe->capab->other_intr;
4289
4290 /*
4291 * msi-x mode
4292 */
4293 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4294 /* enable autoclear but not on bits 29:20 */
4295 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
4296
4297 /* general purpose interrupt enable */
4298 gpie |= (IXGBE_GPIE_MSIX_MODE
4299 | IXGBE_GPIE_PBA_SUPPORT
4300 | IXGBE_GPIE_OCD
4301 | IXGBE_GPIE_EIAME);
4302 /*
4303 * non-msi-x mode
4304 */
4305 } else {
4306
4307 /* disable autoclear, leave gpie at default */
4308 eiac = 0;
4309
4310 /*
4311 * General purpose interrupt enable.
4312 * For 82599, X540 and X550, extended interrupt
4313 * automask enable only in MSI or MSI-X mode
4314 */
4315 if ((hw->mac.type == ixgbe_mac_82598EB) ||
4316 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
4317 gpie |= IXGBE_GPIE_EIAME;
4318 }
4319 }
4320
4321 /* Enable specific "other" interrupt types */
4322 switch (hw->mac.type) {
4323 case ixgbe_mac_82598EB:
4324 gpie |= ixgbe->capab->other_gpie;
4325 break;
4326
4327 case ixgbe_mac_82599EB:
4328 case ixgbe_mac_X540:
4329 case ixgbe_mac_X550:
4330 case ixgbe_mac_X550EM_x:
4331 case ixgbe_mac_X550EM_a:
4332 gpie |= ixgbe->capab->other_gpie;
4333
4334 /* Enable RSC Delay 8us when LRO enabled */
4335 if (ixgbe->lro_enable) {
4336 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
4337 }
4338 break;
4339
4340 default:
4341 break;
4342 }
4343
4344 /* write to interrupt control registers */
4345 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4346 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
4347 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
4348 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4349 IXGBE_WRITE_FLUSH(hw);
4350 }
4351
4352 /*
4353 * ixgbe_loopback_ioctl - Loopback support.
4354 */
4355 enum ioc_reply
4356 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
4357 {
4358 lb_info_sz_t *lbsp;
4359 lb_property_t *lbpp;
4360 uint32_t *lbmp;
4361 uint32_t size;
4362 uint32_t value;
4363
4364 if (mp->b_cont == NULL)
4365 return (IOC_INVAL);
4366
4367 switch (iocp->ioc_cmd) {
4368 default:
4369 return (IOC_INVAL);
4370
4371 case LB_GET_INFO_SIZE:
4372 size = sizeof (lb_info_sz_t);
4373 if (iocp->ioc_count != size)
4374 return (IOC_INVAL);
4375
4376 value = sizeof (lb_normal);
4377 value += sizeof (lb_mac);
4378 value += sizeof (lb_external);
4379
4380 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
4381 *lbsp = value;
4382 break;
4383
4384 case LB_GET_INFO:
4385 value = sizeof (lb_normal);
4386 value += sizeof (lb_mac);
4387 value += sizeof (lb_external);
4388
4389 size = value;
4390 if (iocp->ioc_count != size)
4391 return (IOC_INVAL);
4392
4393 value = 0;
4394 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
4395
4396 lbpp[value++] = lb_normal;
4397 lbpp[value++] = lb_mac;
4398 lbpp[value++] = lb_external;
4399 break;
4400
4401 case LB_GET_MODE:
4402 size = sizeof (uint32_t);
4403 if (iocp->ioc_count != size)
4404 return (IOC_INVAL);
4405
4406 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4407 *lbmp = ixgbe->loopback_mode;
4408 break;
4409
4410 case LB_SET_MODE:
4411 size = 0;
4412 if (iocp->ioc_count != sizeof (uint32_t))
4413 return (IOC_INVAL);
4414
4415 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4416 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
4417 return (IOC_INVAL);
4418 break;
4419 }
4420
4421 iocp->ioc_count = size;
4422 iocp->ioc_error = 0;
4423
4424 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4425 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4426 return (IOC_INVAL);
4427 }
4428
4429 return (IOC_REPLY);
4430 }
4431
4432 /*
4433 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
4434 */
4435 static boolean_t
4436 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
4437 {
4438 if (mode == ixgbe->loopback_mode)
4439 return (B_TRUE);
4440
4441 ixgbe->loopback_mode = mode;
4442
4443 if (mode == IXGBE_LB_NONE) {
4444 /*
4445 * Reset the chip
4446 */
4447 (void) ixgbe_reset(ixgbe);
4448 return (B_TRUE);
4449 }
4450
4451 mutex_enter(&ixgbe->gen_lock);
4452
4453 switch (mode) {
4454 default:
4455 mutex_exit(&ixgbe->gen_lock);
4456 return (B_FALSE);
4457
4458 case IXGBE_LB_EXTERNAL:
4459 break;
4460
4461 case IXGBE_LB_INTERNAL_MAC:
4462 ixgbe_set_internal_mac_loopback(ixgbe);
4463 break;
4464 }
4465
4466 mutex_exit(&ixgbe->gen_lock);
4467
4468 return (B_TRUE);
4469 }
4470
4471 /*
4472 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4473 */
4474 static void
4475 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4476 {
4477 struct ixgbe_hw *hw;
4478 uint32_t reg;
4479 uint8_t atlas;
4480
4481 hw = &ixgbe->hw;
4482
4483 /*
4484 * Setup MAC loopback
4485 */
4486 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4487 reg |= IXGBE_HLREG0_LPBK;
4488 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4489
4490 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4491 reg &= ~IXGBE_AUTOC_LMS_MASK;
4492 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4493
4494 /*
4495 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4496 */
4497 switch (hw->mac.type) {
4498 case ixgbe_mac_82598EB:
4499 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4500 &atlas);
4501 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4502 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4503 atlas);
4504
4505 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4506 &atlas);
4507 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4508 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4509 atlas);
4510
4511 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4512 &atlas);
4513 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4514 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4515 atlas);
4516
4517 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4518 &atlas);
4519 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4520 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4521 atlas);
4522 break;
4523
4524 case ixgbe_mac_82599EB:
4525 case ixgbe_mac_X540:
4526 case ixgbe_mac_X550:
4527 case ixgbe_mac_X550EM_x:
4528 case ixgbe_mac_X550EM_a:
4529 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4530 reg |= (IXGBE_AUTOC_FLU |
4531 IXGBE_AUTOC_10G_KX4);
4532 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4533
4534 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4535 B_FALSE);
4536 break;
4537
4538 default:
4539 break;
4540 }
4541 }
4542
4543 #pragma inline(ixgbe_intr_rx_work)
4544 /*
4545 * ixgbe_intr_rx_work - RX processing of ISR.
4546 */
4547 static void
4548 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4549 {
4550 mblk_t *mp;
4551
4552 mutex_enter(&rx_ring->rx_lock);
4553
4554 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4555 mutex_exit(&rx_ring->rx_lock);
4556
4557 if (mp != NULL)
4558 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4559 rx_ring->ring_gen_num);
4560 }
4561
4562 #pragma inline(ixgbe_intr_tx_work)
4563 /*
4564 * ixgbe_intr_tx_work - TX processing of ISR.
4565 */
4566 static void
4567 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4568 {
4569 ixgbe_t *ixgbe = tx_ring->ixgbe;
4570
4571 /*
4572 * Recycle the tx descriptors
4573 */
4574 tx_ring->tx_recycle(tx_ring);
4575
4576 /*
4577 * Schedule the re-transmit
4578 */
4579 if (tx_ring->reschedule &&
4580 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4581 tx_ring->reschedule = B_FALSE;
4582 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4583 tx_ring->ring_handle);
4584 tx_ring->stat_reschedule++;
4585 }
4586 }
4587
4588 #pragma inline(ixgbe_intr_other_work)
4589 /*
4590 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4591 */
4592 static void
4593 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4594 {
4595 struct ixgbe_hw *hw = &ixgbe->hw;
4596
4597 ASSERT(mutex_owned(&ixgbe->gen_lock));
4598
4599 /*
4600 * handle link status change
4601 */
4602 if (eicr & IXGBE_EICR_LSC) {
4603 ixgbe_driver_link_check(ixgbe);
4604 ixgbe_get_hw_state(ixgbe);
4605 }
4606
4607 /*
4608 * check for fan failure on adapters with fans
4609 */
4610 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4611 (eicr & IXGBE_EICR_GPI_SDP1)) {
4612 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4613
4614 /*
4615 * Disable the adapter interrupts
4616 */
4617 ixgbe_disable_adapter_interrupts(ixgbe);
4618
4619 /*
4620 * Disable Rx/Tx units
4621 */
4622 (void) ixgbe_stop_adapter(&ixgbe->hw);
4623
4624 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4625 ixgbe_error(ixgbe,
4626 "Problem: Network adapter has been stopped "
4627 "because the fan has stopped.\n");
4628 ixgbe_error(ixgbe,
4629 "Action: Replace the adapter.\n");
4630
4631 /* re-enable the interrupt, which was automasked */
4632 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4633 }
4634
4635 /*
4636 * Do SFP check for adapters with hot-plug capability
4637 */
4638 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4639 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4640 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4641 ixgbe->eicr = eicr;
4642 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4643 ixgbe_sfp_check, (void *)ixgbe,
4644 DDI_NOSLEEP)) != DDI_SUCCESS) {
4645 ixgbe_log(ixgbe, "No memory available to dispatch "
4646 "taskq for SFP check");
4647 }
4648 }
4649
4650 /*
4651 * Do over-temperature check for adapters with temp sensor
4652 */
4653 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4654 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) ||
4655 (eicr & IXGBE_EICR_LSC))) {
4656 ixgbe->eicr = eicr;
4657 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4658 ixgbe_overtemp_check, (void *)ixgbe,
4659 DDI_NOSLEEP)) != DDI_SUCCESS) {
4660 ixgbe_log(ixgbe, "No memory available to dispatch "
4661 "taskq for overtemp check");
4662 }
4663 }
4664
4665 /*
4666 * Process an external PHY interrupt
4667 */
4668 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
4669 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
4670 ixgbe->eicr = eicr;
4671 if ((ddi_taskq_dispatch(ixgbe->phy_taskq,
4672 ixgbe_phy_check, (void *)ixgbe,
4673 DDI_NOSLEEP)) != DDI_SUCCESS) {
4674 ixgbe_log(ixgbe, "No memory available to dispatch "
4675 "taskq for PHY check");
4676 }
4677 }
4678 }
4679
4680 /*
4681 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4682 */
4683 static uint_t
4684 ixgbe_intr_legacy(void *arg1, void *arg2)
4685 {
4686 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4687 struct ixgbe_hw *hw = &ixgbe->hw;
4688 ixgbe_tx_ring_t *tx_ring;
4689 ixgbe_rx_ring_t *rx_ring;
4690 uint32_t eicr;
4691 mblk_t *mp;
4692 boolean_t tx_reschedule;
4693 uint_t result;
4694
4695 _NOTE(ARGUNUSED(arg2));
4696
4697 mutex_enter(&ixgbe->gen_lock);
4698 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4699 mutex_exit(&ixgbe->gen_lock);
4700 return (DDI_INTR_UNCLAIMED);
4701 }
4702
4703 mp = NULL;
4704 tx_reschedule = B_FALSE;
4705
4706 /*
4707 * Any bit set in eicr: claim this interrupt
4708 */
4709 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4710
4711 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4712 mutex_exit(&ixgbe->gen_lock);
4713 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4714 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4715 return (DDI_INTR_CLAIMED);
4716 }
4717
4718 if (eicr) {
4719 /*
4720 * For legacy interrupt, we have only one interrupt,
4721 * so we have only one rx ring and one tx ring enabled.
4722 */
4723 ASSERT(ixgbe->num_rx_rings == 1);
4724 ASSERT(ixgbe->num_tx_rings == 1);
4725
4726 /*
4727 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4728 */
4729 if (eicr & 0x1) {
4730 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4731 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4732 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4733 /*
4734 * Clean the rx descriptors
4735 */
4736 rx_ring = &ixgbe->rx_rings[0];
4737 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4738 }
4739
4740 /*
4741 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4742 */
4743 if (eicr & 0x2) {
4744 /*
4745 * Recycle the tx descriptors
4746 */
4747 tx_ring = &ixgbe->tx_rings[0];
4748 tx_ring->tx_recycle(tx_ring);
4749
4750 /*
4751 * Schedule the re-transmit
4752 */
4753 tx_reschedule = (tx_ring->reschedule &&
4754 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4755 }
4756
4757 /* any interrupt type other than tx/rx */
4758 if (eicr & ixgbe->capab->other_intr) {
4759 switch (hw->mac.type) {
4760 case ixgbe_mac_82598EB:
4761 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4762 break;
4763
4764 case ixgbe_mac_82599EB:
4765 case ixgbe_mac_X540:
4766 case ixgbe_mac_X550:
4767 case ixgbe_mac_X550EM_x:
4768 case ixgbe_mac_X550EM_a:
4769 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4770 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4771 break;
4772
4773 default:
4774 break;
4775 }
4776 ixgbe_intr_other_work(ixgbe, eicr);
4777 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4778 }
4779
4780 mutex_exit(&ixgbe->gen_lock);
4781
4782 result = DDI_INTR_CLAIMED;
4783 } else {
4784 mutex_exit(&ixgbe->gen_lock);
4785
4786 /*
4787 * No interrupt cause bits set: don't claim this interrupt.
4788 */
4789 result = DDI_INTR_UNCLAIMED;
4790 }
4791
4792 /* re-enable the interrupts which were automasked */
4793 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4794
4795 /*
4796 * Do the following work outside of the gen_lock
4797 */
4798 if (mp != NULL) {
4799 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4800 rx_ring->ring_gen_num);
4801 }
4802
4803 if (tx_reschedule) {
4804 tx_ring->reschedule = B_FALSE;
4805 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4806 tx_ring->stat_reschedule++;
4807 }
4808
4809 return (result);
4810 }
4811
4812 /*
4813 * ixgbe_intr_msi - Interrupt handler for MSI.
4814 */
4815 static uint_t
4816 ixgbe_intr_msi(void *arg1, void *arg2)
4817 {
4818 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4819 struct ixgbe_hw *hw = &ixgbe->hw;
4820 uint32_t eicr;
4821
4822 _NOTE(ARGUNUSED(arg2));
4823
4824 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4825
4826 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4827 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4828 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4829 return (DDI_INTR_CLAIMED);
4830 }
4831
4832 /*
4833 * For MSI interrupt, we have only one vector,
4834 * so we have only one rx ring and one tx ring enabled.
4835 */
4836 ASSERT(ixgbe->num_rx_rings == 1);
4837 ASSERT(ixgbe->num_tx_rings == 1);
4838
4839 /*
4840 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4841 */
4842 if (eicr & 0x1) {
4843 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4844 }
4845
4846 /*
4847 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4848 */
4849 if (eicr & 0x2) {
4850 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4851 }
4852
4853 /* any interrupt type other than tx/rx */
4854 if (eicr & ixgbe->capab->other_intr) {
4855 mutex_enter(&ixgbe->gen_lock);
4856 switch (hw->mac.type) {
4857 case ixgbe_mac_82598EB:
4858 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4859 break;
4860
4861 case ixgbe_mac_82599EB:
4862 case ixgbe_mac_X540:
4863 case ixgbe_mac_X550:
4864 case ixgbe_mac_X550EM_x:
4865 case ixgbe_mac_X550EM_a:
4866 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4867 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4868 break;
4869
4870 default:
4871 break;
4872 }
4873 ixgbe_intr_other_work(ixgbe, eicr);
4874 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4875 mutex_exit(&ixgbe->gen_lock);
4876 }
4877
4878 /* re-enable the interrupts which were automasked */
4879 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4880
4881 return (DDI_INTR_CLAIMED);
4882 }
4883
4884 /*
4885 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4886 */
4887 static uint_t
4888 ixgbe_intr_msix(void *arg1, void *arg2)
4889 {
4890 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4891 ixgbe_t *ixgbe = vect->ixgbe;
4892 struct ixgbe_hw *hw = &ixgbe->hw;
4893 uint32_t eicr;
4894 int r_idx = 0;
4895
4896 _NOTE(ARGUNUSED(arg2));
4897
4898 /*
4899 * Clean each rx ring that has its bit set in the map
4900 */
4901 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4902 while (r_idx >= 0) {
4903 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4904 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4905 (ixgbe->num_rx_rings - 1));
4906 }
4907
4908 /*
4909 * Clean each tx ring that has its bit set in the map
4910 */
4911 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4912 while (r_idx >= 0) {
4913 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4914 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4915 (ixgbe->num_tx_rings - 1));
4916 }
4917
4918
4919 /*
4920 * Clean other interrupt (link change) that has its bit set in the map
4921 */
4922 if (BT_TEST(vect->other_map, 0) == 1) {
4923 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4924
4925 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4926 DDI_FM_OK) {
4927 ddi_fm_service_impact(ixgbe->dip,
4928 DDI_SERVICE_DEGRADED);
4929 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4930 return (DDI_INTR_CLAIMED);
4931 }
4932
4933 /*
4934 * Check "other" cause bits: any interrupt type other than tx/rx
4935 */
4936 if (eicr & ixgbe->capab->other_intr) {
4937 mutex_enter(&ixgbe->gen_lock);
4938 switch (hw->mac.type) {
4939 case ixgbe_mac_82598EB:
4940 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4941 ixgbe_intr_other_work(ixgbe, eicr);
4942 break;
4943
4944 case ixgbe_mac_82599EB:
4945 case ixgbe_mac_X540:
4946 case ixgbe_mac_X550:
4947 case ixgbe_mac_X550EM_x:
4948 case ixgbe_mac_X550EM_a:
4949 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4950 ixgbe_intr_other_work(ixgbe, eicr);
4951 break;
4952
4953 default:
4954 break;
4955 }
4956 mutex_exit(&ixgbe->gen_lock);
4957 }
4958
4959 /* re-enable the interrupts which were automasked */
4960 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4961 }
4962
4963 return (DDI_INTR_CLAIMED);
4964 }
4965
4966 /*
4967 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4968 *
4969 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4970 * if not successful, try Legacy.
4971 * ixgbe->intr_force can be used to force sequence to start with
4972 * any of the 3 types.
4973 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4974 */
4975 static int
4976 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4977 {
4978 dev_info_t *devinfo;
4979 int intr_types;
4980 int rc;
4981
4982 devinfo = ixgbe->dip;
4983
4984 /*
4985 * Get supported interrupt types
4986 */
4987 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4988
4989 if (rc != DDI_SUCCESS) {
4990 ixgbe_log(ixgbe,
4991 "Get supported interrupt types failed: %d", rc);
4992 return (IXGBE_FAILURE);
4993 }
4994 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4995
4996 ixgbe->intr_type = 0;
4997
4998 /*
4999 * Install MSI-X interrupts
5000 */
5001 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
5002 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
5003 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
5004 if (rc == IXGBE_SUCCESS)
5005 return (IXGBE_SUCCESS);
5006
5007 ixgbe_log(ixgbe,
5008 "Allocate MSI-X failed, trying MSI interrupts...");
5009 }
5010
5011 /*
5012 * MSI-X not used, force rings and groups to 1
5013 */
5014 ixgbe->num_rx_rings = 1;
5015 ixgbe->num_rx_groups = 1;
5016 ixgbe->num_tx_rings = 1;
5017 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
5018 ixgbe_log(ixgbe,
5019 "MSI-X not used, force rings and groups number to 1");
5020
5021 /*
5022 * Install MSI interrupts
5023 */
5024 if ((intr_types & DDI_INTR_TYPE_MSI) &&
5025 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
5026 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
5027 if (rc == IXGBE_SUCCESS)
5028 return (IXGBE_SUCCESS);
5029
5030 ixgbe_log(ixgbe,
5031 "Allocate MSI failed, trying Legacy interrupts...");
5032 }
5033
5034 /*
5035 * Install legacy interrupts
5036 */
5037 if (intr_types & DDI_INTR_TYPE_FIXED) {
5038 /*
5039 * Disallow legacy interrupts for X550. X550 has a silicon
5040 * bug which prevents Shared Legacy interrupts from working.
5041 * For details, please reference:
5042 *
5043 * Intel Ethernet Controller X550 Specification Update rev. 2.1
5044 * May 2016, erratum 22: PCIe Interrupt Status Bit
5045 */
5046 if (ixgbe->hw.mac.type == ixgbe_mac_X550 ||
5047 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x ||
5048 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a ||
5049 ixgbe->hw.mac.type == ixgbe_mac_X550_vf ||
5050 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf ||
5051 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a_vf) {
5052 ixgbe_log(ixgbe,
5053 "Legacy interrupts are not supported on this "
5054 "adapter. Please use MSI or MSI-X instead.");
5055 return (IXGBE_FAILURE);
5056 }
5057 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
5058 if (rc == IXGBE_SUCCESS)
5059 return (IXGBE_SUCCESS);
5060
5061 ixgbe_log(ixgbe,
5062 "Allocate Legacy interrupts failed");
5063 }
5064
5065 /*
5066 * If none of the 3 types succeeded, return failure
5067 */
5068 return (IXGBE_FAILURE);
5069 }
5070
5071 /*
5072 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
5073 *
5074 * For legacy and MSI, only 1 handle is needed. For MSI-X,
5075 * if fewer than 2 handles are available, return failure.
5076 * Upon success, this maps the vectors to rx and tx rings for
5077 * interrupts.
5078 */
5079 static int
5080 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
5081 {
5082 dev_info_t *devinfo;
5083 int request, count, actual;
5084 int minimum;
5085 int rc;
5086 uint32_t ring_per_group;
5087
5088 devinfo = ixgbe->dip;
5089
5090 switch (intr_type) {
5091 case DDI_INTR_TYPE_FIXED:
5092 request = 1; /* Request 1 legacy interrupt handle */
5093 minimum = 1;
5094 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
5095 break;
5096
5097 case DDI_INTR_TYPE_MSI:
5098 request = 1; /* Request 1 MSI interrupt handle */
5099 minimum = 1;
5100 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
5101 break;
5102
5103 case DDI_INTR_TYPE_MSIX:
5104 /*
5105 * Best number of vectors for the adapter is
5106 * (# rx rings + # tx rings), however we will
5107 * limit the request number.
5108 */
5109 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
5110 if (request > ixgbe->capab->max_ring_vect)
5111 request = ixgbe->capab->max_ring_vect;
5112 minimum = 1;
5113 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
5114 break;
5115
5116 default:
5117 ixgbe_log(ixgbe,
5118 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
5119 intr_type);
5120 return (IXGBE_FAILURE);
5121 }
5122 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
5123 request, minimum);
5124
5125 /*
5126 * Get number of supported interrupts
5127 */
5128 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
5129 if ((rc != DDI_SUCCESS) || (count < minimum)) {
5130 ixgbe_log(ixgbe,
5131 "Get interrupt number failed. Return: %d, count: %d",
5132 rc, count);
5133 return (IXGBE_FAILURE);
5134 }
5135 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
5136
5137 actual = 0;
5138 ixgbe->intr_cnt = 0;
5139 ixgbe->intr_cnt_max = 0;
5140 ixgbe->intr_cnt_min = 0;
5141
5142 /*
5143 * Allocate an array of interrupt handles
5144 */
5145 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
5146 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
5147
5148 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
5149 request, &actual, DDI_INTR_ALLOC_NORMAL);
5150 if (rc != DDI_SUCCESS) {
5151 ixgbe_log(ixgbe, "Allocate interrupts failed. "
5152 "return: %d, request: %d, actual: %d",
5153 rc, request, actual);
5154 goto alloc_handle_fail;
5155 }
5156 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
5157
5158 /*
5159 * upper/lower limit of interrupts
5160 */
5161 ixgbe->intr_cnt = actual;
5162 ixgbe->intr_cnt_max = request;
5163 ixgbe->intr_cnt_min = minimum;
5164
5165 /*
5166 * rss number per group should not exceed the rx interrupt number,
5167 * else need to adjust rx ring number.
5168 */
5169 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5170 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
5171 if (actual < ring_per_group) {
5172 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
5173 ixgbe_setup_vmdq_rss_conf(ixgbe);
5174 }
5175
5176 /*
5177 * Now we know the actual number of vectors. Here we map the vector
5178 * to other, rx rings and tx ring.
5179 */
5180 if (actual < minimum) {
5181 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
5182 actual);
5183 goto alloc_handle_fail;
5184 }
5185
5186 /*
5187 * Get priority for first vector, assume remaining are all the same
5188 */
5189 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
5190 if (rc != DDI_SUCCESS) {
5191 ixgbe_log(ixgbe,
5192 "Get interrupt priority failed: %d", rc);
5193 goto alloc_handle_fail;
5194 }
5195
5196 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
5197 if (rc != DDI_SUCCESS) {
5198 ixgbe_log(ixgbe,
5199 "Get interrupt cap failed: %d", rc);
5200 goto alloc_handle_fail;
5201 }
5202
5203 ixgbe->intr_type = intr_type;
5204
5205 return (IXGBE_SUCCESS);
5206
5207 alloc_handle_fail:
5208 ixgbe_rem_intrs(ixgbe);
5209
5210 return (IXGBE_FAILURE);
5211 }
5212
5213 /*
5214 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
5215 *
5216 * Before adding the interrupt handlers, the interrupt vectors have
5217 * been allocated, and the rx/tx rings have also been allocated.
5218 */
5219 static int
5220 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
5221 {
5222 int vector = 0;
5223 int rc;
5224
5225 switch (ixgbe->intr_type) {
5226 case DDI_INTR_TYPE_MSIX:
5227 /*
5228 * Add interrupt handler for all vectors
5229 */
5230 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
5231 /*
5232 * install pointer to vect_map[vector]
5233 */
5234 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5235 (ddi_intr_handler_t *)ixgbe_intr_msix,
5236 (void *)&ixgbe->vect_map[vector], NULL);
5237
5238 if (rc != DDI_SUCCESS) {
5239 ixgbe_log(ixgbe,
5240 "Add interrupt handler failed. "
5241 "return: %d, vector: %d", rc, vector);
5242 for (vector--; vector >= 0; vector--) {
5243 (void) ddi_intr_remove_handler(
5244 ixgbe->htable[vector]);
5245 }
5246 return (IXGBE_FAILURE);
5247 }
5248 }
5249
5250 break;
5251
5252 case DDI_INTR_TYPE_MSI:
5253 /*
5254 * Add interrupt handlers for the only vector
5255 */
5256 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5257 (ddi_intr_handler_t *)ixgbe_intr_msi,
5258 (void *)ixgbe, NULL);
5259
5260 if (rc != DDI_SUCCESS) {
5261 ixgbe_log(ixgbe,
5262 "Add MSI interrupt handler failed: %d", rc);
5263 return (IXGBE_FAILURE);
5264 }
5265
5266 break;
5267
5268 case DDI_INTR_TYPE_FIXED:
5269 /*
5270 * Add interrupt handlers for the only vector
5271 */
5272 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5273 (ddi_intr_handler_t *)ixgbe_intr_legacy,
5274 (void *)ixgbe, NULL);
5275
5276 if (rc != DDI_SUCCESS) {
5277 ixgbe_log(ixgbe,
5278 "Add legacy interrupt handler failed: %d", rc);
5279 return (IXGBE_FAILURE);
5280 }
5281
5282 break;
5283
5284 default:
5285 return (IXGBE_FAILURE);
5286 }
5287
5288 return (IXGBE_SUCCESS);
5289 }
5290
5291 #pragma inline(ixgbe_map_rxring_to_vector)
5292 /*
5293 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
5294 */
5295 static void
5296 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
5297 {
5298 /*
5299 * Set bit in map
5300 */
5301 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5302
5303 /*
5304 * Count bits set
5305 */
5306 ixgbe->vect_map[v_idx].rxr_cnt++;
5307
5308 /*
5309 * Remember bit position
5310 */
5311 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
5312 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
5313 }
5314
5315 #pragma inline(ixgbe_map_txring_to_vector)
5316 /*
5317 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
5318 */
5319 static void
5320 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
5321 {
5322 /*
5323 * Set bit in map
5324 */
5325 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
5326
5327 /*
5328 * Count bits set
5329 */
5330 ixgbe->vect_map[v_idx].txr_cnt++;
5331
5332 /*
5333 * Remember bit position
5334 */
5335 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
5336 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
5337 }
5338
5339 /*
5340 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
5341 * allocation register (IVAR).
5342 * cause:
5343 * -1 : other cause
5344 * 0 : rx
5345 * 1 : tx
5346 */
5347 static void
5348 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
5349 int8_t cause)
5350 {
5351 struct ixgbe_hw *hw = &ixgbe->hw;
5352 u32 ivar, index;
5353
5354 switch (hw->mac.type) {
5355 case ixgbe_mac_82598EB:
5356 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5357 if (cause == -1) {
5358 cause = 0;
5359 }
5360 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5361 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5362 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
5363 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
5364 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5365 break;
5366
5367 case ixgbe_mac_82599EB:
5368 case ixgbe_mac_X540:
5369 case ixgbe_mac_X550:
5370 case ixgbe_mac_X550EM_x:
5371 case ixgbe_mac_X550EM_a:
5372 if (cause == -1) {
5373 /* other causes */
5374 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5375 index = (intr_alloc_entry & 1) * 8;
5376 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5377 ivar &= ~(0xFF << index);
5378 ivar |= (msix_vector << index);
5379 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5380 } else {
5381 /* tx or rx causes */
5382 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5383 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5384 ivar = IXGBE_READ_REG(hw,
5385 IXGBE_IVAR(intr_alloc_entry >> 1));
5386 ivar &= ~(0xFF << index);
5387 ivar |= (msix_vector << index);
5388 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5389 ivar);
5390 }
5391 break;
5392
5393 default:
5394 break;
5395 }
5396 }
5397
5398 /*
5399 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
5400 * given interrupt vector allocation register (IVAR).
5401 * cause:
5402 * -1 : other cause
5403 * 0 : rx
5404 * 1 : tx
5405 */
5406 static void
5407 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5408 {
5409 struct ixgbe_hw *hw = &ixgbe->hw;
5410 u32 ivar, index;
5411
5412 switch (hw->mac.type) {
5413 case ixgbe_mac_82598EB:
5414 if (cause == -1) {
5415 cause = 0;
5416 }
5417 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5418 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5419 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
5420 (intr_alloc_entry & 0x3)));
5421 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5422 break;
5423
5424 case ixgbe_mac_82599EB:
5425 case ixgbe_mac_X540:
5426 case ixgbe_mac_X550:
5427 case ixgbe_mac_X550EM_x:
5428 case ixgbe_mac_X550EM_a:
5429 if (cause == -1) {
5430 /* other causes */
5431 index = (intr_alloc_entry & 1) * 8;
5432 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5433 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5434 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5435 } else {
5436 /* tx or rx causes */
5437 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5438 ivar = IXGBE_READ_REG(hw,
5439 IXGBE_IVAR(intr_alloc_entry >> 1));
5440 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5441 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5442 ivar);
5443 }
5444 break;
5445
5446 default:
5447 break;
5448 }
5449 }
5450
5451 /*
5452 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
5453 * given interrupt vector allocation register (IVAR).
5454 * cause:
5455 * -1 : other cause
5456 * 0 : rx
5457 * 1 : tx
5458 */
5459 static void
5460 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5461 {
5462 struct ixgbe_hw *hw = &ixgbe->hw;
5463 u32 ivar, index;
5464
5465 switch (hw->mac.type) {
5466 case ixgbe_mac_82598EB:
5467 if (cause == -1) {
5468 cause = 0;
5469 }
5470 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5471 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5472 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5473 (intr_alloc_entry & 0x3)));
5474 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5475 break;
5476
5477 case ixgbe_mac_82599EB:
5478 case ixgbe_mac_X540:
5479 case ixgbe_mac_X550:
5480 case ixgbe_mac_X550EM_x:
5481 case ixgbe_mac_X550EM_a:
5482 if (cause == -1) {
5483 /* other causes */
5484 index = (intr_alloc_entry & 1) * 8;
5485 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5486 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5487 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5488 } else {
5489 /* tx or rx causes */
5490 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5491 ivar = IXGBE_READ_REG(hw,
5492 IXGBE_IVAR(intr_alloc_entry >> 1));
5493 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5494 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5495 ivar);
5496 }
5497 break;
5498
5499 default:
5500 break;
5501 }
5502 }
5503
5504 /*
5505 * Convert the rx ring index driver maintained to the rx ring index
5506 * in h/w.
5507 */
5508 static uint32_t
5509 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5510 {
5511
5512 struct ixgbe_hw *hw = &ixgbe->hw;
5513 uint32_t rx_ring_per_group, hw_rx_index;
5514
5515 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5516 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5517 return (sw_rx_index);
5518 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5519 switch (hw->mac.type) {
5520 case ixgbe_mac_82598EB:
5521 return (sw_rx_index);
5522
5523 case ixgbe_mac_82599EB:
5524 case ixgbe_mac_X540:
5525 case ixgbe_mac_X550:
5526 case ixgbe_mac_X550EM_x:
5527 case ixgbe_mac_X550EM_a:
5528 return (sw_rx_index * 2);
5529
5530 default:
5531 break;
5532 }
5533 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5534 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5535
5536 switch (hw->mac.type) {
5537 case ixgbe_mac_82598EB:
5538 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5539 16 + (sw_rx_index % rx_ring_per_group);
5540 return (hw_rx_index);
5541
5542 case ixgbe_mac_82599EB:
5543 case ixgbe_mac_X540:
5544 case ixgbe_mac_X550:
5545 case ixgbe_mac_X550EM_x:
5546 case ixgbe_mac_X550EM_a:
5547 if (ixgbe->num_rx_groups > 32) {
5548 hw_rx_index = (sw_rx_index /
5549 rx_ring_per_group) * 2 +
5550 (sw_rx_index % rx_ring_per_group);
5551 } else {
5552 hw_rx_index = (sw_rx_index /
5553 rx_ring_per_group) * 4 +
5554 (sw_rx_index % rx_ring_per_group);
5555 }
5556 return (hw_rx_index);
5557
5558 default:
5559 break;
5560 }
5561 }
5562
5563 /*
5564 * Should never reach. Just to make compiler happy.
5565 */
5566 return (sw_rx_index);
5567 }
5568
5569 /*
5570 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5571 *
5572 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5573 * to vector[0 - (intr_cnt -1)].
5574 */
5575 static int
5576 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5577 {
5578 int i, vector = 0;
5579
5580 /* initialize vector map */
5581 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5582 for (i = 0; i < ixgbe->intr_cnt; i++) {
5583 ixgbe->vect_map[i].ixgbe = ixgbe;
5584 }
5585
5586 /*
5587 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5588 * tx rings[0] on RTxQ[1].
5589 */
5590 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5591 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5592 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5593 return (IXGBE_SUCCESS);
5594 }
5595
5596 /*
5597 * Interrupts/vectors mapping for MSI-X
5598 */
5599
5600 /*
5601 * Map other interrupt to vector 0,
5602 * Set bit in map and count the bits set.
5603 */
5604 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5605 ixgbe->vect_map[vector].other_cnt++;
5606
5607 /*
5608 * Map rx ring interrupts to vectors
5609 */
5610 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5611 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5612 vector = (vector +1) % ixgbe->intr_cnt;
5613 }
5614
5615 /*
5616 * Map tx ring interrupts to vectors
5617 */
5618 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5619 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5620 vector = (vector +1) % ixgbe->intr_cnt;
5621 }
5622
5623 return (IXGBE_SUCCESS);
5624 }
5625
5626 /*
5627 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5628 *
5629 * This relies on ring/vector mapping already set up in the
5630 * vect_map[] structures
5631 */
5632 static void
5633 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5634 {
5635 struct ixgbe_hw *hw = &ixgbe->hw;
5636 ixgbe_intr_vector_t *vect; /* vector bitmap */
5637 int r_idx; /* ring index */
5638 int v_idx; /* vector index */
5639 uint32_t hw_index;
5640
5641 /*
5642 * Clear any previous entries
5643 */
5644 switch (hw->mac.type) {
5645 case ixgbe_mac_82598EB:
5646 for (v_idx = 0; v_idx < 25; v_idx++)
5647 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5648 break;
5649
5650 case ixgbe_mac_82599EB:
5651 case ixgbe_mac_X540:
5652 case ixgbe_mac_X550:
5653 case ixgbe_mac_X550EM_x:
5654 case ixgbe_mac_X550EM_a:
5655 for (v_idx = 0; v_idx < 64; v_idx++)
5656 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5657 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5658 break;
5659
5660 default:
5661 break;
5662 }
5663
5664 /*
5665 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5666 * tx rings[0] will use RTxQ[1].
5667 */
5668 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5669 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5670 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5671 return;
5672 }
5673
5674 /*
5675 * For MSI-X interrupt, "Other" is always on vector[0].
5676 */
5677 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5678
5679 /*
5680 * For each interrupt vector, populate the IVAR table
5681 */
5682 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5683 vect = &ixgbe->vect_map[v_idx];
5684
5685 /*
5686 * For each rx ring bit set
5687 */
5688 r_idx = bt_getlowbit(vect->rx_map, 0,
5689 (ixgbe->num_rx_rings - 1));
5690
5691 while (r_idx >= 0) {
5692 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5693 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5694 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5695 (ixgbe->num_rx_rings - 1));
5696 }
5697
5698 /*
5699 * For each tx ring bit set
5700 */
5701 r_idx = bt_getlowbit(vect->tx_map, 0,
5702 (ixgbe->num_tx_rings - 1));
5703
5704 while (r_idx >= 0) {
5705 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5706 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5707 (ixgbe->num_tx_rings - 1));
5708 }
5709 }
5710 }
5711
5712 /*
5713 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5714 */
5715 static void
5716 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5717 {
5718 int i;
5719 int rc;
5720
5721 for (i = 0; i < ixgbe->intr_cnt; i++) {
5722 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5723 if (rc != DDI_SUCCESS) {
5724 IXGBE_DEBUGLOG_1(ixgbe,
5725 "Remove intr handler failed: %d", rc);
5726 }
5727 }
5728 }
5729
5730 /*
5731 * ixgbe_rem_intrs - Remove the allocated interrupts.
5732 */
5733 static void
5734 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5735 {
5736 int i;
5737 int rc;
5738
5739 for (i = 0; i < ixgbe->intr_cnt; i++) {
5740 rc = ddi_intr_free(ixgbe->htable[i]);
5741 if (rc != DDI_SUCCESS) {
5742 IXGBE_DEBUGLOG_1(ixgbe,
5743 "Free intr failed: %d", rc);
5744 }
5745 }
5746
5747 kmem_free(ixgbe->htable, ixgbe->intr_size);
5748 ixgbe->htable = NULL;
5749 }
5750
5751 /*
5752 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5753 */
5754 static int
5755 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5756 {
5757 int i;
5758 int rc;
5759
5760 /*
5761 * Enable interrupts
5762 */
5763 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5764 /*
5765 * Call ddi_intr_block_enable() for MSI
5766 */
5767 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5768 if (rc != DDI_SUCCESS) {
5769 ixgbe_log(ixgbe,
5770 "Enable block intr failed: %d", rc);
5771 return (IXGBE_FAILURE);
5772 }
5773 } else {
5774 /*
5775 * Call ddi_intr_enable() for Legacy/MSI non block enable
5776 */
5777 for (i = 0; i < ixgbe->intr_cnt; i++) {
5778 rc = ddi_intr_enable(ixgbe->htable[i]);
5779 if (rc != DDI_SUCCESS) {
5780 ixgbe_log(ixgbe,
5781 "Enable intr failed: %d", rc);
5782 return (IXGBE_FAILURE);
5783 }
5784 }
5785 }
5786
5787 return (IXGBE_SUCCESS);
5788 }
5789
5790 /*
5791 * ixgbe_disable_intrs - Disable all the interrupts.
5792 */
5793 static int
5794 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5795 {
5796 int i;
5797 int rc;
5798
5799 /*
5800 * Disable all interrupts
5801 */
5802 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5803 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5804 if (rc != DDI_SUCCESS) {
5805 ixgbe_log(ixgbe,
5806 "Disable block intr failed: %d", rc);
5807 return (IXGBE_FAILURE);
5808 }
5809 } else {
5810 for (i = 0; i < ixgbe->intr_cnt; i++) {
5811 rc = ddi_intr_disable(ixgbe->htable[i]);
5812 if (rc != DDI_SUCCESS) {
5813 ixgbe_log(ixgbe,
5814 "Disable intr failed: %d", rc);
5815 return (IXGBE_FAILURE);
5816 }
5817 }
5818 }
5819
5820 return (IXGBE_SUCCESS);
5821 }
5822
5823 /*
5824 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5825 */
5826 static void
5827 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5828 {
5829 struct ixgbe_hw *hw = &ixgbe->hw;
5830 ixgbe_link_speed speed = 0;
5831 boolean_t link_up = B_FALSE;
5832 uint32_t pcs1g_anlp = 0;
5833
5834 ASSERT(mutex_owned(&ixgbe->gen_lock));
5835 ixgbe->param_lp_1000fdx_cap = 0;
5836 ixgbe->param_lp_100fdx_cap = 0;
5837
5838 /* check for link, don't wait */
5839 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
5840
5841 /*
5842 * Update the observed Link Partner's capabilities. Not all adapters
5843 * can provide full information on the LP's capable speeds, so we
5844 * provide what we can.
5845 */
5846 if (link_up) {
5847 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5848
5849 ixgbe->param_lp_1000fdx_cap =
5850 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5851 ixgbe->param_lp_100fdx_cap =
5852 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5853 }
5854
5855 /*
5856 * Update GLD's notion of the adapter's currently advertised speeds.
5857 * Since the common code doesn't always record the current autonegotiate
5858 * settings in the phy struct for all parts (specifically, adapters with
5859 * SFPs) we first test to see if it is 0, and if so, we fall back to
5860 * using the adapter's speed capabilities which we saved during instance
5861 * init in ixgbe_init_params().
5862 *
5863 * Adapters with SFPs will always be shown as advertising all of their
5864 * supported speeds, and adapters with baseT PHYs (where the phy struct
5865 * is maintained by the common code) will always have a factual view of
5866 * their currently-advertised speeds. In the case of SFPs, this is
5867 * acceptable as we default to advertising all speeds that the adapter
5868 * claims to support, and those properties are immutable; unlike on
5869 * baseT (copper) PHYs, where speeds can be enabled or disabled at will.
5870 */
5871 speed = hw->phy.autoneg_advertised;
5872 if (speed == 0)
5873 speed = ixgbe->speeds_supported;
5874
5875 ixgbe->param_adv_10000fdx_cap =
5876 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0;
5877 ixgbe->param_adv_5000fdx_cap =
5878 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0;
5879 ixgbe->param_adv_2500fdx_cap =
5880 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0;
5881 ixgbe->param_adv_1000fdx_cap =
5882 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0;
5883 ixgbe->param_adv_100fdx_cap =
5884 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0;
5885 }
5886
5887 /*
5888 * ixgbe_get_driver_control - Notify that driver is in control of device.
5889 */
5890 static void
5891 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5892 {
5893 uint32_t ctrl_ext;
5894
5895 /*
5896 * Notify firmware that driver is in control of device
5897 */
5898 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5899 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5900 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5901 }
5902
5903 /*
5904 * ixgbe_release_driver_control - Notify that driver is no longer in control
5905 * of device.
5906 */
5907 static void
5908 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5909 {
5910 uint32_t ctrl_ext;
5911
5912 /*
5913 * Notify firmware that driver is no longer in control of device
5914 */
5915 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5916 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5917 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5918 }
5919
5920 /*
5921 * ixgbe_atomic_reserve - Atomic decrease operation.
5922 */
5923 int
5924 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5925 {
5926 uint32_t oldval;
5927 uint32_t newval;
5928
5929 /*
5930 * ATOMICALLY
5931 */
5932 do {
5933 oldval = *count_p;
5934 if (oldval < n)
5935 return (-1);
5936 newval = oldval - n;
5937 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5938
5939 return (newval);
5940 }
5941
5942 /*
5943 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5944 */
5945 static uint8_t *
5946 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5947 {
5948 uint8_t *addr = *upd_ptr;
5949 uint8_t *new_ptr;
5950
5951 _NOTE(ARGUNUSED(hw));
5952 _NOTE(ARGUNUSED(vmdq));
5953
5954 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5955 *upd_ptr = new_ptr;
5956 return (addr);
5957 }
5958
5959 /*
5960 * FMA support
5961 */
5962 int
5963 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5964 {
5965 ddi_fm_error_t de;
5966
5967 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5968 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5969 return (de.fme_status);
5970 }
5971
5972 int
5973 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5974 {
5975 ddi_fm_error_t de;
5976
5977 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5978 return (de.fme_status);
5979 }
5980
5981 /*
5982 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5983 */
5984 static int
5985 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5986 {
5987 _NOTE(ARGUNUSED(impl_data));
5988 /*
5989 * as the driver can always deal with an error in any dma or
5990 * access handle, we can just return the fme_status value.
5991 */
5992 pci_ereport_post(dip, err, NULL);
5993 return (err->fme_status);
5994 }
5995
5996 static void
5997 ixgbe_fm_init(ixgbe_t *ixgbe)
5998 {
5999 ddi_iblock_cookie_t iblk;
6000 int fma_dma_flag;
6001
6002 /*
6003 * Only register with IO Fault Services if we have some capability
6004 */
6005 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
6006 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
6007 } else {
6008 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
6009 }
6010
6011 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
6012 fma_dma_flag = 1;
6013 } else {
6014 fma_dma_flag = 0;
6015 }
6016
6017 ixgbe_set_fma_flags(fma_dma_flag);
6018
6019 if (ixgbe->fm_capabilities) {
6020
6021 /*
6022 * Register capabilities with IO Fault Services
6023 */
6024 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
6025
6026 /*
6027 * Initialize pci ereport capabilities if ereport capable
6028 */
6029 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
6030 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6031 pci_ereport_setup(ixgbe->dip);
6032
6033 /*
6034 * Register error callback if error callback capable
6035 */
6036 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6037 ddi_fm_handler_register(ixgbe->dip,
6038 ixgbe_fm_error_cb, (void*) ixgbe);
6039 }
6040 }
6041
6042 static void
6043 ixgbe_fm_fini(ixgbe_t *ixgbe)
6044 {
6045 /*
6046 * Only unregister FMA capabilities if they are registered
6047 */
6048 if (ixgbe->fm_capabilities) {
6049
6050 /*
6051 * Release any resources allocated by pci_ereport_setup()
6052 */
6053 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
6054 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6055 pci_ereport_teardown(ixgbe->dip);
6056
6057 /*
6058 * Un-register error callback if error callback capable
6059 */
6060 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6061 ddi_fm_handler_unregister(ixgbe->dip);
6062
6063 /*
6064 * Unregister from IO Fault Service
6065 */
6066 ddi_fm_fini(ixgbe->dip);
6067 }
6068 }
6069
6070 void
6071 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
6072 {
6073 uint64_t ena;
6074 char buf[FM_MAX_CLASS];
6075
6076 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
6077 ena = fm_ena_generate(0, FM_ENA_FMT1);
6078 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
6079 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
6080 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
6081 }
6082 }
6083
6084 static int
6085 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
6086 {
6087 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
6088
6089 mutex_enter(&rx_ring->rx_lock);
6090 rx_ring->ring_gen_num = mr_gen_num;
6091 mutex_exit(&rx_ring->rx_lock);
6092 return (0);
6093 }
6094
6095 /*
6096 * Get the global ring index by a ring index within a group.
6097 */
6098 static int
6099 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
6100 {
6101 ixgbe_rx_ring_t *rx_ring;
6102 int i;
6103
6104 for (i = 0; i < ixgbe->num_rx_rings; i++) {
6105 rx_ring = &ixgbe->rx_rings[i];
6106 if (rx_ring->group_index == gindex)
6107 rindex--;
6108 if (rindex < 0)
6109 return (i);
6110 }
6111
6112 return (-1);
6113 }
6114
6115 /*
6116 * Callback funtion for MAC layer to register all rings.
6117 */
6118 /* ARGSUSED */
6119 void
6120 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
6121 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
6122 {
6123 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6124 mac_intr_t *mintr = &infop->mri_intr;
6125
6126 switch (rtype) {
6127 case MAC_RING_TYPE_RX: {
6128 /*
6129 * 'index' is the ring index within the group.
6130 * Need to get the global ring index by searching in groups.
6131 */
6132 int global_ring_index = ixgbe_get_rx_ring_index(
6133 ixgbe, group_index, ring_index);
6134
6135 ASSERT(global_ring_index >= 0);
6136
6137 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
6138 rx_ring->ring_handle = rh;
6139
6140 infop->mri_driver = (mac_ring_driver_t)rx_ring;
6141 infop->mri_start = ixgbe_ring_start;
6142 infop->mri_stop = NULL;
6143 infop->mri_poll = ixgbe_ring_rx_poll;
6144 infop->mri_stat = ixgbe_rx_ring_stat;
6145
6146 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
6147 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
6148 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
6149 if (ixgbe->intr_type &
6150 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6151 mintr->mi_ddi_handle =
6152 ixgbe->htable[rx_ring->intr_vector];
6153 }
6154
6155 break;
6156 }
6157 case MAC_RING_TYPE_TX: {
6158 ASSERT(group_index == -1);
6159 ASSERT(ring_index < ixgbe->num_tx_rings);
6160
6161 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
6162 tx_ring->ring_handle = rh;
6163
6164 infop->mri_driver = (mac_ring_driver_t)tx_ring;
6165 infop->mri_start = NULL;
6166 infop->mri_stop = NULL;
6167 infop->mri_tx = ixgbe_ring_tx;
6168 infop->mri_stat = ixgbe_tx_ring_stat;
6169 if (ixgbe->intr_type &
6170 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6171 mintr->mi_ddi_handle =
6172 ixgbe->htable[tx_ring->intr_vector];
6173 }
6174 break;
6175 }
6176 default:
6177 break;
6178 }
6179 }
6180
6181 /*
6182 * Callback funtion for MAC layer to register all groups.
6183 */
6184 void
6185 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
6186 mac_group_info_t *infop, mac_group_handle_t gh)
6187 {
6188 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6189
6190 switch (rtype) {
6191 case MAC_RING_TYPE_RX: {
6192 ixgbe_rx_group_t *rx_group;
6193
6194 rx_group = &ixgbe->rx_groups[index];
6195 rx_group->group_handle = gh;
6196
6197 infop->mgi_driver = (mac_group_driver_t)rx_group;
6198 infop->mgi_start = NULL;
6199 infop->mgi_stop = NULL;
6200 infop->mgi_addmac = ixgbe_addmac;
6201 infop->mgi_remmac = ixgbe_remmac;
6202 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
6203
6204 break;
6205 }
6206 case MAC_RING_TYPE_TX:
6207 break;
6208 default:
6209 break;
6210 }
6211 }
6212
6213 /*
6214 * Enable interrupt on the specificed rx ring.
6215 */
6216 int
6217 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
6218 {
6219 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6220 ixgbe_t *ixgbe = rx_ring->ixgbe;
6221 int r_idx = rx_ring->index;
6222 int hw_r_idx = rx_ring->hw_index;
6223 int v_idx = rx_ring->intr_vector;
6224
6225 mutex_enter(&ixgbe->gen_lock);
6226 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6227 mutex_exit(&ixgbe->gen_lock);
6228 /*
6229 * Simply return 0.
6230 * Interrupts are being adjusted. ixgbe_intr_adjust()
6231 * will eventually re-enable the interrupt when it's
6232 * done with the adjustment.
6233 */
6234 return (0);
6235 }
6236
6237 /*
6238 * To enable interrupt by setting the VAL bit of given interrupt
6239 * vector allocation register (IVAR).
6240 */
6241 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
6242
6243 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
6244
6245 /*
6246 * Trigger a Rx interrupt on this ring
6247 */
6248 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
6249 IXGBE_WRITE_FLUSH(&ixgbe->hw);
6250
6251 mutex_exit(&ixgbe->gen_lock);
6252
6253 return (0);
6254 }
6255
6256 /*
6257 * Disable interrupt on the specificed rx ring.
6258 */
6259 int
6260 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
6261 {
6262 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6263 ixgbe_t *ixgbe = rx_ring->ixgbe;
6264 int r_idx = rx_ring->index;
6265 int hw_r_idx = rx_ring->hw_index;
6266 int v_idx = rx_ring->intr_vector;
6267
6268 mutex_enter(&ixgbe->gen_lock);
6269 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6270 mutex_exit(&ixgbe->gen_lock);
6271 /*
6272 * Simply return 0.
6273 * In the rare case where an interrupt is being
6274 * disabled while interrupts are being adjusted,
6275 * we don't fail the operation. No interrupts will
6276 * be generated while they are adjusted, and
6277 * ixgbe_intr_adjust() will cause the interrupts
6278 * to be re-enabled once it completes. Note that
6279 * in this case, packets may be delivered to the
6280 * stack via interrupts before xgbe_rx_ring_intr_enable()
6281 * is called again. This is acceptable since interrupt
6282 * adjustment is infrequent, and the stack will be
6283 * able to handle these packets.
6284 */
6285 return (0);
6286 }
6287
6288 /*
6289 * To disable interrupt by clearing the VAL bit of given interrupt
6290 * vector allocation register (IVAR).
6291 */
6292 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
6293
6294 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
6295
6296 mutex_exit(&ixgbe->gen_lock);
6297
6298 return (0);
6299 }
6300
6301 /*
6302 * Add a mac address.
6303 */
6304 static int
6305 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
6306 {
6307 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6308 ixgbe_t *ixgbe = rx_group->ixgbe;
6309 struct ixgbe_hw *hw = &ixgbe->hw;
6310 int slot, i;
6311
6312 mutex_enter(&ixgbe->gen_lock);
6313
6314 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6315 mutex_exit(&ixgbe->gen_lock);
6316 return (ECANCELED);
6317 }
6318
6319 if (ixgbe->unicst_avail == 0) {
6320 /* no slots available */
6321 mutex_exit(&ixgbe->gen_lock);
6322 return (ENOSPC);
6323 }
6324
6325 /*
6326 * The first ixgbe->num_rx_groups slots are reserved for each respective
6327 * group. The rest slots are shared by all groups. While adding a
6328 * MAC address, reserved slots are firstly checked then the shared
6329 * slots are searched.
6330 */
6331 slot = -1;
6332 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
6333 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
6334 if (ixgbe->unicst_addr[i].mac.set == 0) {
6335 slot = i;
6336 break;
6337 }
6338 }
6339 } else {
6340 slot = rx_group->index;
6341 }
6342
6343 if (slot == -1) {
6344 /* no slots available */
6345 mutex_exit(&ixgbe->gen_lock);
6346 return (ENOSPC);
6347 }
6348
6349 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6350 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
6351 rx_group->index, IXGBE_RAH_AV);
6352 ixgbe->unicst_addr[slot].mac.set = 1;
6353 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
6354 ixgbe->unicst_avail--;
6355
6356 mutex_exit(&ixgbe->gen_lock);
6357
6358 return (0);
6359 }
6360
6361 /*
6362 * Remove a mac address.
6363 */
6364 static int
6365 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
6366 {
6367 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6368 ixgbe_t *ixgbe = rx_group->ixgbe;
6369 struct ixgbe_hw *hw = &ixgbe->hw;
6370 int slot;
6371
6372 mutex_enter(&ixgbe->gen_lock);
6373
6374 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6375 mutex_exit(&ixgbe->gen_lock);
6376 return (ECANCELED);
6377 }
6378
6379 slot = ixgbe_unicst_find(ixgbe, mac_addr);
6380 if (slot == -1) {
6381 mutex_exit(&ixgbe->gen_lock);
6382 return (EINVAL);
6383 }
6384
6385 if (ixgbe->unicst_addr[slot].mac.set == 0) {
6386 mutex_exit(&ixgbe->gen_lock);
6387 return (EINVAL);
6388 }
6389
6390 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6391 (void) ixgbe_clear_rar(hw, slot);
6392 ixgbe->unicst_addr[slot].mac.set = 0;
6393 ixgbe->unicst_avail++;
6394
6395 mutex_exit(&ixgbe->gen_lock);
6396
6397 return (0);
6398 }