1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2018 Joyent, Inc.
24 */
25
26 #include <sys/types.h>
27 #include <sys/callb.h>
28 #include <sys/cpupart.h>
29 #include <sys/pool.h>
30 #include <sys/pool_pset.h>
31 #include <sys/sdt.h>
32 #include <sys/strsubr.h>
33 #include <sys/strsun.h>
34 #include <sys/vlan.h>
35 #include <inet/ipsec_impl.h>
36 #include <inet/ip_impl.h>
37 #include <inet/sadb.h>
38 #include <inet/ipsecesp.h>
39 #include <inet/ipsecah.h>
40
41 #include <sys/mac_impl.h>
42 #include <sys/mac_client_impl.h>
43 #include <sys/mac_client_priv.h>
44 #include <sys/mac_soft_ring.h>
45 #include <sys/mac_flow_impl.h>
46 #include <sys/mac_stat.h>
47
48 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t);
49 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *);
50 static void mac_srs_poll_unbind(mac_soft_ring_set_t *);
51 static void mac_srs_worker_unbind(mac_soft_ring_set_t *);
52 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t);
53
54 static int mac_srs_cpu_setup(cpu_setup_t, int, void *);
55 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t);
56 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t);
57 static void mac_srs_threads_unbind(mac_soft_ring_set_t *);
58 static void mac_srs_add_glist(mac_soft_ring_set_t *);
59 static void mac_srs_remove_glist(mac_soft_ring_set_t *);
60 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *);
61 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *);
62
63 static int mac_compute_soft_ring_count(flow_entry_t *, int, int);
64 static void mac_walk_srs_and_bind(int);
65 static void mac_walk_srs_and_unbind(int);
66
67 extern boolean_t mac_latency_optimize;
68
69 static kmem_cache_t *mac_srs_cache;
70 kmem_cache_t *mac_soft_ring_cache;
71
72 /*
73 * The duration in msec we wait before signalling the soft ring
74 * worker thread in case packets get queued.
75 */
76 uint32_t mac_soft_ring_worker_wait = 0;
77
78 /*
79 * A global tunable for turning polling on/off. By default, dynamic
80 * polling is always on and is always very beneficial. It should be
81 * turned off with absolute care and for the rare workload (very
82 * low latency sensitive traffic).
83 */
84 int mac_poll_enable = B_TRUE;
85
86 /*
87 * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency.
88 * Large values could end up in consuming lot of system memory and cause
89 * system hang.
90 */
91 int mac_soft_ring_max_q_cnt = 1024;
92 int mac_soft_ring_min_q_cnt = 256;
93 int mac_soft_ring_poll_thres = 16;
94
95 boolean_t mac_tx_serialize = B_FALSE;
96
97 /*
98 * mac_tx_srs_hiwat is the queue depth threshold at which callers of
99 * mac_tx() will be notified of flow control condition.
100 *
101 * TCP does not honour flow control condition sent up by mac_tx().
102 * Thus provision is made for TCP to allow more packets to be queued
103 * in SRS upto a maximum of mac_tx_srs_max_q_cnt.
104 *
105 * Note that mac_tx_srs_hiwat is always be lesser than
106 * mac_tx_srs_max_q_cnt.
107 */
108 uint32_t mac_tx_srs_max_q_cnt = 100000;
109 uint32_t mac_tx_srs_hiwat = 1000;
110
111 /*
112 * mac_rx_soft_ring_count, mac_soft_ring_10gig_count:
113 *
114 * Global tunables that determines the number of soft rings to be used for
115 * fanning out incoming traffic on a link. These count will be used only
116 * when no explicit set of CPUs was assigned to the data-links.
117 *
118 * mac_rx_soft_ring_count tunable will come into effect only if
119 * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by
120 * default only for sun4v platforms.
121 *
122 * mac_rx_soft_ring_10gig_count will come into effect if you are running on a
123 * 10Gbps link and is not dependent upon mac_soft_ring_enable.
124 *
125 * The number of soft rings for fanout for a link or a flow is determined
126 * by mac_compute_soft_ring_count() routine. This routine will take into
127 * account mac_soft_ring_enable, mac_rx_soft_ring_count and
128 * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link.
129 *
130 * If a bandwidth is specified, the determination of the number of soft
131 * rings is based on specified bandwidth, CPU speed and number of CPUs in
132 * the system.
133 */
134 uint_t mac_rx_soft_ring_count = 8;
135 uint_t mac_rx_soft_ring_10gig_count = 8;
136
137 /*
138 * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added
139 * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The
140 * list is used to walk the list of all MAC threads when a CPU is
141 * coming online or going offline.
142 */
143 static mac_soft_ring_set_t *mac_srs_g_list = NULL;
144 static krwlock_t mac_srs_g_lock;
145
146 /*
147 * Whether the SRS threads should be bound, or not.
148 */
149 boolean_t mac_srs_thread_bind = B_TRUE;
150
151 /*
152 * Whether Rx/Tx interrupts should be re-targeted. Disabled by default.
153 * dladm command would override this.
154 */
155 boolean_t mac_tx_intr_retarget = B_FALSE;
156 boolean_t mac_rx_intr_retarget = B_FALSE;
157
158 /*
159 * If cpu bindings are specified by user, then Tx SRS and its soft
160 * rings should also be bound to the CPUs specified by user. The
161 * CPUs for Tx bindings are at the end of the cpu list provided by
162 * the user. If enough CPUs are not available (for Tx and Rx
163 * SRSes), then the CPUs are shared by both Tx and Rx SRSes.
164 */
165 #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \
166 processorid_t cpuid; \
167 int i; \
168 mac_soft_ring_t *softring; \
169 mac_cpus_t *srs_cpu; \
170 \
171 srs_cpu = &mac_tx_srs->srs_cpu; \
172 cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \
173 mac_srs_worker_bind(mac_tx_srs, cpuid); \
174 if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \
175 for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \
176 cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \
177 softring = mac_tx_srs->srs_tx_soft_rings[i]; \
178 if (cpuid != -1) { \
179 (void) mac_soft_ring_bind(softring, \
180 cpuid); \
181 } \
182 } \
183 } \
184 }
185
186 /*
187 * Re-targeting is allowed only for exclusive group or for primary.
188 */
189 #define RETARGETABLE_CLIENT(group, mcip) \
190 ((((group) != NULL) && \
191 ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \
192 mac_is_primary_client(mcip))
193
194 #define MAC_RING_RETARGETABLE(ring) \
195 (((ring) != NULL) && \
196 ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \
197 !((ring)->mr_info.mri_intr.mi_ddi_shared))
198
199
200 /* INIT and FINI ROUTINES */
201
202 void
203 mac_soft_ring_init(void)
204 {
205 mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache",
206 sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0);
207
208 mac_srs_cache = kmem_cache_create("mac_srs_cache",
209 sizeof (mac_soft_ring_set_t),
210 64, NULL, NULL, NULL, NULL, NULL, 0);
211
212 rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL);
213 mutex_enter(&cpu_lock);
214 register_cpu_setup_func(mac_srs_cpu_setup, NULL);
215 mutex_exit(&cpu_lock);
216 }
217
218 void
219 mac_soft_ring_finish(void)
220 {
221 mutex_enter(&cpu_lock);
222 unregister_cpu_setup_func(mac_srs_cpu_setup, NULL);
223 mutex_exit(&cpu_lock);
224 rw_destroy(&mac_srs_g_lock);
225 kmem_cache_destroy(mac_soft_ring_cache);
226 kmem_cache_destroy(mac_srs_cache);
227 }
228
229 static void
230 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs)
231 {
232 mac_soft_ring_t *softring, *next, *head;
233
234 /*
235 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from
236 * DR. The callbacks from DR are called with cpu_lock held, and hence
237 * can't wait to grab the mac perimeter. The soft ring list is hence
238 * protected for read access by srs_lock. Changing the soft ring list
239 * needs the mac perimeter and the srs_lock.
240 */
241 mutex_enter(&mac_srs->srs_lock);
242
243 head = mac_srs->srs_soft_ring_head;
244 mac_srs->srs_soft_ring_head = NULL;
245 mac_srs->srs_soft_ring_tail = NULL;
246 mac_srs->srs_soft_ring_count = 0;
247
248 mutex_exit(&mac_srs->srs_lock);
249
250 for (softring = head; softring != NULL; softring = next) {
251 next = softring->s_ring_next;
252 mac_soft_ring_free(softring);
253 }
254 }
255
256 static void
257 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs)
258 {
259 ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL);
260 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
261
262 rw_enter(&mac_srs_g_lock, RW_WRITER);
263 mutex_enter(&mac_srs->srs_lock);
264
265 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0);
266
267 if (mac_srs_g_list == NULL) {
268 mac_srs_g_list = mac_srs;
269 } else {
270 mac_srs->srs_next = mac_srs_g_list;
271 mac_srs_g_list->srs_prev = mac_srs;
272 mac_srs->srs_prev = NULL;
273 mac_srs_g_list = mac_srs;
274 }
275 mac_srs->srs_state |= SRS_IN_GLIST;
276
277 mutex_exit(&mac_srs->srs_lock);
278 rw_exit(&mac_srs_g_lock);
279 }
280
281 static void
282 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs)
283 {
284 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
285
286 rw_enter(&mac_srs_g_lock, RW_WRITER);
287 mutex_enter(&mac_srs->srs_lock);
288
289 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0);
290
291 if (mac_srs == mac_srs_g_list) {
292 mac_srs_g_list = mac_srs->srs_next;
293 if (mac_srs_g_list != NULL)
294 mac_srs_g_list->srs_prev = NULL;
295 } else {
296 mac_srs->srs_prev->srs_next = mac_srs->srs_next;
297 if (mac_srs->srs_next != NULL)
298 mac_srs->srs_next->srs_prev = mac_srs->srs_prev;
299 }
300 mac_srs->srs_state &= ~SRS_IN_GLIST;
301
302 mutex_exit(&mac_srs->srs_lock);
303 rw_exit(&mac_srs_g_lock);
304 }
305
306 /* POLLING SETUP AND TEAR DOWN ROUTINES */
307
308 /*
309 * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart
310 *
311 * These routines are used to call back into the upper layer
312 * (primarily TCP squeue) to stop polling the soft rings or
313 * restart polling.
314 */
315 void
316 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip,
317 mac_soft_ring_set_t *mac_srs)
318 {
319 mac_soft_ring_t *softring;
320
321 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
322
323 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
324 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
325 return;
326 }
327
328 for (softring = mac_srs->srs_soft_ring_head;
329 softring != NULL; softring = softring->s_ring_next) {
330 if ((softring->s_ring_type & ST_RING_TCP) &&
331 (softring->s_ring_rx_arg2 != NULL)) {
332 mcip->mci_resource_quiesce(mcip->mci_resource_arg,
333 softring->s_ring_rx_arg2);
334 }
335 }
336 }
337
338 void
339 mac_srs_client_poll_restart(mac_client_impl_t *mcip,
340 mac_soft_ring_set_t *mac_srs)
341 {
342 mac_soft_ring_t *softring;
343
344 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
345
346 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
347 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
348 return;
349 }
350
351 for (softring = mac_srs->srs_soft_ring_head;
352 softring != NULL; softring = softring->s_ring_next) {
353 if ((softring->s_ring_type & ST_RING_TCP) &&
354 (softring->s_ring_rx_arg2 != NULL)) {
355 mcip->mci_resource_restart(mcip->mci_resource_arg,
356 softring->s_ring_rx_arg2);
357 }
358 }
359 }
360
361 /*
362 * Register the given SRS and associated soft rings with the consumer and
363 * enable the polling interface used by the consumer.(i.e IP) over this
364 * SRS and associated soft rings.
365 */
366 void
367 mac_srs_client_poll_enable(mac_client_impl_t *mcip,
368 mac_soft_ring_set_t *mac_srs)
369 {
370 mac_rx_fifo_t mrf;
371 mac_soft_ring_t *softring;
372
373 ASSERT(mac_srs->srs_mcip == mcip);
374 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
375
376 if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE))
377 return;
378
379 bzero(&mrf, sizeof (mac_rx_fifo_t));
380 mrf.mrf_type = MAC_RX_FIFO;
381
382 /*
383 * A SRS is capable of acting as a soft ring for cases
384 * where no fanout is needed. This is the case for userland
385 * flows.
386 */
387 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS)
388 return;
389
390 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll;
391 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable;
392 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable;
393 mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED;
394
395 softring = mac_srs->srs_soft_ring_head;
396 while (softring != NULL) {
397 if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) {
398 /*
399 * TCP and UDP support DLS bypass. Squeue polling
400 * support implies DLS bypass since the squeue poll
401 * path does not have DLS processing.
402 */
403 mac_soft_ring_dls_bypass(softring,
404 mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg);
405 }
406 /*
407 * Non-TCP protocols don't support squeues. Hence we don't
408 * make any ring addition callbacks for non-TCP rings
409 */
410 if (!(softring->s_ring_type & ST_RING_TCP)) {
411 softring->s_ring_rx_arg2 = NULL;
412 softring = softring->s_ring_next;
413 continue;
414 }
415 mrf.mrf_rx_arg = softring;
416 mrf.mrf_intr_handle = (mac_intr_handle_t)softring;
417 mrf.mrf_cpu_id = softring->s_ring_cpuid;
418 mrf.mrf_flow_priority = mac_srs->srs_pri;
419
420 softring->s_ring_rx_arg2 = mcip->mci_resource_add(
421 mcip->mci_resource_arg, (mac_resource_t *)&mrf);
422
423 softring = softring->s_ring_next;
424 }
425 }
426
427 /*
428 * Unregister the given SRS and associated soft rings with the consumer and
429 * disable the polling interface used by the consumer.(i.e IP) over this
430 * SRS and associated soft rings.
431 */
432 void
433 mac_srs_client_poll_disable(mac_client_impl_t *mcip,
434 mac_soft_ring_set_t *mac_srs)
435 {
436 mac_soft_ring_t *softring;
437
438 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
439
440 /*
441 * A SRS is capable of acting as a soft ring for cases
442 * where no protocol fanout is needed. This is the case
443 * for userland flows. Nothing to do here.
444 */
445 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS)
446 return;
447
448 mutex_enter(&mac_srs->srs_lock);
449 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
450 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
451 mutex_exit(&mac_srs->srs_lock);
452 return;
453 }
454 mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS);
455 mutex_exit(&mac_srs->srs_lock);
456
457 /*
458 * DLS bypass is now disabled in the case of both TCP and UDP.
459 * Reset the soft ring callbacks to the standard 'mac_rx_deliver'
460 * callback. In addition, in the case of TCP, invoke IP's callback
461 * for ring removal.
462 */
463 for (softring = mac_srs->srs_soft_ring_head;
464 softring != NULL; softring = softring->s_ring_next) {
465 if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP)))
466 continue;
467
468 if ((softring->s_ring_type & ST_RING_TCP) &&
469 softring->s_ring_rx_arg2 != NULL) {
470 mcip->mci_resource_remove(mcip->mci_resource_arg,
471 softring->s_ring_rx_arg2);
472 }
473
474 mutex_enter(&softring->s_ring_lock);
475 while (softring->s_ring_state & S_RING_PROC) {
476 softring->s_ring_state |= S_RING_CLIENT_WAIT;
477 cv_wait(&softring->s_ring_client_cv,
478 &softring->s_ring_lock);
479 }
480 softring->s_ring_state &= ~S_RING_CLIENT_WAIT;
481 softring->s_ring_rx_arg2 = NULL;
482 softring->s_ring_rx_func = mac_rx_deliver;
483 softring->s_ring_rx_arg1 = mcip;
484 mutex_exit(&softring->s_ring_lock);
485 }
486 }
487
488 /*
489 * Enable or disable poll capability of the SRS on the underlying Rx ring.
490 *
491 * There is a need to enable or disable the poll capability of an SRS over an
492 * Rx ring depending on the number of mac clients sharing the ring and also
493 * whether user flows are configured on it. However the poll state is actively
494 * manipulated by the SRS worker and poll threads and uncoordinated changes by
495 * yet another thread to the underlying capability can surprise them leading
496 * to assert failures. Instead we quiesce the SRS, make the changes and then
497 * restart the SRS.
498 */
499 static void
500 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs,
501 boolean_t turn_off_poll_capab, mac_rx_func_t rx_func)
502 {
503 boolean_t need_restart = B_FALSE;
504 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
505 mac_ring_t *ring;
506
507 if (!SRS_QUIESCED(mac_srs)) {
508 mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE);
509 need_restart = B_TRUE;
510 }
511
512 ring = mac_srs->srs_ring;
513 if ((ring != NULL) &&
514 (ring->mr_classify_type == MAC_HW_CLASSIFIER)) {
515 if (turn_off_poll_capab)
516 mac_srs->srs_state &= ~SRS_POLLING_CAPAB;
517 else if (mac_poll_enable)
518 mac_srs->srs_state |= SRS_POLLING_CAPAB;
519 }
520 srs_rx->sr_lower_proc = rx_func;
521
522 if (need_restart)
523 mac_rx_srs_restart(mac_srs);
524 }
525
526 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */
527
528 /*
529 * Return the next CPU to be used to bind a MAC kernel thread.
530 * If a cpupart is specified, the cpu chosen must be from that
531 * cpu partition.
532 */
533 static processorid_t
534 mac_next_bind_cpu(cpupart_t *cpupart)
535 {
536 static cpu_t *cp = NULL;
537 cpu_t *cp_start;
538
539 ASSERT(MUTEX_HELD(&cpu_lock));
540
541 if (cp == NULL)
542 cp = cpu_list;
543
544 cp = cp->cpu_next_onln;
545 cp_start = cp;
546
547 do {
548 if ((cpupart == NULL) || (cp->cpu_part == cpupart))
549 return (cp->cpu_id);
550
551 } while ((cp = cp->cpu_next_onln) != cp_start);
552
553 return (-1); /* No matching CPU found online */
554 }
555
556 /* ARGSUSED */
557 static int
558 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg)
559 {
560 ASSERT(MUTEX_HELD(&cpu_lock));
561 switch (what) {
562 case CPU_CONFIG:
563 case CPU_ON:
564 case CPU_CPUPART_IN:
565 mac_walk_srs_and_bind(id);
566 break;
567
568 case CPU_UNCONFIG:
569 case CPU_OFF:
570 case CPU_CPUPART_OUT:
571 mac_walk_srs_and_unbind(id);
572 break;
573
574 default:
575 break;
576 }
577 return (0);
578 }
579
580 /*
581 * mac_compute_soft_ring_count():
582 *
583 * This routine computes the number of soft rings needed to handle incoming
584 * load given a flow_entry.
585 *
586 * The routine does the following:
587 * 1) soft rings will be created if mac_soft_ring_enable is set.
588 * 2) If the underlying link is a 10Gbps link, then soft rings will be
589 * created even if mac_soft_ring_enable is not set. The number of soft
590 * rings, so created, will equal mac_rx_soft_ring_10gig_count.
591 * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the
592 * mac_rx_soft_ring_10gig_count number of soft rings will be created for a
593 * 10Gbps link.
594 *
595 * If a bandwidth limit is specified, the number that gets computed is
596 * dependent upon CPU speed, the number of Rx rings configured, and
597 * the bandwidth limit.
598 * If more Rx rings are available, less number of soft rings is needed.
599 *
600 * mac_use_bw_heuristic is another "hidden" variable that can be used to
601 * override the default use of soft ring count computation. Depending upon
602 * the usefulness of it, mac_use_bw_heuristic can later be made into a
603 * data-link property or removed altogether.
604 *
605 * TODO: Cleanup and tighten some of the assumptions.
606 */
607 boolean_t mac_use_bw_heuristic = B_TRUE;
608 static int
609 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus)
610 {
611 uint64_t cpu_speed, bw = 0;
612 int srings = 0;
613 boolean_t bw_enabled = B_FALSE;
614
615 ASSERT(!(flent->fe_type & FLOW_USER));
616 if (flent->fe_resource_props.mrp_mask & MRP_MAXBW &&
617 mac_use_bw_heuristic) {
618 /* bandwidth enabled */
619 bw_enabled = B_TRUE;
620 bw = flent->fe_resource_props.mrp_maxbw;
621 }
622 if (!bw_enabled) {
623 /* No bandwidth enabled */
624 if (mac_soft_ring_enable)
625 srings = mac_rx_soft_ring_count;
626
627 /* Is this a 10Gig link? */
628 flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip,
629 MAC_STAT_IFSPEED);
630 /* convert to Mbps */
631 if (((flent->fe_nic_speed)/1000000) > 1000 &&
632 mac_rx_soft_ring_10gig_count > 0) {
633 /* This is a 10Gig link */
634 srings = mac_rx_soft_ring_10gig_count;
635 /*
636 * Use 2 times mac_rx_soft_ring_10gig_count for
637 * sun4v systems.
638 */
639 if (mac_soft_ring_enable)
640 srings = srings * 2;
641 }
642 } else {
643 /*
644 * Soft ring computation using CPU speed and specified
645 * bandwidth limit.
646 */
647 /* Assumption: all CPUs have the same frequency */
648 cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock;
649
650 /* cpu_speed is in MHz; make bw in units of Mbps. */
651 bw = bw/1000000;
652
653 if (bw >= 1000) {
654 /*
655 * bw is greater than or equal to 1Gbps.
656 * The number of soft rings required is a function
657 * of bandwidth and CPU speed. To keep this simple,
658 * let's use this rule: 1GHz CPU can handle 1Gbps.
659 * If bw is less than 1 Gbps, then there is no need
660 * for soft rings. Assumption is that CPU speeds
661 * (on modern systems) are at least 1GHz.
662 */
663 srings = bw/cpu_speed;
664 if (srings <= 1 && mac_soft_ring_enable) {
665 /*
666 * Give at least 2 soft rings
667 * for sun4v systems
668 */
669 srings = 2;
670 }
671 }
672 }
673 /*
674 * If the flent has multiple Rx SRSs, then each SRS need not
675 * have that many soft rings on top of it. The number of
676 * soft rings for each Rx SRS is found by dividing srings by
677 * rx_srs_cnt.
678 */
679 if (rx_srs_cnt > 1) {
680 int remainder;
681
682 remainder = srings%rx_srs_cnt;
683 srings = srings/rx_srs_cnt;
684 if (remainder != 0)
685 srings++;
686 /*
687 * Fanning out to 1 soft ring is not very useful.
688 * Set it as well to 0 and mac_srs_fanout_init()
689 * will take care of creating a single soft ring
690 * for proto fanout.
691 */
692 if (srings == 1)
693 srings = 0;
694 }
695 /* Do some more massaging */
696 srings = min(srings, maxcpus);
697 srings = min(srings, MAX_SR_FANOUT);
698 return (srings);
699 }
700
701 /*
702 * mac_tx_cpu_init:
703 * set up CPUs for Tx interrupt re-targeting and Tx worker
704 * thread binding
705 */
706 static void
707 mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp,
708 cpupart_t *cpupart)
709 {
710 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs;
711 mac_srs_tx_t *srs_tx = &tx_srs->srs_tx;
712 mac_cpus_t *srs_cpu = &tx_srs->srs_cpu;
713 mac_soft_ring_t *sringp;
714 mac_ring_t *ring;
715 processorid_t worker_cpuid;
716 boolean_t retargetable_client = B_FALSE;
717 int i, j;
718
719 if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group,
720 flent->fe_mcip)) {
721 retargetable_client = B_TRUE;
722 }
723
724 if (MAC_TX_SOFT_RINGS(tx_srs)) {
725 if (mrp != NULL)
726 j = mrp->mrp_ncpus - 1;
727 for (i = 0; i < tx_srs->srs_tx_ring_count; i++) {
728 if (mrp != NULL) {
729 if (j < 0)
730 j = mrp->mrp_ncpus - 1;
731 worker_cpuid = mrp->mrp_cpu[j];
732 } else {
733 /*
734 * Bind interrupt to the next CPU available
735 * and leave the worker unbound.
736 */
737 worker_cpuid = -1;
738 }
739 sringp = tx_srs->srs_tx_soft_rings[i];
740 ring = (mac_ring_t *)sringp->s_ring_tx_arg2;
741 srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid;
742 if (MAC_RING_RETARGETABLE(ring) &&
743 retargetable_client) {
744 mutex_enter(&cpu_lock);
745 srs_cpu->mc_tx_intr_cpu[i] =
746 (mrp != NULL) ? mrp->mrp_cpu[j] :
747 (mac_tx_intr_retarget ?
748 mac_next_bind_cpu(cpupart) : -1);
749 mutex_exit(&cpu_lock);
750 } else {
751 srs_cpu->mc_tx_intr_cpu[i] = -1;
752 }
753 if (mrp != NULL)
754 j--;
755 }
756 } else {
757 /* Tx mac_ring_handle_t is stored in st_arg2 */
758 srs_cpu->mc_tx_fanout_cpus[0] =
759 (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1;
760 ring = (mac_ring_t *)srs_tx->st_arg2;
761 if (MAC_RING_RETARGETABLE(ring) && retargetable_client) {
762 mutex_enter(&cpu_lock);
763 srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ?
764 mrp->mrp_cpu[mrp->mrp_ncpus - 1] :
765 (mac_tx_intr_retarget ?
766 mac_next_bind_cpu(cpupart) : -1);
767 mutex_exit(&cpu_lock);
768 } else {
769 srs_cpu->mc_tx_intr_cpu[0] = -1;
770 }
771 }
772 }
773
774 /*
775 * Assignment of user specified CPUs to a link.
776 *
777 * Minimum CPUs required to get an optimal assignmet:
778 * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize
779 * flag is set -- one for polling, one for fanout soft ring.
780 * If mac_latency_optimize is not set, then 3 CPUs are needed -- one
781 * for polling, one for SRS worker thread and one for fanout soft ring.
782 *
783 * The CPUs needed for Tx side is equal to the number of Tx rings
784 * the link is using.
785 *
786 * mac_flow_user_cpu_init() categorizes the CPU assignment depending
787 * upon the number of CPUs in 3 different buckets.
788 *
789 * In the first bucket, the most optimal case is handled. The user has
790 * passed enough number of CPUs and every thread gets its own CPU.
791 *
792 * The second and third are the sub-optimal cases. Enough CPUs are not
793 * available.
794 *
795 * The second bucket handles the case where atleast one distinct CPU is
796 * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx
797 * SRS or soft rings).
798 *
799 * In the third case (worst case scenario), specified CPU count is less
800 * than the Rx rings configured for the link. In this case, we round
801 * robin the CPUs among the Rx SRSes and Tx SRS/soft rings.
802 */
803 static void
804 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp)
805 {
806 mac_soft_ring_set_t *rx_srs, *tx_srs;
807 int i, srs_cnt;
808 mac_cpus_t *srs_cpu;
809 int no_of_cpus, cpu_cnt;
810 int rx_srs_cnt, reqd_rx_cpu_cnt;
811 int fanout_cpu_cnt, reqd_tx_cpu_cnt;
812 int reqd_poll_worker_cnt, fanout_cnt_per_srs;
813 mac_resource_props_t *emrp = &flent->fe_effective_props;
814
815 ASSERT(mrp->mrp_fanout_mode == MCM_CPUS);
816 /*
817 * The check for nbc_ncpus to be within limits for
818 * the user specified case was done earlier and if
819 * not within limits, an error would have been
820 * returned to the user.
821 */
822 ASSERT(mrp->mrp_ncpus > 0);
823
824 no_of_cpus = mrp->mrp_ncpus;
825
826 if (mrp->mrp_rx_intr_cpu != -1) {
827 /*
828 * interrupt has been re-targetted. Poll
829 * thread needs to be bound to interrupt
830 * CPU.
831 *
832 * Find where in the list is the intr
833 * CPU and swap it with the first one.
834 * We will be using the first CPU in the
835 * list for poll.
836 */
837 for (i = 0; i < no_of_cpus; i++) {
838 if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu)
839 break;
840 }
841 mrp->mrp_cpu[i] = mrp->mrp_cpu[0];
842 mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu;
843 }
844
845 /*
846 * Requirements:
847 * The number of CPUs that each Rx ring needs is dependent
848 * upon mac_latency_optimize flag.
849 * 1) If set, atleast 2 CPUs are needed -- one for
850 * polling, one for fanout soft ring.
851 * 2) If not set, then atleast 3 CPUs are needed -- one
852 * for polling, one for srs worker thread, and one for
853 * fanout soft ring.
854 */
855 rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ?
856 (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt;
857 reqd_rx_cpu_cnt = mac_latency_optimize ?
858 (rx_srs_cnt * 2) : (rx_srs_cnt * 3);
859
860 /* How many CPUs are needed for Tx side? */
861 tx_srs = flent->fe_tx_srs;
862 reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ?
863 tx_srs->srs_tx_ring_count : 1;
864
865 /* CPUs needed for Rx SRSes poll and worker threads */
866 reqd_poll_worker_cnt = mac_latency_optimize ?
867 rx_srs_cnt : rx_srs_cnt * 2;
868
869 /* Has the user provided enough CPUs? */
870 if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) {
871 /*
872 * Best case scenario. There is enough CPUs. All
873 * Rx rings will get their own set of CPUs plus
874 * Tx soft rings will get their own.
875 */
876 /*
877 * fanout_cpu_cnt is the number of CPUs available
878 * for Rx side fanout soft rings.
879 */
880 fanout_cpu_cnt = no_of_cpus -
881 reqd_poll_worker_cnt - reqd_tx_cpu_cnt;
882
883 /*
884 * Divide fanout_cpu_cnt by rx_srs_cnt to find
885 * out how many fanout soft rings each Rx SRS
886 * can have.
887 */
888 fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt;
889
890 /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */
891 fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT);
892
893 /* Do the assignment for the default Rx ring */
894 cpu_cnt = 0;
895 rx_srs = flent->fe_rx_srs[0];
896 ASSERT(rx_srs->srs_ring == NULL);
897 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
898 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
899 srs_cpu = &rx_srs->srs_cpu;
900 srs_cpu->mc_ncpus = no_of_cpus;
901 bcopy(mrp->mrp_cpu,
902 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
903 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs;
904 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
905 /* Retarget the interrupt to the same CPU as the poll */
906 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
907 srs_cpu->mc_rx_workerid = (mac_latency_optimize ?
908 srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]);
909 for (i = 0; i < fanout_cnt_per_srs; i++)
910 srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++];
911
912 /* Do the assignment for h/w Rx SRSes */
913 if (flent->fe_rx_srs_cnt > 1) {
914 cpu_cnt = 0;
915 for (srs_cnt = 1;
916 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
917 rx_srs = flent->fe_rx_srs[srs_cnt];
918 ASSERT(rx_srs->srs_ring != NULL);
919 if (rx_srs->srs_fanout_state ==
920 SRS_FANOUT_INIT) {
921 rx_srs->srs_fanout_state =
922 SRS_FANOUT_REINIT;
923 }
924 srs_cpu = &rx_srs->srs_cpu;
925 srs_cpu->mc_ncpus = no_of_cpus;
926 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus,
927 sizeof (srs_cpu->mc_cpus));
928 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs;
929 /* The first CPU in the list is the intr CPU */
930 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
931 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
932 srs_cpu->mc_rx_workerid =
933 (mac_latency_optimize ?
934 srs_cpu->mc_rx_pollid :
935 mrp->mrp_cpu[cpu_cnt++]);
936 for (i = 0; i < fanout_cnt_per_srs; i++) {
937 srs_cpu->mc_rx_fanout_cpus[i] =
938 mrp->mrp_cpu[cpu_cnt++];
939 }
940 ASSERT(cpu_cnt <= no_of_cpus);
941 }
942 }
943 goto tx_cpu_init;
944 }
945
946 /*
947 * Sub-optimal case.
948 * We have the following information:
949 * no_of_cpus - no. of cpus that user passed.
950 * rx_srs_cnt - no. of rx rings.
951 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3
952 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side.
953 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2
954 */
955 /*
956 * If we bind the Rx fanout soft rings to the same CPUs
957 * as poll/worker, would that be enough?
958 */
959 if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) {
960 boolean_t worker_assign = B_FALSE;
961
962 /*
963 * If mac_latency_optimize is not set, are there
964 * enough CPUs to assign a CPU for worker also?
965 */
966 if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt))
967 worker_assign = B_TRUE;
968 /*
969 * Zero'th Rx SRS is the default Rx ring. It is not
970 * associated with h/w Rx ring.
971 */
972 rx_srs = flent->fe_rx_srs[0];
973 ASSERT(rx_srs->srs_ring == NULL);
974 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
975 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
976 cpu_cnt = 0;
977 srs_cpu = &rx_srs->srs_cpu;
978 srs_cpu->mc_ncpus = no_of_cpus;
979 bcopy(mrp->mrp_cpu,
980 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
981 srs_cpu->mc_rx_fanout_cnt = 1;
982 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
983 /* Retarget the interrupt to the same CPU as the poll */
984 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
985 srs_cpu->mc_rx_workerid =
986 ((!mac_latency_optimize && worker_assign) ?
987 mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid);
988
989 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt];
990
991 /* Do CPU bindings for SRSes having h/w Rx rings */
992 if (flent->fe_rx_srs_cnt > 1) {
993 cpu_cnt = 0;
994 for (srs_cnt = 1;
995 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
996 rx_srs = flent->fe_rx_srs[srs_cnt];
997 ASSERT(rx_srs->srs_ring != NULL);
998 if (rx_srs->srs_fanout_state ==
999 SRS_FANOUT_INIT) {
1000 rx_srs->srs_fanout_state =
1001 SRS_FANOUT_REINIT;
1002 }
1003 srs_cpu = &rx_srs->srs_cpu;
1004 srs_cpu->mc_ncpus = no_of_cpus;
1005 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus,
1006 sizeof (srs_cpu->mc_cpus));
1007 srs_cpu->mc_rx_pollid =
1008 mrp->mrp_cpu[cpu_cnt];
1009 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
1010 srs_cpu->mc_rx_workerid =
1011 ((!mac_latency_optimize && worker_assign) ?
1012 mrp->mrp_cpu[++cpu_cnt] :
1013 srs_cpu->mc_rx_pollid);
1014 srs_cpu->mc_rx_fanout_cnt = 1;
1015 srs_cpu->mc_rx_fanout_cpus[0] =
1016 mrp->mrp_cpu[cpu_cnt];
1017 cpu_cnt++;
1018 ASSERT(cpu_cnt <= no_of_cpus);
1019 }
1020 }
1021 goto tx_cpu_init;
1022 }
1023
1024 /*
1025 * Real sub-optimal case. Not enough CPUs for poll and
1026 * Tx soft rings. Do a round robin assignment where
1027 * each Rx SRS will get the same CPU for poll, worker
1028 * and fanout soft ring.
1029 */
1030 cpu_cnt = 0;
1031 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1032 rx_srs = flent->fe_rx_srs[srs_cnt];
1033 srs_cpu = &rx_srs->srs_cpu;
1034 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
1035 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
1036 srs_cpu->mc_ncpus = no_of_cpus;
1037 bcopy(mrp->mrp_cpu,
1038 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
1039 srs_cpu->mc_rx_fanout_cnt = 1;
1040 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt];
1041 /* Retarget the interrupt to the same CPU as the poll */
1042 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
1043 srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt];
1044 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt];
1045 if (++cpu_cnt >= no_of_cpus)
1046 cpu_cnt = 0;
1047 }
1048
1049 tx_cpu_init:
1050 mac_tx_cpu_init(flent, mrp, NULL);
1051
1052 /*
1053 * Copy the user specified CPUs to the effective CPUs
1054 */
1055 for (i = 0; i < mrp->mrp_ncpus; i++) {
1056 emrp->mrp_cpu[i] = mrp->mrp_cpu[i];
1057 }
1058 emrp->mrp_ncpus = mrp->mrp_ncpus;
1059 emrp->mrp_mask = mrp->mrp_mask;
1060 bzero(emrp->mrp_pool, MAXPATHLEN);
1061 }
1062
1063 /*
1064 * mac_flow_cpu_init():
1065 *
1066 * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in
1067 * the CPU binding information in srs_cpu for all Rx SRSes associated
1068 * with a flent.
1069 */
1070 static void
1071 mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart)
1072 {
1073 mac_soft_ring_set_t *rx_srs;
1074 processorid_t cpuid;
1075 int i, j, k, srs_cnt, nscpus, maxcpus, soft_ring_cnt = 0;
1076 mac_cpus_t *srs_cpu;
1077 mac_resource_props_t *emrp = &flent->fe_effective_props;
1078 uint32_t cpus[MRP_NCPUS];
1079
1080 /*
1081 * The maximum number of CPUs available can either be
1082 * the number of CPUs in the pool or the number of CPUs
1083 * in the system.
1084 */
1085 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus;
1086
1087 /*
1088 * Compute the number of soft rings needed on top for each Rx
1089 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS
1090 * associated with h/w Rx rings. Soft ring count needed for
1091 * each h/w Rx SRS is computed and the same is applied to
1092 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[]
1093 * is the software classified Rx SRS.
1094 */
1095 soft_ring_cnt = mac_compute_soft_ring_count(flent,
1096 flent->fe_rx_srs_cnt - 1, maxcpus);
1097 if (soft_ring_cnt == 0) {
1098 /*
1099 * Even when soft_ring_cnt is 0, we still need
1100 * to create a soft ring for TCP, UDP and
1101 * OTHER. So set it to 1.
1102 */
1103 soft_ring_cnt = 1;
1104 }
1105 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1106 rx_srs = flent->fe_rx_srs[srs_cnt];
1107 srs_cpu = &rx_srs->srs_cpu;
1108 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
1109 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
1110 srs_cpu->mc_ncpus = soft_ring_cnt;
1111 srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt;
1112 mutex_enter(&cpu_lock);
1113 for (j = 0; j < soft_ring_cnt; j++) {
1114 cpuid = mac_next_bind_cpu(cpupart);
1115 srs_cpu->mc_cpus[j] = cpuid;
1116 srs_cpu->mc_rx_fanout_cpus[j] = cpuid;
1117 }
1118 cpuid = mac_next_bind_cpu(cpupart);
1119 srs_cpu->mc_rx_pollid = cpuid;
1120 srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ?
1121 srs_cpu->mc_rx_pollid : -1);
1122 /* increment ncpus to account for polling cpu */
1123 srs_cpu->mc_ncpus++;
1124 srs_cpu->mc_cpus[j++] = cpuid;
1125 if (!mac_latency_optimize) {
1126 cpuid = mac_next_bind_cpu(cpupart);
1127 srs_cpu->mc_ncpus++;
1128 srs_cpu->mc_cpus[j++] = cpuid;
1129 }
1130 srs_cpu->mc_rx_workerid = cpuid;
1131 mutex_exit(&cpu_lock);
1132 }
1133
1134 nscpus = 0;
1135 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1136 rx_srs = flent->fe_rx_srs[srs_cnt];
1137 srs_cpu = &rx_srs->srs_cpu;
1138 for (j = 0; j < srs_cpu->mc_ncpus; j++) {
1139 cpus[nscpus++] = srs_cpu->mc_cpus[j];
1140 }
1141 }
1142
1143
1144 /*
1145 * Copy cpu list to fe_effective_props
1146 * without duplicates.
1147 */
1148 k = 0;
1149 for (i = 0; i < nscpus; i++) {
1150 for (j = 0; j < k; j++) {
1151 if (emrp->mrp_cpu[j] == cpus[i])
1152 break;
1153 }
1154 if (j == k)
1155 emrp->mrp_cpu[k++] = cpus[i];
1156 }
1157 emrp->mrp_ncpus = k;
1158
1159 mac_tx_cpu_init(flent, NULL, cpupart);
1160 }
1161
1162 /*
1163 * DATAPATH SETUP ROUTINES
1164 * (setup SRS and set/update FANOUT, B/W and PRIORITY)
1165 */
1166
1167 /*
1168 * mac_srs_fanout_list_alloc:
1169 *
1170 * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of
1171 * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of
1172 * array space is needed to store Tx soft rings. Thus we allocate so
1173 * much array space for srs_tx_soft_rings.
1174 *
1175 * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth
1176 * of space to st_soft_rings. This array is used for quick access to
1177 * soft ring associated with a pseudo Tx ring based on the pseudo
1178 * ring's index (mr_index).
1179 */
1180 static void
1181 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs)
1182 {
1183 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1184
1185 if (mac_srs->srs_type & SRST_TX) {
1186 mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **)
1187 kmem_zalloc(sizeof (mac_soft_ring_t *) *
1188 MAX_RINGS_PER_GROUP, KM_SLEEP);
1189 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) {
1190 mac_srs_tx_t *tx = &mac_srs->srs_tx;
1191
1192 tx->st_soft_rings = (mac_soft_ring_t **)
1193 kmem_zalloc(sizeof (mac_soft_ring_t *) *
1194 MAX_RINGS_PER_GROUP, KM_SLEEP);
1195 }
1196 } else {
1197 mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **)
1198 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1199 KM_SLEEP);
1200 mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **)
1201 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1202 KM_SLEEP);
1203 mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **)
1204 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1205 KM_SLEEP);
1206 }
1207 }
1208
1209 static void
1210 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1211 {
1212 cpu_t *cp;
1213 boolean_t clear = B_FALSE;
1214
1215 ASSERT(MUTEX_HELD(&cpu_lock));
1216
1217 if (!mac_srs_thread_bind)
1218 return;
1219
1220 cp = cpu_get(cpuid);
1221 if (cp == NULL || !cpu_is_online(cp))
1222 return;
1223
1224 mutex_enter(&mac_srs->srs_lock);
1225 mac_srs->srs_state |= SRS_WORKER_BOUND;
1226 if (mac_srs->srs_worker_cpuid != -1)
1227 clear = B_TRUE;
1228 mac_srs->srs_worker_cpuid = cpuid;
1229 mutex_exit(&mac_srs->srs_lock);
1230
1231 if (clear)
1232 thread_affinity_clear(mac_srs->srs_worker);
1233
1234 thread_affinity_set(mac_srs->srs_worker, cpuid);
1235 DTRACE_PROBE1(worker__CPU, processorid_t, cpuid);
1236 }
1237
1238 static void
1239 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1240 {
1241 cpu_t *cp;
1242 boolean_t clear = B_FALSE;
1243
1244 ASSERT(MUTEX_HELD(&cpu_lock));
1245
1246 if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL)
1247 return;
1248
1249 cp = cpu_get(cpuid);
1250 if (cp == NULL || !cpu_is_online(cp))
1251 return;
1252
1253 mutex_enter(&mac_srs->srs_lock);
1254 mac_srs->srs_state |= SRS_POLL_BOUND;
1255 if (mac_srs->srs_poll_cpuid != -1)
1256 clear = B_TRUE;
1257 mac_srs->srs_poll_cpuid = cpuid;
1258 mutex_exit(&mac_srs->srs_lock);
1259
1260 if (clear)
1261 thread_affinity_clear(mac_srs->srs_poll_thr);
1262
1263 thread_affinity_set(mac_srs->srs_poll_thr, cpuid);
1264 DTRACE_PROBE1(poll__CPU, processorid_t, cpuid);
1265 }
1266
1267 /*
1268 * Re-target interrupt to the passed CPU. If re-target is successful,
1269 * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1.
1270 */
1271 void
1272 mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1273 {
1274 cpu_t *cp;
1275 mac_ring_t *ring = mac_srs->srs_ring;
1276 mac_intr_t *mintr = &ring->mr_info.mri_intr;
1277 flow_entry_t *flent = mac_srs->srs_flent;
1278 boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip);
1279
1280 ASSERT(MUTEX_HELD(&cpu_lock));
1281
1282 /*
1283 * Don't re-target the interrupt for these cases:
1284 * 1) ring is NULL
1285 * 2) the interrupt is shared (mi_ddi_shared)
1286 * 3) ddi_handle is NULL and !primary
1287 * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2
1288 * Case 3 & 4 are because of mac_client_intr_cpu() routine.
1289 * This routine will re-target fixed interrupt for primary
1290 * mac client if the client has only one ring. In that
1291 * case, mc_rx_intr_cpu will already have the correct value.
1292 */
1293 if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 ||
1294 (mintr->mi_ddi_handle == NULL && !primary) || (primary &&
1295 mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) {
1296 mac_srs->srs_cpu.mc_rx_intr_cpu = -1;
1297 return;
1298 }
1299
1300 if (mintr->mi_ddi_handle == NULL)
1301 return;
1302
1303 cp = cpu_get(cpuid);
1304 if (cp == NULL || !cpu_is_online(cp))
1305 return;
1306
1307 /* Drop the cpu_lock as set_intr_affinity() holds it */
1308 mutex_exit(&cpu_lock);
1309 if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS)
1310 mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid;
1311 else
1312 mac_srs->srs_cpu.mc_rx_intr_cpu = -1;
1313 mutex_enter(&cpu_lock);
1314 }
1315
1316 /*
1317 * Re-target Tx interrupts
1318 */
1319 void
1320 mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs)
1321 {
1322 cpu_t *cp;
1323 mac_ring_t *ring;
1324 mac_intr_t *mintr;
1325 mac_soft_ring_t *sringp;
1326 mac_srs_tx_t *srs_tx;
1327 mac_cpus_t *srs_cpu;
1328 processorid_t cpuid;
1329 int i;
1330
1331 ASSERT(MUTEX_HELD(&cpu_lock));
1332
1333 srs_cpu = &mac_srs->srs_cpu;
1334 if (MAC_TX_SOFT_RINGS(mac_srs)) {
1335 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
1336 sringp = mac_srs->srs_tx_soft_rings[i];
1337 ring = (mac_ring_t *)sringp->s_ring_tx_arg2;
1338 cpuid = srs_cpu->mc_tx_intr_cpu[i];
1339 cp = cpu_get(cpuid);
1340 if (cp == NULL || !cpu_is_online(cp) ||
1341 !MAC_RING_RETARGETABLE(ring)) {
1342 srs_cpu->mc_tx_retargeted_cpu[i] = -1;
1343 continue;
1344 }
1345 mintr = &ring->mr_info.mri_intr;
1346 /*
1347 * Drop the cpu_lock as set_intr_affinity()
1348 * holds it
1349 */
1350 mutex_exit(&cpu_lock);
1351 if (set_intr_affinity(mintr->mi_ddi_handle,
1352 cpuid) == DDI_SUCCESS) {
1353 srs_cpu->mc_tx_retargeted_cpu[i] = cpuid;
1354 } else {
1355 srs_cpu->mc_tx_retargeted_cpu[i] = -1;
1356 }
1357 mutex_enter(&cpu_lock);
1358 }
1359 } else {
1360 cpuid = srs_cpu->mc_tx_intr_cpu[0];
1361 cp = cpu_get(cpuid);
1362 if (cp == NULL || !cpu_is_online(cp)) {
1363 srs_cpu->mc_tx_retargeted_cpu[0] = -1;
1364 return;
1365 }
1366 srs_tx = &mac_srs->srs_tx;
1367 ring = (mac_ring_t *)srs_tx->st_arg2;
1368 if (MAC_RING_RETARGETABLE(ring)) {
1369 mintr = &ring->mr_info.mri_intr;
1370 mutex_exit(&cpu_lock);
1371 if ((set_intr_affinity(mintr->mi_ddi_handle,
1372 cpuid) == DDI_SUCCESS)) {
1373 srs_cpu->mc_tx_retargeted_cpu[0] = cpuid;
1374 } else {
1375 srs_cpu->mc_tx_retargeted_cpu[0] = -1;
1376 }
1377 mutex_enter(&cpu_lock);
1378 }
1379 }
1380 }
1381
1382 /*
1383 * When a CPU comes back online, bind the MAC kernel threads which
1384 * were previously bound to that CPU, and had to be unbound because
1385 * the CPU was going away.
1386 *
1387 * These functions are called with cpu_lock held and hence we can't
1388 * cv_wait to grab the mac perimeter. Since these functions walk the soft
1389 * ring list of an SRS without being in the perimeter, the list itself
1390 * is protected by the SRS lock.
1391 */
1392 static void
1393 mac_walk_srs_and_bind(int cpuid)
1394 {
1395 mac_soft_ring_set_t *mac_srs;
1396 mac_soft_ring_t *soft_ring;
1397
1398 rw_enter(&mac_srs_g_lock, RW_READER);
1399
1400 if ((mac_srs = mac_srs_g_list) == NULL)
1401 goto done;
1402
1403 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) {
1404 if (mac_srs->srs_worker_cpuid == -1 &&
1405 mac_srs->srs_worker_cpuid_save == cpuid) {
1406 mac_srs->srs_worker_cpuid_save = -1;
1407 mac_srs_worker_bind(mac_srs, cpuid);
1408 }
1409
1410 if (!(mac_srs->srs_type & SRST_TX)) {
1411 if (mac_srs->srs_poll_cpuid == -1 &&
1412 mac_srs->srs_poll_cpuid_save == cpuid) {
1413 mac_srs->srs_poll_cpuid_save = -1;
1414 mac_srs_poll_bind(mac_srs, cpuid);
1415 }
1416 }
1417
1418 /* Next tackle the soft rings associated with the srs */
1419 mutex_enter(&mac_srs->srs_lock);
1420 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
1421 soft_ring = soft_ring->s_ring_next) {
1422 if (soft_ring->s_ring_cpuid == -1 &&
1423 soft_ring->s_ring_cpuid_save == cpuid) {
1424 soft_ring->s_ring_cpuid_save = -1;
1425 (void) mac_soft_ring_bind(soft_ring, cpuid);
1426 }
1427 }
1428 mutex_exit(&mac_srs->srs_lock);
1429 }
1430 done:
1431 rw_exit(&mac_srs_g_lock);
1432 }
1433
1434 /*
1435 * Change the priority of the SRS's poll and worker thread. Additionally,
1436 * update the priority of the worker threads for the SRS's soft rings.
1437 * Need to modify any associated squeue threads.
1438 */
1439 void
1440 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival)
1441 {
1442 mac_soft_ring_t *ringp;
1443
1444 mac_srs->srs_pri = prival;
1445 thread_lock(mac_srs->srs_worker);
1446 (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0);
1447 thread_unlock(mac_srs->srs_worker);
1448 if (mac_srs->srs_poll_thr != NULL) {
1449 thread_lock(mac_srs->srs_poll_thr);
1450 (void) thread_change_pri(mac_srs->srs_poll_thr,
1451 mac_srs->srs_pri, 0);
1452 thread_unlock(mac_srs->srs_poll_thr);
1453 }
1454 if ((ringp = mac_srs->srs_soft_ring_head) == NULL)
1455 return;
1456 while (ringp != mac_srs->srs_soft_ring_tail) {
1457 thread_lock(ringp->s_ring_worker);
1458 (void) thread_change_pri(ringp->s_ring_worker,
1459 mac_srs->srs_pri, 0);
1460 thread_unlock(ringp->s_ring_worker);
1461 ringp = ringp->s_ring_next;
1462 }
1463 ASSERT(ringp == mac_srs->srs_soft_ring_tail);
1464 thread_lock(ringp->s_ring_worker);
1465 (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0);
1466 thread_unlock(ringp->s_ring_worker);
1467 }
1468
1469 /*
1470 * Change the receive bandwidth limit.
1471 */
1472 static void
1473 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp)
1474 {
1475 mac_soft_ring_t *softring;
1476
1477 mutex_enter(&srs->srs_lock);
1478 mutex_enter(&srs->srs_bw->mac_bw_lock);
1479
1480 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
1481 /* Reset bandwidth limit */
1482 if (srs->srs_type & SRST_BW_CONTROL) {
1483 softring = srs->srs_soft_ring_head;
1484 while (softring != NULL) {
1485 softring->s_ring_type &= ~ST_RING_BW_CTL;
1486 softring = softring->s_ring_next;
1487 }
1488 srs->srs_type &= ~SRST_BW_CONTROL;
1489 srs->srs_drain_func = mac_rx_srs_drain;
1490 }
1491 } else {
1492 /* Set/Modify bandwidth limit */
1493 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
1494 /*
1495 * Give twice the queuing capability before
1496 * dropping packets. The unit is bytes/tick.
1497 */
1498 srs->srs_bw->mac_bw_drop_threshold =
1499 srs->srs_bw->mac_bw_limit << 1;
1500 if (!(srs->srs_type & SRST_BW_CONTROL)) {
1501 softring = srs->srs_soft_ring_head;
1502 while (softring != NULL) {
1503 softring->s_ring_type |= ST_RING_BW_CTL;
1504 softring = softring->s_ring_next;
1505 }
1506 srs->srs_type |= SRST_BW_CONTROL;
1507 srs->srs_drain_func = mac_rx_srs_drain_bw;
1508 }
1509 }
1510 done:
1511 mutex_exit(&srs->srs_bw->mac_bw_lock);
1512 mutex_exit(&srs->srs_lock);
1513 }
1514
1515 /* Change the transmit bandwidth limit */
1516 static void
1517 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp)
1518 {
1519 uint32_t tx_mode, ring_info = 0;
1520 mac_srs_tx_t *srs_tx = &srs->srs_tx;
1521 mac_client_impl_t *mcip = srs->srs_mcip;
1522
1523 /*
1524 * We need to quiesce/restart the client here because mac_tx() and
1525 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing
1526 * st_mode and related fields, which are modified by the code below.
1527 */
1528 mac_tx_client_quiesce((mac_client_handle_t)mcip);
1529
1530 mutex_enter(&srs->srs_lock);
1531 mutex_enter(&srs->srs_bw->mac_bw_lock);
1532
1533 tx_mode = srs_tx->st_mode;
1534 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
1535 /* Reset bandwidth limit */
1536 if (tx_mode == SRS_TX_BW) {
1537 if (srs_tx->st_arg2 != NULL)
1538 ring_info = mac_hwring_getinfo(srs_tx->st_arg2);
1539 if (mac_tx_serialize ||
1540 (ring_info & MAC_RING_TX_SERIALIZE)) {
1541 srs_tx->st_mode = SRS_TX_SERIALIZE;
1542 } else {
1543 srs_tx->st_mode = SRS_TX_DEFAULT;
1544 }
1545 } else if (tx_mode == SRS_TX_BW_FANOUT) {
1546 srs_tx->st_mode = SRS_TX_FANOUT;
1547 } else if (tx_mode == SRS_TX_BW_AGGR) {
1548 srs_tx->st_mode = SRS_TX_AGGR;
1549 }
1550 srs->srs_type &= ~SRST_BW_CONTROL;
1551 } else {
1552 /* Set/Modify bandwidth limit */
1553 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
1554 /*
1555 * Give twice the queuing capability before
1556 * dropping packets. The unit is bytes/tick.
1557 */
1558 srs->srs_bw->mac_bw_drop_threshold =
1559 srs->srs_bw->mac_bw_limit << 1;
1560 srs->srs_type |= SRST_BW_CONTROL;
1561 if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT &&
1562 tx_mode != SRS_TX_BW_AGGR) {
1563 if (tx_mode == SRS_TX_SERIALIZE ||
1564 tx_mode == SRS_TX_DEFAULT) {
1565 srs_tx->st_mode = SRS_TX_BW;
1566 } else if (tx_mode == SRS_TX_FANOUT) {
1567 srs_tx->st_mode = SRS_TX_BW_FANOUT;
1568 } else if (tx_mode == SRS_TX_AGGR) {
1569 srs_tx->st_mode = SRS_TX_BW_AGGR;
1570 } else {
1571 ASSERT(0);
1572 }
1573 }
1574 }
1575 done:
1576 srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode);
1577 mutex_exit(&srs->srs_bw->mac_bw_lock);
1578 mutex_exit(&srs->srs_lock);
1579
1580 mac_tx_client_restart((mac_client_handle_t)mcip);
1581 }
1582
1583 /*
1584 * The uber function that deals with any update to bandwidth limits.
1585 */
1586 void
1587 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp)
1588 {
1589 int count;
1590
1591 for (count = 0; count < flent->fe_rx_srs_cnt; count++)
1592 mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp);
1593 mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp);
1594 }
1595
1596 /*
1597 * When the first sub-flow is added to a link, we disable polling on the
1598 * link and also modify the entry point to mac_rx_srs_subflow_process().
1599 * (polling is disabled because with the subflow added, accounting
1600 * for polling needs additional logic, it is assumed that when a subflow is
1601 * added, we can take some hit as a result of disabling polling rather than
1602 * adding more complexity - if this becomes a perf. issue we need to
1603 * re-rvaluate this logic). When the last subflow is removed, we turn back
1604 * polling and also reset the entry point to mac_rx_srs_process().
1605 *
1606 * In the future if there are multiple SRS, we can simply
1607 * take one and give it to the flow rather than disabling polling and
1608 * resetting the entry point.
1609 */
1610 void
1611 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable)
1612 {
1613 flow_entry_t *flent = mcip->mci_flent;
1614 int i;
1615 mac_impl_t *mip = mcip->mci_mip;
1616 mac_rx_func_t rx_func;
1617 uint_t rx_srs_cnt;
1618 boolean_t enable_classifier;
1619
1620 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
1621
1622 enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable;
1623
1624 rx_func = enable_classifier ? mac_rx_srs_subflow_process :
1625 mac_rx_srs_process;
1626
1627 /* Tell mac_srs_poll_state_change to disable polling if necessary */
1628 if (mip->mi_state_flags & MIS_POLL_DISABLE)
1629 enable_classifier = B_TRUE;
1630
1631 /*
1632 * If receive function has already been configured correctly for
1633 * current subflow configuration, do nothing.
1634 */
1635 if (flent->fe_cb_fn == (flow_fn_t)rx_func)
1636 return;
1637
1638 rx_srs_cnt = flent->fe_rx_srs_cnt;
1639 for (i = 0; i < rx_srs_cnt; i++) {
1640 ASSERT(flent->fe_rx_srs[i] != NULL);
1641 mac_srs_poll_state_change(flent->fe_rx_srs[i],
1642 enable_classifier, rx_func);
1643 }
1644
1645 /*
1646 * Change the S/W classifier so that we can land in the
1647 * correct processing function with correct argument.
1648 * If all subflows have been removed we can revert to
1649 * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process().
1650 */
1651 mutex_enter(&flent->fe_lock);
1652 flent->fe_cb_fn = (flow_fn_t)rx_func;
1653 flent->fe_cb_arg1 = (void *)mip;
1654 flent->fe_cb_arg2 = flent->fe_rx_srs[0];
1655 mutex_exit(&flent->fe_lock);
1656 }
1657
1658 static void
1659 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs)
1660 {
1661 int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0;
1662 mac_soft_ring_t *softring;
1663
1664 softring = mac_srs->srs_soft_ring_head;
1665 if (softring == NULL) {
1666 ASSERT(mac_srs->srs_soft_ring_count == 0);
1667 mac_srs->srs_tcp_ring_count = 0;
1668 mac_srs->srs_udp_ring_count = 0;
1669 mac_srs->srs_oth_ring_count = 0;
1670 mac_srs->srs_tx_ring_count = 0;
1671 return;
1672 }
1673
1674 while (softring != NULL) {
1675 if (softring->s_ring_type & ST_RING_TCP) {
1676 mac_srs->srs_tcp_soft_rings[tcp_count++] = softring;
1677 } else if (softring->s_ring_type & ST_RING_UDP) {
1678 mac_srs->srs_udp_soft_rings[udp_count++] = softring;
1679 } else if (softring->s_ring_type & ST_RING_OTH) {
1680 mac_srs->srs_oth_soft_rings[oth_count++] = softring;
1681 } else {
1682 ASSERT(softring->s_ring_type & ST_RING_TX);
1683 mac_srs->srs_tx_soft_rings[tx_count++] = softring;
1684 }
1685 softring = softring->s_ring_next;
1686 }
1687
1688 ASSERT(mac_srs->srs_soft_ring_count ==
1689 (tcp_count + udp_count + oth_count + tx_count));
1690 mac_srs->srs_tcp_ring_count = tcp_count;
1691 mac_srs->srs_udp_ring_count = udp_count;
1692 mac_srs->srs_oth_ring_count = oth_count;
1693 mac_srs->srs_tx_ring_count = tx_count;
1694 }
1695
1696 void
1697 mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri,
1698 mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs,
1699 processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1,
1700 mac_resource_handle_t x_arg2, boolean_t set_bypass)
1701 {
1702 mac_soft_ring_t *softring;
1703 mac_rx_fifo_t mrf;
1704
1705 bzero(&mrf, sizeof (mac_rx_fifo_t));
1706 mrf.mrf_type = MAC_RX_FIFO;
1707 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll;
1708 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable;
1709 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable;
1710 mrf.mrf_flow_priority = pri;
1711
1712 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1713 (type|ST_RING_TCP), pri, mcip, mac_srs,
1714 cpuid, rx_func, x_arg1, x_arg2);
1715 softring->s_ring_rx_arg2 = NULL;
1716
1717 /*
1718 * TCP and UDP support DLS bypass. In addition TCP
1719 * squeue can also poll their corresponding soft rings.
1720 */
1721 if (set_bypass && (mcip->mci_resource_arg != NULL)) {
1722 mac_soft_ring_dls_bypass(softring,
1723 mcip->mci_direct_rx_fn,
1724 mcip->mci_direct_rx_arg);
1725
1726 mrf.mrf_rx_arg = softring;
1727 mrf.mrf_intr_handle = (mac_intr_handle_t)softring;
1728
1729 /*
1730 * Make a call in IP to get a TCP squeue assigned to
1731 * this softring to maintain full CPU locality through
1732 * the stack and allow the squeue to be able to poll
1733 * the softring so the flow control can be pushed
1734 * all the way to H/W.
1735 */
1736 softring->s_ring_rx_arg2 =
1737 mcip->mci_resource_add((void *)mcip->mci_resource_arg,
1738 (mac_resource_t *)&mrf);
1739 }
1740
1741 /*
1742 * Non-TCP protocols don't support squeues. Hence we
1743 * don't make any ring addition callbacks for non-TCP
1744 * rings. Now create the UDP softring and allow it to
1745 * bypass the DLS layer.
1746 */
1747 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1748 (type|ST_RING_UDP), pri, mcip, mac_srs,
1749 cpuid, rx_func, x_arg1, x_arg2);
1750 softring->s_ring_rx_arg2 = NULL;
1751
1752 if (set_bypass && (mcip->mci_resource_arg != NULL)) {
1753 mac_soft_ring_dls_bypass(softring,
1754 mcip->mci_direct_rx_fn,
1755 mcip->mci_direct_rx_arg);
1756 }
1757
1758 /* Create the Oth softrings which has to go through the DLS */
1759 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1760 (type|ST_RING_OTH), pri, mcip, mac_srs,
1761 cpuid, rx_func, x_arg1, x_arg2);
1762 softring->s_ring_rx_arg2 = NULL;
1763 }
1764
1765 /*
1766 * This routine associates a CPU or a set of CPU to process incoming
1767 * traffic from a mac client. If multiple CPUs are specified, then
1768 * so many soft rings are created with each soft ring worker thread
1769 * bound to a CPU in the set. Each soft ring in turn will be
1770 * associated with an squeue and the squeue will be moved to the
1771 * same CPU as that of the soft ring's.
1772 */
1773 static void
1774 mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func,
1775 void *x_arg1, mac_resource_handle_t x_arg2,
1776 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs)
1777 {
1778 mac_soft_ring_t *softring;
1779 uint32_t soft_ring_flag = 0;
1780 processorid_t cpuid = -1;
1781 int i, srings_present, new_fanout_cnt;
1782 mac_cpus_t *srs_cpu;
1783
1784 /* fanout state is REINIT. Set it back to INIT */
1785 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT);
1786 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT;
1787
1788 /* how many are present right now */
1789 srings_present = mac_rx_srs->srs_tcp_ring_count;
1790 /* new request */
1791 srs_cpu = &mac_rx_srs->srs_cpu;
1792 new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt;
1793
1794 mutex_enter(&mac_rx_srs->srs_lock);
1795 if (mac_rx_srs->srs_type & SRST_BW_CONTROL)
1796 soft_ring_flag |= ST_RING_BW_CTL;
1797 mutex_exit(&mac_rx_srs->srs_lock);
1798
1799 if (new_fanout_cnt > srings_present) {
1800 /* soft rings increased */
1801 mutex_enter(&mac_rx_srs->srs_lock);
1802 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP;
1803 mutex_exit(&mac_rx_srs->srs_lock);
1804
1805 for (i = mac_rx_srs->srs_tcp_ring_count;
1806 i < new_fanout_cnt; i++) {
1807 /*
1808 * Create the protocol softrings and set the
1809 * DLS bypass where possible.
1810 */
1811 mac_srs_create_proto_softrings(i, soft_ring_flag,
1812 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1813 rx_func, x_arg1, x_arg2, B_TRUE);
1814 }
1815 mac_srs_update_fanout_list(mac_rx_srs);
1816 } else if (new_fanout_cnt < srings_present) {
1817 /* soft rings decreased */
1818 if (new_fanout_cnt == 1) {
1819 mutex_enter(&mac_rx_srs->srs_lock);
1820 mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP;
1821 ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO);
1822 mutex_exit(&mac_rx_srs->srs_lock);
1823 }
1824 /* Get rid of extra soft rings */
1825 for (i = new_fanout_cnt;
1826 i < mac_rx_srs->srs_tcp_ring_count; i++) {
1827 softring = mac_rx_srs->srs_tcp_soft_rings[i];
1828 if (softring->s_ring_rx_arg2 != NULL) {
1829 mcip->mci_resource_remove(
1830 (void *)mcip->mci_resource_arg,
1831 softring->s_ring_rx_arg2);
1832 }
1833 mac_soft_ring_remove(mac_rx_srs,
1834 mac_rx_srs->srs_tcp_soft_rings[i]);
1835 mac_soft_ring_remove(mac_rx_srs,
1836 mac_rx_srs->srs_udp_soft_rings[i]);
1837 mac_soft_ring_remove(mac_rx_srs,
1838 mac_rx_srs->srs_oth_soft_rings[i]);
1839 }
1840 mac_srs_update_fanout_list(mac_rx_srs);
1841 }
1842
1843 ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count);
1844 mutex_enter(&cpu_lock);
1845 for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) {
1846 cpuid = srs_cpu->mc_rx_fanout_cpus[i];
1847 (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i],
1848 cpuid);
1849 (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i],
1850 cpuid);
1851 (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i],
1852 cpuid);
1853 softring = mac_rx_srs->srs_tcp_soft_rings[i];
1854 if (softring->s_ring_rx_arg2 != NULL) {
1855 mcip->mci_resource_bind((void *)mcip->mci_resource_arg,
1856 softring->s_ring_rx_arg2, cpuid);
1857 }
1858 }
1859
1860 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid);
1861 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid);
1862 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu);
1863 /*
1864 * Bind Tx srs and soft ring threads too. Let's bind tx
1865 * srs to the last cpu in mrp list.
1866 */
1867 if (mac_tx_srs != NULL) {
1868 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp);
1869 mac_tx_srs_retarget_intr(mac_tx_srs);
1870 }
1871 mutex_exit(&cpu_lock);
1872 }
1873
1874 /*
1875 * Bind SRS threads and soft rings to CPUs/create fanout list.
1876 */
1877 void
1878 mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp,
1879 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2,
1880 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs,
1881 cpupart_t *cpupart)
1882 {
1883 int i;
1884 processorid_t cpuid;
1885 uint32_t soft_ring_flag = 0;
1886 int soft_ring_cnt;
1887 mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu;
1888
1889 /*
1890 * Remove the no soft ring flag and we will adjust it
1891 * appropriately further down.
1892 */
1893 mutex_enter(&mac_rx_srs->srs_lock);
1894 mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS;
1895 mutex_exit(&mac_rx_srs->srs_lock);
1896
1897 ASSERT(mac_rx_srs->srs_soft_ring_head == NULL);
1898
1899 if (mac_rx_srs->srs_type & SRST_BW_CONTROL)
1900 soft_ring_flag |= ST_RING_BW_CTL;
1901
1902 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT);
1903 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT;
1904 /*
1905 * Ring count can be 0 if no fanout is required and no cpu
1906 * were specified. Leave the SRS worker and poll thread
1907 * unbound
1908 */
1909 ASSERT(mrp != NULL);
1910 soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt;
1911
1912 /* Step 1: bind cpu contains cpu list where threads need to bind */
1913 if (soft_ring_cnt > 0) {
1914 mutex_enter(&cpu_lock);
1915 for (i = 0; i < soft_ring_cnt; i++) {
1916 cpuid = srs_cpu->mc_rx_fanout_cpus[i];
1917 /* Create the protocol softrings */
1918 mac_srs_create_proto_softrings(i, soft_ring_flag,
1919 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1920 rx_func, x_arg1, x_arg2, B_FALSE);
1921 }
1922 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid);
1923 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid);
1924 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu);
1925 /*
1926 * Bind Tx srs and soft ring threads too.
1927 * Let's bind tx srs to the last cpu in
1928 * mrp list.
1929 */
1930 if (mac_tx_srs == NULL) {
1931 mutex_exit(&cpu_lock);
1932 goto alldone;
1933 }
1934
1935 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp);
1936 mac_tx_srs_retarget_intr(mac_tx_srs);
1937 mutex_exit(&cpu_lock);
1938 } else {
1939 mutex_enter(&cpu_lock);
1940 /*
1941 * For a subflow, mrp_workerid and mrp_pollid
1942 * is not set.
1943 */
1944 mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid);
1945 mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid);
1946 mutex_exit(&cpu_lock);
1947 goto no_softrings;
1948 }
1949
1950 alldone:
1951 if (soft_ring_cnt > 1)
1952 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP;
1953 mac_srs_update_fanout_list(mac_rx_srs);
1954 mac_srs_client_poll_enable(mcip, mac_rx_srs);
1955 return;
1956
1957 no_softrings:
1958 if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) {
1959 mutex_enter(&cpu_lock);
1960 cpuid = mac_next_bind_cpu(cpupart);
1961 /* Create the protocol softrings */
1962 mac_srs_create_proto_softrings(0, soft_ring_flag,
1963 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1964 rx_func, x_arg1, x_arg2, B_FALSE);
1965 mutex_exit(&cpu_lock);
1966 } else {
1967 /*
1968 * This is the case when there is no fanout which is
1969 * true for subflows.
1970 */
1971 mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS;
1972 }
1973 mac_srs_update_fanout_list(mac_rx_srs);
1974 mac_srs_client_poll_enable(mcip, mac_rx_srs);
1975 }
1976
1977 /*
1978 * Calls mac_srs_fanout_init() or modify() depending upon whether
1979 * the SRS is getting initialized or re-initialized.
1980 */
1981 void
1982 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
1983 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1,
1984 mac_resource_handle_t x_arg2, cpupart_t *cpupart)
1985 {
1986 mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs;
1987 int i, rx_srs_cnt;
1988
1989 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
1990
1991 /*
1992 * Aggr ports do not have SRSes. This function should never be
1993 * called on an aggr port.
1994 */
1995 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
1996 mac_rx_srs = flent->fe_rx_srs[0];
1997
1998 /*
1999 * Set up the fanout on the tx side only once, with the
2000 * first rx SRS. The CPU binding, fanout, and bandwidth
2001 * criteria are common to both RX and TX, so
2002 * initializing them along side avoids redundant code.
2003 */
2004 mac_tx_srs = flent->fe_tx_srs;
2005 rx_srs_cnt = flent->fe_rx_srs_cnt;
2006
2007 /* No fanout for subflows */
2008 if (flent->fe_type & FLOW_USER) {
2009 mac_srs_fanout_init(mcip, mrp, rx_func,
2010 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs,
2011 cpupart);
2012 return;
2013 }
2014
2015 if (mrp->mrp_mask & MRP_CPUS_USERSPEC)
2016 mac_flow_user_cpu_init(flent, mrp);
2017 else
2018 mac_flow_cpu_init(flent, cpupart);
2019
2020 mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt;
2021
2022 /*
2023 * Set up fanout for both SW (0th SRS) and HW classified
2024 * SRS (the rest of Rx SRSs in flent).
2025 */
2026 for (i = 0; i < rx_srs_cnt; i++) {
2027 mac_rx_srs = flent->fe_rx_srs[i];
2028 if (i != 0)
2029 mac_tx_srs = NULL;
2030 switch (mac_rx_srs->srs_fanout_state) {
2031 case SRS_FANOUT_UNINIT:
2032 mac_srs_fanout_init(mcip, mrp, rx_func,
2033 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs,
2034 cpupart);
2035 break;
2036 case SRS_FANOUT_INIT:
2037 break;
2038 case SRS_FANOUT_REINIT:
2039 mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE);
2040 mac_srs_fanout_modify(mcip, rx_func, x_arg1,
2041 x_arg2, mac_rx_srs, mac_tx_srs);
2042 mac_rx_srs_restart(mac_rx_srs);
2043 break;
2044 default:
2045 VERIFY(mac_rx_srs->srs_fanout_state <=
2046 SRS_FANOUT_REINIT);
2047 break;
2048 }
2049 }
2050 }
2051
2052 /*
2053 * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is
2054 * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side
2055 * processing is created.
2056 *
2057 * Details on Rx SRS:
2058 * Create a SRS and also add the necessary soft rings for TCP and
2059 * non-TCP based on fanout type and count specified.
2060 *
2061 * mac_soft_ring_fanout, mac_srs_fanout_modify (?),
2062 * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need
2063 * to be heavily modified.
2064 *
2065 * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear.
2066 */
2067 mac_soft_ring_set_t *
2068 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type,
2069 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2,
2070 mac_ring_t *ring)
2071 {
2072 mac_soft_ring_set_t *mac_srs;
2073 mac_srs_rx_t *srs_rx;
2074 mac_srs_tx_t *srs_tx;
2075 mac_bw_ctl_t *mac_bw;
2076 mac_resource_props_t *mrp;
2077 boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0);
2078
2079 mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP);
2080 bzero(mac_srs, sizeof (mac_soft_ring_set_t));
2081 srs_rx = &mac_srs->srs_rx;
2082 srs_tx = &mac_srs->srs_tx;
2083
2084 mutex_enter(&flent->fe_lock);
2085
2086 /*
2087 * Get the bandwidth control structure from the flent. Get
2088 * rid of any residual values in the control structure for
2089 * the tx bw struct and also for the rx, if the rx srs is
2090 * the 1st one being brought up (the rx bw ctl struct may
2091 * be shared by multiple SRSs)
2092 */
2093 if (is_tx_srs) {
2094 mac_srs->srs_bw = &flent->fe_tx_bw;
2095 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t));
2096 flent->fe_tx_srs = mac_srs;
2097 } else {
2098 /*
2099 * The bw counter (stored in the flent) is shared
2100 * by SRS's within an rx group.
2101 */
2102 mac_srs->srs_bw = &flent->fe_rx_bw;
2103 /* First rx SRS, clear the bw structure */
2104 if (flent->fe_rx_srs_cnt == 0)
2105 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t));
2106
2107 /*
2108 * It is better to panic here rather than just assert because
2109 * on a non-debug kernel we might end up courrupting memory
2110 * and making it difficult to debug.
2111 */
2112 if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) {
2113 panic("Array Overrun detected due to MAC client %p "
2114 " having more rings than %d", (void *)mcip,
2115 MAX_RINGS_PER_GROUP);
2116 }
2117 flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs;
2118 flent->fe_rx_srs_cnt++;
2119 }
2120 mac_srs->srs_flent = flent;
2121 mutex_exit(&flent->fe_lock);
2122
2123 mac_srs->srs_state = 0;
2124 mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS);
2125 mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1;
2126 mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1;
2127 mac_srs->srs_mcip = mcip;
2128 mac_srs_fanout_list_alloc(mac_srs);
2129
2130 /*
2131 * For a flow we use the underlying MAC client's priority range with
2132 * the priority value to find an absolute priority value. For a MAC
2133 * client we use the MAC client's maximum priority as the value.
2134 */
2135 mrp = &flent->fe_effective_props;
2136 if ((mac_srs->srs_type & SRST_FLOW) != 0) {
2137 mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri,
2138 mcip->mci_max_pri, mrp->mrp_priority);
2139 } else {
2140 mac_srs->srs_pri = mcip->mci_max_pri;
2141 }
2142 /*
2143 * We need to insert the SRS in the global list before
2144 * binding the SRS and SR threads. Otherwise there is a
2145 * is a small window where the cpu reconfig callbacks
2146 * may miss the SRS in the list walk and DR could fail
2147 * as there are bound threads.
2148 */
2149 mac_srs_add_glist(mac_srs);
2150
2151 /* Initialize bw limit */
2152 if ((mrp->mrp_mask & MRP_MAXBW) != 0) {
2153 mac_srs->srs_drain_func = mac_rx_srs_drain_bw;
2154
2155 mac_bw = mac_srs->srs_bw;
2156 mutex_enter(&mac_bw->mac_bw_lock);
2157 mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
2158
2159 /*
2160 * Give twice the queuing capability before
2161 * dropping packets. The unit is bytes/tick.
2162 */
2163 mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1;
2164 mutex_exit(&mac_bw->mac_bw_lock);
2165 mac_srs->srs_type |= SRST_BW_CONTROL;
2166 } else {
2167 mac_srs->srs_drain_func = mac_rx_srs_drain;
2168 }
2169
2170 /*
2171 * We use the following policy to control Receive
2172 * Side Dynamic Polling:
2173 * 1) We switch to poll mode anytime the processing thread causes
2174 * a backlog to build up in SRS and its associated Soft Rings
2175 * (sr_poll_pkt_cnt > 0).
2176 * 2) As long as the backlog stays under the low water mark
2177 * (sr_lowat), we poll the H/W for more packets.
2178 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we
2179 * stay in poll mode but don't poll the H/W for more packets.
2180 * 4) Anytime in polling mode, if we poll the H/W for packets and
2181 * find nothing plus we have an existing backlog
2182 * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll
2183 * the H/W for packets anymore (let the polling thread go to sleep).
2184 * 5) Once the backlog is relieved (packets are processed) we reenable
2185 * polling (by signalling the poll thread) only when the backlog
2186 * dips below sr_poll_thres.
2187 * 6) sr_hiwat is used exclusively when we are not polling capable
2188 * and is used to decide when to drop packets so the SRS queue
2189 * length doesn't grow infinitely.
2190 */
2191 if (!is_tx_srs) {
2192 srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt;
2193 /* Low water mark needs to be less than high water mark */
2194 srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <=
2195 mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt :
2196 (mac_soft_ring_max_q_cnt >> 2);
2197 /* Poll threshold need to be half of low water mark or less */
2198 srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <=
2199 (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres :
2200 (srs_rx->sr_lowat >> 1);
2201 if (mac_latency_optimize)
2202 mac_srs->srs_state |= SRS_LATENCY_OPT;
2203 else
2204 mac_srs->srs_state |= SRS_SOFTRING_QUEUE;
2205 }
2206
2207 mac_srs->srs_worker = thread_create(NULL, 0,
2208 mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri);
2209
2210 if (is_tx_srs) {
2211 /* Handle everything about Tx SRS and return */
2212 mac_srs->srs_drain_func = mac_tx_srs_drain;
2213 srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt;
2214 srs_tx->st_hiwat =
2215 (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ?
2216 mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat;
2217 srs_tx->st_arg1 = x_arg1;
2218 srs_tx->st_arg2 = x_arg2;
2219 goto done;
2220 }
2221
2222 if ((srs_type & SRST_FLOW) != 0 ||
2223 FLOW_TAB_EMPTY(mcip->mci_subflow_tab))
2224 srs_rx->sr_lower_proc = mac_rx_srs_process;
2225 else
2226 srs_rx->sr_lower_proc = mac_rx_srs_subflow_process;
2227
2228 srs_rx->sr_func = rx_func;
2229 srs_rx->sr_arg1 = x_arg1;
2230 srs_rx->sr_arg2 = x_arg2;
2231
2232 if (ring != NULL) {
2233 uint_t ring_info;
2234
2235 /* Is the mac_srs created over the RX default group? */
2236 if (ring->mr_gh == (mac_group_handle_t)
2237 MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) {
2238 mac_srs->srs_type |= SRST_DEFAULT_GRP;
2239 }
2240 mac_srs->srs_ring = ring;
2241 ring->mr_srs = mac_srs;
2242 ring->mr_classify_type = MAC_HW_CLASSIFIER;
2243 ring->mr_flag |= MR_INCIPIENT;
2244
2245 if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) &&
2246 FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable)
2247 mac_srs->srs_state |= SRS_POLLING_CAPAB;
2248
2249 mac_srs->srs_poll_thr = thread_create(NULL, 0,
2250 mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN,
2251 mac_srs->srs_pri);
2252 /*
2253 * Some drivers require serialization and don't send
2254 * packet chains in interrupt context. For such
2255 * drivers, we should always queue in the soft ring
2256 * so that we get a chance to switch into polling
2257 * mode under backlog.
2258 */
2259 ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring);
2260 if (ring_info & MAC_RING_RX_ENQUEUE)
2261 mac_srs->srs_state |= SRS_SOFTRING_QUEUE;
2262 }
2263 done:
2264 mac_srs_stat_create(mac_srs);
2265 return (mac_srs);
2266 }
2267
2268 /*
2269 * Figure out the number of soft rings required. Its dependant on
2270 * if protocol fanout is required (for LINKs), global settings
2271 * require us to do fanout for performance (based on mac_soft_ring_enable),
2272 * or user has specifically requested fanout.
2273 */
2274 static uint32_t
2275 mac_find_fanout(flow_entry_t *flent, uint32_t link_type)
2276 {
2277 uint32_t fanout_type;
2278 mac_resource_props_t *mrp = &flent->fe_effective_props;
2279
2280 /* no fanout for subflows */
2281 switch (link_type) {
2282 case SRST_FLOW:
2283 fanout_type = SRST_NO_SOFT_RINGS;
2284 break;
2285 case SRST_LINK:
2286 fanout_type = SRST_FANOUT_PROTO;
2287 break;
2288 }
2289
2290 /* A primary NIC/link is being plumbed */
2291 if (flent->fe_type & FLOW_PRIMARY_MAC) {
2292 if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) {
2293 fanout_type |= SRST_FANOUT_SRC_IP;
2294 }
2295 } else if (flent->fe_type & FLOW_VNIC) {
2296 /* A VNIC is being created */
2297 if (mrp != NULL && mrp->mrp_ncpus > 0) {
2298 fanout_type |= SRST_FANOUT_SRC_IP;
2299 }
2300 }
2301
2302 return (fanout_type);
2303 }
2304
2305 /*
2306 * Change a group from h/w to s/w classification.
2307 */
2308 void
2309 mac_rx_switch_grp_to_sw(mac_group_t *group)
2310 {
2311 mac_ring_t *ring;
2312 mac_soft_ring_set_t *mac_srs;
2313
2314 for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) {
2315 if (ring->mr_classify_type == MAC_HW_CLASSIFIER) {
2316 /*
2317 * Remove the SRS associated with the HW ring.
2318 * As a result, polling will be disabled.
2319 */
2320 mac_srs = ring->mr_srs;
2321 ASSERT(mac_srs != NULL);
2322 mac_rx_srs_remove(mac_srs);
2323 ring->mr_srs = NULL;
2324 }
2325
2326 if (ring->mr_state != MR_INUSE)
2327 (void) mac_start_ring(ring);
2328
2329 /*
2330 * We need to perform SW classification
2331 * for packets landing in these rings
2332 */
2333 ring->mr_flag = 0;
2334 ring->mr_classify_type = MAC_SW_CLASSIFIER;
2335 }
2336 }
2337
2338 /*
2339 * Create the Rx SRS for S/W classifier and for each ring in the
2340 * group (if exclusive group). Also create the Tx SRS.
2341 */
2342 void
2343 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2344 uint32_t link_type)
2345 {
2346 cpupart_t *cpupart;
2347 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip);
2348 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip);
2349 boolean_t use_default = B_FALSE;
2350
2351 mac_rx_srs_group_setup(mcip, flent, link_type);
2352 mac_tx_srs_group_setup(mcip, flent, link_type);
2353
2354 /* Aggr ports don't have SRSes; thus there is no soft ring fanout. */
2355 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0)
2356 return;
2357
2358 pool_lock();
2359 cpupart = mac_pset_find(mrp, &use_default);
2360 mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip),
2361 mac_rx_deliver, mcip, NULL, cpupart);
2362 mac_set_pool_effective(use_default, cpupart, mrp, emrp);
2363 pool_unlock();
2364 }
2365
2366 /*
2367 * Set up the Rx SRSes. If there is no group associated with the
2368 * client, then only setup SW classification. If the client has
2369 * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an
2370 * SRS for each HW ring. If the client is sharing a group, then make
2371 * sure to teardown the HW SRSes.
2372 */
2373 void
2374 mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2375 uint32_t link_type)
2376 {
2377 mac_impl_t *mip = mcip->mci_mip;
2378 mac_soft_ring_set_t *mac_srs;
2379 mac_ring_t *ring;
2380 uint32_t fanout_type;
2381 mac_group_t *rx_group = flent->fe_rx_ring_group;
2382 boolean_t no_unicast;
2383
2384 /*
2385 * If this is an an aggr port, then don't setup Rx SRS and Rx
2386 * soft rings as they won't be used. However, we still need to
2387 * start the rings to receive data on them.
2388 */
2389 if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) {
2390 if (rx_group == NULL)
2391 return;
2392
2393 for (ring = rx_group->mrg_rings; ring != NULL;
2394 ring = ring->mr_next) {
2395 if (ring->mr_state != MR_INUSE)
2396 (void) mac_start_ring(ring);
2397 }
2398
2399 return;
2400 }
2401
2402 /*
2403 * Aggr ports should never have SRSes.
2404 */
2405 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
2406
2407 fanout_type = mac_find_fanout(flent, link_type);
2408 no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0;
2409
2410 /* Create the SRS for SW classification if none exists */
2411 if (flent->fe_rx_srs[0] == NULL) {
2412 ASSERT(flent->fe_rx_srs_cnt == 0);
2413 mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type,
2414 mac_rx_deliver, mcip, NULL, NULL);
2415 mutex_enter(&flent->fe_lock);
2416 flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc;
2417 flent->fe_cb_arg1 = (void *)mip;
2418 flent->fe_cb_arg2 = (void *)mac_srs;
2419 mutex_exit(&flent->fe_lock);
2420 }
2421
2422 if (rx_group == NULL)
2423 return;
2424
2425 /*
2426 * If the group is marked RESERVED then setup an SRS and
2427 * fanout for each HW ring.
2428 */
2429 switch (rx_group->mrg_state) {
2430 case MAC_GROUP_STATE_RESERVED:
2431 for (ring = rx_group->mrg_rings; ring != NULL;
2432 ring = ring->mr_next) {
2433 uint16_t vid = i_mac_flow_vid(mcip->mci_flent);
2434
2435 switch (ring->mr_state) {
2436 case MR_INUSE:
2437 case MR_FREE:
2438 if (ring->mr_srs != NULL)
2439 break;
2440 if (ring->mr_state != MR_INUSE)
2441 (void) mac_start_ring(ring);
2442
2443 /*
2444 * If a client requires SW VLAN
2445 * filtering or has no unicast address
2446 * then we don't create any HW ring
2447 * SRSes.
2448 */
2449 if ((!MAC_GROUP_HW_VLAN(rx_group) &&
2450 vid != VLAN_ID_NONE) || no_unicast)
2451 break;
2452
2453 /*
2454 * When a client has exclusive use of
2455 * a group, and that group's traffic
2456 * is fully HW classified, we create
2457 * an SRS for each HW ring in order to
2458 * make use of dynamic polling of said
2459 * HW rings.
2460 */
2461 mac_srs = mac_srs_create(mcip, flent,
2462 fanout_type | link_type,
2463 mac_rx_deliver, mcip, NULL, ring);
2464 break;
2465 default:
2466 cmn_err(CE_PANIC,
2467 "srs_setup: mcip = %p "
2468 "trying to add UNKNOWN ring = %p\n",
2469 (void *)mcip, (void *)ring);
2470 break;
2471 }
2472 }
2473 break;
2474 case MAC_GROUP_STATE_SHARED:
2475 /*
2476 * When a group is shared by multiple clients, we must
2477 * use SW classifiction to ensure packets are
2478 * delivered to the correct client.
2479 */
2480 mac_rx_switch_grp_to_sw(rx_group);
2481 break;
2482 default:
2483 ASSERT(B_FALSE);
2484 break;
2485 }
2486 }
2487
2488 /*
2489 * Set up the TX SRS.
2490 */
2491 void
2492 mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2493 uint32_t link_type)
2494 {
2495 /*
2496 * If this is an exclusive client (e.g. an aggr port), then
2497 * don't setup Tx SRS and Tx soft rings as they won't be used.
2498 * However, we still need to start the rings to send data
2499 * across them.
2500 */
2501 if (mcip->mci_state_flags & MCIS_EXCLUSIVE) {
2502 mac_ring_t *ring;
2503 mac_group_t *grp;
2504
2505 grp = (mac_group_t *)flent->fe_tx_ring_group;
2506
2507 if (grp == NULL)
2508 return;
2509
2510 for (ring = grp->mrg_rings; ring != NULL;
2511 ring = ring->mr_next) {
2512 if (ring->mr_state != MR_INUSE)
2513 (void) mac_start_ring(ring);
2514 }
2515
2516 return;
2517 }
2518
2519 /*
2520 * Aggr ports should never have SRSes.
2521 */
2522 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
2523
2524 if (flent->fe_tx_srs == NULL) {
2525 (void) mac_srs_create(mcip, flent, SRST_TX | link_type,
2526 NULL, mcip, NULL, NULL);
2527 }
2528
2529 mac_tx_srs_setup(mcip, flent);
2530 }
2531
2532 /*
2533 * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown
2534 * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set
2535 * when we wish to move a MAC client from one group to another. In
2536 * that case, we need to release the current HW SRSes but keep the SW
2537 * SRS for continued traffic classifiction.
2538 */
2539 void
2540 mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly)
2541 {
2542 mac_soft_ring_set_t *mac_srs;
2543 int i;
2544 int count = flent->fe_rx_srs_cnt;
2545
2546 for (i = 0; i < count; i++) {
2547 if (i == 0 && hwonly)
2548 continue;
2549 mac_srs = flent->fe_rx_srs[i];
2550 mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED);
2551 mac_srs_free(mac_srs);
2552 flent->fe_rx_srs[i] = NULL;
2553 flent->fe_rx_srs_cnt--;
2554 }
2555
2556 /*
2557 * If we are only tearing down the HW SRSes then there must be
2558 * one SRS left for SW classification. Otherwise we are tearing
2559 * down both HW and SW and there should be no SRSes left.
2560 */
2561 if (hwonly)
2562 VERIFY3S(flent->fe_rx_srs_cnt, ==, 1);
2563 else
2564 VERIFY3S(flent->fe_rx_srs_cnt, ==, 0);
2565 }
2566
2567 /*
2568 * Remove the TX SRS.
2569 */
2570 void
2571 mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent,
2572 uint32_t link_type)
2573 {
2574 mac_soft_ring_set_t *tx_srs;
2575 mac_srs_tx_t *tx;
2576
2577 if ((tx_srs = flent->fe_tx_srs) == NULL)
2578 return;
2579
2580 tx = &tx_srs->srs_tx;
2581 switch (link_type) {
2582 case SRST_FLOW:
2583 /*
2584 * For flows, we need to work with passed
2585 * flent to find the Rx/Tx SRS.
2586 */
2587 mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED);
2588 break;
2589 case SRST_LINK:
2590 mac_tx_client_condemn((mac_client_handle_t)mcip);
2591 if (tx->st_arg2 != NULL) {
2592 ASSERT(tx_srs->srs_type & SRST_TX);
2593 /*
2594 * The ring itself will be stopped when
2595 * we release the group or in the
2596 * mac_datapath_teardown (for the default
2597 * group)
2598 */
2599 tx->st_arg2 = NULL;
2600 }
2601 break;
2602 default:
2603 ASSERT(B_FALSE);
2604 break;
2605 }
2606 mac_srs_free(tx_srs);
2607 flent->fe_tx_srs = NULL;
2608 }
2609
2610 /*
2611 * This is the group state machine.
2612 *
2613 * The state of an Rx group is given by
2614 * the following table. The default group and its rings are started in
2615 * mac_start itself and the default group stays in SHARED state until
2616 * mac_stop at which time the group and rings are stopped and and it
2617 * reverts to the Registered state.
2618 *
2619 * Typically this function is called on a group after adding or removing a
2620 * client from it, to find out what should be the new state of the group.
2621 * If the new state is RESERVED, then the client that owns this group
2622 * exclusively is also returned. Note that adding or removing a client from
2623 * a group could also impact the default group and the caller needs to
2624 * evaluate the effect on the default group.
2625 *
2626 * Group type # of clients mi_nactiveclients Group State
2627 * in the group
2628 *
2629 * Non-default 0 N.A. REGISTERED
2630 * Non-default 1 N.A. RESERVED
2631 *
2632 * Default 0 N.A. SHARED
2633 * Default 1 1 RESERVED
2634 * Default 1 > 1 SHARED
2635 * Default > 1 N.A. SHARED
2636 *
2637 * For a TX group, the following is the state table.
2638 *
2639 * Group type # of clients Group State
2640 * in the group
2641 *
2642 * Non-default 0 REGISTERED
2643 * Non-default 1 RESERVED
2644 *
2645 * Default 0 REGISTERED
2646 * Default 1 RESERVED
2647 * Default > 1 SHARED
2648 */
2649 mac_group_state_t
2650 mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip,
2651 mac_group_t *defgrp, boolean_t rx_group)
2652 {
2653 mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh;
2654
2655 *group_only_mcip = NULL;
2656
2657 /* Non-default group */
2658
2659 if (grp != defgrp) {
2660 if (MAC_GROUP_NO_CLIENT(grp))
2661 return (MAC_GROUP_STATE_REGISTERED);
2662
2663 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp);
2664 if (*group_only_mcip != NULL)
2665 return (MAC_GROUP_STATE_RESERVED);
2666
2667 return (MAC_GROUP_STATE_SHARED);
2668 }
2669
2670 /* Default group */
2671
2672 if (MAC_GROUP_NO_CLIENT(grp)) {
2673 if (rx_group)
2674 return (MAC_GROUP_STATE_SHARED);
2675 else
2676 return (MAC_GROUP_STATE_REGISTERED);
2677 }
2678 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp);
2679 if (*group_only_mcip == NULL)
2680 return (MAC_GROUP_STATE_SHARED);
2681
2682 if (rx_group && mip->mi_nactiveclients != 1)
2683 return (MAC_GROUP_STATE_SHARED);
2684
2685 ASSERT(*group_only_mcip != NULL);
2686 return (MAC_GROUP_STATE_RESERVED);
2687 }
2688
2689 /*
2690 * OVERVIEW NOTES FOR DATAPATH
2691 * ===========================
2692 *
2693 * Create an SRS and setup the corresponding flow function and args.
2694 * Add a classification rule for the flow specified by 'flent' and program
2695 * the hardware classifier when applicable.
2696 *
2697 * Rx ring assignment, SRS, polling and B/W enforcement
2698 * ----------------------------------------------------
2699 *
2700 * We try to use H/W classification on NIC and assign traffic to a
2701 * MAC address to a particular Rx ring. There is a 1-1 mapping
2702 * between a SRS and a Rx ring. The SRS (short for soft ring set)
2703 * dynamically switches the underlying Rx ring between interrupt
2704 * and polling mode and enforces any specified B/W control.
2705 *
2706 * There is always a SRS created and tied to each H/W and S/W rule.
2707 * Whenever we create a H/W rule, we always add the the same rule to
2708 * S/W classifier and tie a SRS to it.
2709 *
2710 * In case a B/W control is specified, its broken into bytes
2711 * per ticks and as soon as the quota for a tick is exhausted,
2712 * the underlying Rx ring is forced into poll mode for remianing
2713 * tick. The SRS poll thread only polls for bytes that are
2714 * allowed to come in the SRS. We typically let 4x the configured
2715 * B/W worth of packets to come in the SRS (to prevent unnecessary
2716 * drops due to bursts) but only process the specified amount.
2717 *
2718 * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more
2719 * Rx rings (and corresponding SRSs) assigned to it. The SRS
2720 * in turn can have softrings to do protocol level fanout or
2721 * softrings to do S/W based fanout or both. In case the NIC
2722 * has no Rx rings, we do S/W classification to respective SRS.
2723 * The S/W classification rule is always setup and ready. This
2724 * allows the MAC layer to reassign Rx rings whenever needed
2725 * but packets still continue to flow via the default path and
2726 * getting S/W classified to correct SRS.
2727 *
2728 * In other cases where a NIC or VNIC is plumbed, our goal is use
2729 * H/W classifier and get two Rx ring assigned for the Link. One
2730 * for TCP and one for UDP|SCTP. The respective SRS still do the
2731 * polling on the Rx ring. For Link that is plumbed for IP, there
2732 * is a TCP squeue which also does polling and can control the
2733 * the Rx ring directly (where SRS is just pass through). For
2734 * the following cases, the SRS does the polling underneath.
2735 * 1) non IP based Links (Links which are not plumbed via ifconfig)
2736 * and paths which have no IP squeues (UDP & SCTP)
2737 * 2) If B/W control is specified on the Link
2738 * 3) If S/W fanout is secified
2739 *
2740 * Note1: As of current implementation, we try to assign only 1 Rx
2741 * ring per Link and more than 1 Rx ring for primary Link for
2742 * H/W based fanout. We always create following softrings per SRS:
2743 * 1) TCP softring which is polled by TCP squeue where possible
2744 * (and also bypasses DLS)
2745 * 2) UDP/SCTP based which bypasses DLS
2746 * 3) OTH softring which goes via DLS (currently deal with IPv6
2747 * and non TCP/UDP/SCTP for IPv4 packets).
2748 *
2749 * It is necessary to create 3 softrings since SRS has to poll
2750 * the single Rx ring underneath and enforce any link level B/W
2751 * control (we can't switch the Rx ring in poll mode just based
2752 * on TCP squeue if the same Rx ring is sharing UDP and other
2753 * traffic as well). Once polling is done and any Link level B/W
2754 * control is specified, the packets are assigned to respective
2755 * softring based on protocol. Since TCP has IP based squeue
2756 * which benefits by polling, we separate TCP packets into
2757 * its own softring which can be polled by IP squeue. We need
2758 * to separate out UDP/SCTP to UDP softring since it can bypass
2759 * the DLS layer which has heavy performance advanatges and we
2760 * need a softring (OTH) for rest.
2761 *
2762 * ToDo: The 3 softrings for protocol are needed only till we can
2763 * get rid of DLS from datapath, make IPv4 and IPv6 paths
2764 * symmetric (deal with mac_header_info for v6 and polling for
2765 * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues
2766 * are generic), and bring SAP based classification to MAC layer
2767 *
2768 * H/W and S/W based fanout and multiple Rx rings per Link
2769 * -------------------------------------------------------
2770 *
2771 * In case, fanout is requested (or determined automatically based
2772 * on Link speed and processor speed), we try to assign multiple
2773 * Rx rings per Link with their respective SRS. In this case
2774 * the NIC should be capable of fanning out incoming packets between
2775 * the assigned Rx rings (H/W based fanout). All the SRS
2776 * individually switch their Rx ring between interrupt and polling
2777 * mode but share a common B/W control counter in case of Link
2778 * level B/W is specified.
2779 *
2780 * If S/W based fanout is specified in lieu of H/W based fanout,
2781 * the Link SRS creates the specified number of softrings for
2782 * each protocol (TCP, UDP, OTH). Incoming packets are fanned
2783 * out to the correct softring based on their protocol and
2784 * protocol specific hash function.
2785 *
2786 * Primary and non primary MAC clients
2787 * -----------------------------------
2788 *
2789 * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links
2790 * and are a Layer 2 construct.
2791 *
2792 * Primary NIC:
2793 * The Link that owns the primary MAC address and typically
2794 * is used as the data NIC in non virtualized cases. As such
2795 * H/W resources are preferntially given to primary NIC. As
2796 * far as code is concerned, there is no difference in the
2797 * primary NIC vs VNICs. They are all treated as Links.
2798 * At the very first call to mac_unicast_add() we program the S/W
2799 * classifier for the primary MAC address, get a soft ring set
2800 * (and soft rings based on 'ip_soft_ring_cnt')
2801 * and a Rx ring assigned for polling to get enabled.
2802 * When IP get plumbed and negotiates polling, we can
2803 * let squeue do the polling on TCP softring.
2804 *
2805 * VNICs:
2806 * Same as any other Link. As long as the H/W resource assignments
2807 * are equal, the data path and setup for all Links is same.
2808 *
2809 * Flows:
2810 * Can be configured on Links. They have their own SRS and the
2811 * S/W classifier is programmed appropriately based on the flow.
2812 * The flows typically deal with layer 3 and above and
2813 * creates a soft ring set specific to the flow. The receive
2814 * side function is switched from mac_rx_srs_process to
2815 * mac_rx_srs_subflow_process which first tries to assign the
2816 * packet to appropriate flow SRS and failing which assigns it
2817 * to link SRS. This allows us to avoid the layered approach
2818 * which gets complex.
2819 *
2820 * By the time mac_datapath_setup() completes, we already have the
2821 * soft rings set, Rx rings, soft rings, etc figured out and both H/W
2822 * and S/W classifiers programmed. IP is not plumbed yet (and might
2823 * never be for Virtual Machines guest OS path). When IP is plumbed
2824 * (for both NIC and VNIC), we do a capability negotiation for polling
2825 * and upcall functions etc.
2826 *
2827 * Rx ring Assignement NOTES
2828 * -------------------------
2829 *
2830 * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings
2831 * as NIC with a single default ring), we assign the only ring to
2832 * primary Link. The primary Link SRS can do polling on it as long as
2833 * it is the only link in use and we compare the MAC address for unicast
2834 * packets before accepting an incoming packet (there is no need for S/W
2835 * classification in this case). We disable polling on the only ring the
2836 * moment 2nd link gets created (the polling remains enabled even though
2837 * there are broadcast and * multicast flows created).
2838 *
2839 * If the NIC has more than 1 Rx ring, we assign the default ring (the
2840 * 1st ring) to deal with broadcast, multicast and traffic for other
2841 * NICs which needs S/W classification. We assign the primary mac
2842 * addresses to another ring by specifiying a classification rule for
2843 * primary unicast MAC address to the selected ring. The primary Link
2844 * (and its SRS) can continue to poll the assigned Rx ring at all times
2845 * independantly.
2846 *
2847 * Note: In future, if no fanout is specified, we try to assign 2 Rx
2848 * rings for the primary Link with the primary MAC address + TCP going
2849 * to one ring and primary MAC address + UDP|SCTP going to other ring.
2850 * Any remaining traffic for primary MAC address can go to the default
2851 * Rx ring and get S/W classified. This way the respective SRSs don't
2852 * need to do proto fanout and don't need to have softrings at all and
2853 * can poll their respective Rx rings.
2854 *
2855 * As an optimization, when a new NIC or VNIC is created, we can get
2856 * only one Rx ring and make it a TCP specific Rx ring and use the
2857 * H/W default Rx ring for the rest (this Rx ring is never polled).
2858 *
2859 * For clients that don't have MAC address, but want to receive and
2860 * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath.
2861 * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we
2862 * always give the default group and use software classification (i.e.
2863 * even if this is the only client in the default group, we will
2864 * leave group as shared).
2865 */
2866
2867 int
2868 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2869 uint32_t link_type)
2870 {
2871 mac_impl_t *mip = mcip->mci_mip;
2872 mac_group_t *rgroup = NULL;
2873 mac_group_t *tgroup = NULL;
2874 mac_group_t *default_rgroup;
2875 mac_group_t *default_tgroup;
2876 int err;
2877 uint16_t vid;
2878 uint8_t *mac_addr;
2879 mac_group_state_t next_state;
2880 mac_client_impl_t *group_only_mcip;
2881 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip);
2882 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip);
2883 boolean_t rxhw;
2884 boolean_t txhw;
2885 boolean_t use_default = B_FALSE;
2886 cpupart_t *cpupart;
2887 boolean_t no_unicast;
2888 boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC;
2889 mac_client_impl_t *reloc_pmcip = NULL;
2890 boolean_t use_hw;
2891
2892 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
2893
2894 switch (link_type) {
2895 case SRST_FLOW:
2896 mac_srs_group_setup(mcip, flent, link_type);
2897 return (0);
2898
2899 case SRST_LINK:
2900 no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR;
2901 mac_addr = flent->fe_flow_desc.fd_dst_mac;
2902
2903 /* Default RX group */
2904 default_rgroup = MAC_DEFAULT_RX_GROUP(mip);
2905
2906 /* Default TX group */
2907 default_tgroup = MAC_DEFAULT_TX_GROUP(mip);
2908
2909 if (no_unicast) {
2910 rgroup = default_rgroup;
2911 tgroup = default_tgroup;
2912 goto grp_found;
2913 }
2914 rxhw = (mrp->mrp_mask & MRP_RX_RINGS) &&
2915 (mrp->mrp_nrxrings > 0 ||
2916 (mrp->mrp_mask & MRP_RXRINGS_UNSPEC));
2917 txhw = (mrp->mrp_mask & MRP_TX_RINGS) &&
2918 (mrp->mrp_ntxrings > 0 ||
2919 (mrp->mrp_mask & MRP_TXRINGS_UNSPEC));
2920
2921 /*
2922 * All the rings initially belong to the default group
2923 * under dynamic grouping. The primary client uses the
2924 * default group when it is the only client. The
2925 * default group is also used as the destination for
2926 * all multicast and broadcast traffic of all clients.
2927 * Therefore, the primary client loses its ability to
2928 * poll the softrings on addition of a second client.
2929 * To avoid a performance penalty, MAC will move the
2930 * primary client to a dedicated group when it can.
2931 *
2932 * When using static grouping, the primary client
2933 * begins life on a non-default group. There is
2934 * no moving needed upon addition of a second client.
2935 */
2936 if (!isprimary && mip->mi_nactiveclients == 2 &&
2937 (group_only_mcip = mac_primary_client_handle(mip)) !=
2938 NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) {
2939 reloc_pmcip = mac_check_primary_relocation(
2940 group_only_mcip, rxhw);
2941 }
2942
2943 /*
2944 * Check to see if we can get an exclusive group for
2945 * this mac address or if there already exists a
2946 * group that has this mac address (case of VLANs).
2947 * If no groups are available, use the default group.
2948 */
2949 rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE);
2950 if (rgroup == NULL && rxhw) {
2951 err = ENOSPC;
2952 goto setup_failed;
2953 } else if (rgroup == NULL) {
2954 rgroup = default_rgroup;
2955 }
2956
2957 /*
2958 * If we are adding a second client to a
2959 * non-default group then we need to move the
2960 * existing client to the default group and
2961 * add the new client to the default group as
2962 * well.
2963 */
2964 if (rgroup != default_rgroup &&
2965 rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) {
2966 group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup);
2967 err = mac_rx_switch_group(group_only_mcip, rgroup,
2968 default_rgroup);
2969
2970 if (err != 0)
2971 goto setup_failed;
2972
2973 rgroup = default_rgroup;
2974 }
2975
2976 /*
2977 * Check to see if we can get an exclusive group for
2978 * this mac client. If no groups are available, use
2979 * the default group.
2980 */
2981 tgroup = mac_reserve_tx_group(mcip, B_FALSE);
2982 if (tgroup == NULL && txhw) {
2983 if (rgroup != NULL && rgroup != default_rgroup)
2984 mac_release_rx_group(mcip, rgroup);
2985 err = ENOSPC;
2986 goto setup_failed;
2987 } else if (tgroup == NULL) {
2988 tgroup = default_tgroup;
2989 }
2990
2991 /*
2992 * Some NICs don't support any Rx rings, so there may not
2993 * even be a default group.
2994 */
2995 grp_found:
2996 if (rgroup != NULL) {
2997 if (rgroup != default_rgroup &&
2998 MAC_GROUP_NO_CLIENT(rgroup) &&
2999 (rxhw || mcip->mci_share != 0)) {
3000 MAC_RX_GRP_RESERVED(mip);
3001 if (mip->mi_rx_group_type ==
3002 MAC_GROUP_TYPE_DYNAMIC) {
3003 MAC_RX_RING_RESERVED(mip,
3004 rgroup->mrg_cur_count);
3005 }
3006 }
3007
3008 flent->fe_rx_ring_group = rgroup;
3009 /*
3010 * Add the client to the group and update the
3011 * group's state. If rgroup != default_group
3012 * then the rgroup should only ever have one
3013 * client and be in the RESERVED state. But no
3014 * matter what, the default_rgroup will enter
3015 * the SHARED state since it has to receive
3016 * all broadcast and multicast traffic. This
3017 * case is handled later in the function.
3018 */
3019 mac_group_add_client(rgroup, mcip);
3020 next_state = mac_group_next_state(rgroup,
3021 &group_only_mcip, default_rgroup, B_TRUE);
3022 mac_set_group_state(rgroup, next_state);
3023 }
3024
3025 if (tgroup != NULL) {
3026 if (tgroup != default_tgroup &&
3027 MAC_GROUP_NO_CLIENT(tgroup) &&
3028 (txhw || mcip->mci_share != 0)) {
3029 MAC_TX_GRP_RESERVED(mip);
3030 if (mip->mi_tx_group_type ==
3031 MAC_GROUP_TYPE_DYNAMIC) {
3032 MAC_TX_RING_RESERVED(mip,
3033 tgroup->mrg_cur_count);
3034 }
3035 }
3036 flent->fe_tx_ring_group = tgroup;
3037 mac_group_add_client(tgroup, mcip);
3038 next_state = mac_group_next_state(tgroup,
3039 &group_only_mcip, default_tgroup, B_FALSE);
3040 tgroup->mrg_state = next_state;
3041 }
3042
3043 /* We are setting up minimal datapath only */
3044 if (no_unicast) {
3045 mac_srs_group_setup(mcip, flent, link_type);
3046 break;
3047 }
3048
3049 /* Program software classification. */
3050 if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0)
3051 goto setup_failed;
3052
3053 /* Program hardware classification. */
3054 vid = i_mac_flow_vid(flent);
3055 use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0;
3056 err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw);
3057
3058 if (err != 0)
3059 goto setup_failed;
3060
3061 mcip->mci_unicast = mac_find_macaddr(mip, mac_addr);
3062 VERIFY3P(mcip->mci_unicast, !=, NULL);
3063
3064 /*
3065 * Setup the Rx and Tx SRSes. If the client has a
3066 * reserved group, then mac_srs_group_setup() creates
3067 * the required SRSes for the HW rings. If we have a
3068 * shared group, mac_srs_group_setup() dismantles the
3069 * HW SRSes of the previously exclusive group.
3070 */
3071 mac_srs_group_setup(mcip, flent, link_type);
3072
3073 /* (Re)init the v6 token & local addr used by link protection */
3074 mac_protect_update_mac_token(mcip);
3075 break;
3076
3077 default:
3078 ASSERT(B_FALSE);
3079 break;
3080 }
3081
3082 /*
3083 * All broadcast and multicast traffic is received only on the default
3084 * group. If we have setup the datapath for a non-default group above
3085 * then move the default group to shared state to allow distribution of
3086 * incoming broadcast traffic to the other groups and dismantle the
3087 * SRSes over the default group.
3088 */
3089 if (rgroup != NULL) {
3090 if (rgroup != default_rgroup) {
3091 if (default_rgroup->mrg_state ==
3092 MAC_GROUP_STATE_RESERVED) {
3093 group_only_mcip = MAC_GROUP_ONLY_CLIENT(
3094 default_rgroup);
3095 ASSERT(group_only_mcip != NULL &&
3096 mip->mi_nactiveclients > 1);
3097
3098 mac_set_group_state(default_rgroup,
3099 MAC_GROUP_STATE_SHARED);
3100 mac_rx_srs_group_setup(group_only_mcip,
3101 group_only_mcip->mci_flent, SRST_LINK);
3102 pool_lock();
3103 cpupart = mac_pset_find(mrp, &use_default);
3104 mac_fanout_setup(group_only_mcip,
3105 group_only_mcip->mci_flent,
3106 MCIP_RESOURCE_PROPS(group_only_mcip),
3107 mac_rx_deliver, group_only_mcip, NULL,
3108 cpupart);
3109 mac_set_pool_effective(use_default, cpupart,
3110 mrp, emrp);
3111 pool_unlock();
3112 }
3113 ASSERT(default_rgroup->mrg_state ==
3114 MAC_GROUP_STATE_SHARED);
3115 }
3116
3117 /*
3118 * A VLAN MAC client on a reserved group still
3119 * requires SW classification if the MAC doesn't
3120 * provide VLAN HW filtering.
3121 *
3122 * Clients with no unicast address also require SW
3123 * classification.
3124 */
3125 if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED &&
3126 ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) ||
3127 no_unicast)) {
3128 mac_rx_switch_grp_to_sw(rgroup);
3129 }
3130
3131 }
3132
3133 mac_set_rings_effective(mcip);
3134 return (0);
3135
3136 setup_failed:
3137 /* Switch the primary back to default group */
3138 if (reloc_pmcip != NULL) {
3139 (void) mac_rx_switch_group(reloc_pmcip,
3140 reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup);
3141 }
3142 mac_datapath_teardown(mcip, flent, link_type);
3143 return (err);
3144 }
3145
3146 void
3147 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent,
3148 uint32_t link_type)
3149 {
3150 mac_impl_t *mip = mcip->mci_mip;
3151 mac_group_t *group = NULL;
3152 mac_client_impl_t *grp_only_mcip;
3153 flow_entry_t *group_only_flent;
3154 mac_group_t *default_group;
3155 boolean_t check_default_group = B_FALSE;
3156 mac_group_state_t next_state;
3157 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip);
3158 uint16_t vid;
3159
3160 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
3161
3162 switch (link_type) {
3163 case SRST_FLOW:
3164 mac_rx_srs_group_teardown(flent, B_FALSE);
3165 mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW);
3166 return;
3167
3168 case SRST_LINK:
3169 /* Stop sending packets */
3170 mac_tx_client_block(mcip);
3171 group = flent->fe_rx_ring_group;
3172 vid = i_mac_flow_vid(flent);
3173
3174 /*
3175 * Stop the packet flow from the hardware by disabling
3176 * any hardware filters assigned to this client.
3177 */
3178 if (mcip->mci_unicast != NULL) {
3179 int err;
3180
3181 err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid);
3182
3183 if (err != 0) {
3184 cmn_err(CE_WARN, "%s: failed to remove a MAC HW"
3185 " filters because of error 0x%x",
3186 mip->mi_name, err);
3187 }
3188
3189 mcip->mci_unicast = NULL;
3190 }
3191
3192 /* Stop the packets coming from the S/W classifier */
3193 mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE);
3194 mac_flow_wait(flent, FLOW_DRIVER_UPCALL);
3195
3196 /* Quiesce and destroy all the SRSes. */
3197 mac_rx_srs_group_teardown(flent, B_FALSE);
3198 mac_tx_srs_group_teardown(mcip, flent, SRST_LINK);
3199
3200 ASSERT3P(mcip->mci_flent, ==, flent);
3201 ASSERT3P(flent->fe_next, ==, NULL);
3202
3203 /*
3204 * Release our hold on the group as well. We need
3205 * to check if the shared group has only one client
3206 * left who can use it exclusively. Also, if we
3207 * were the last client, release the group.
3208 */
3209 default_group = MAC_DEFAULT_RX_GROUP(mip);
3210 if (group != NULL) {
3211 mac_group_remove_client(group, mcip);
3212 next_state = mac_group_next_state(group,
3213 &grp_only_mcip, default_group, B_TRUE);
3214
3215 if (next_state == MAC_GROUP_STATE_RESERVED) {
3216 /*
3217 * Only one client left on this RX group.
3218 */
3219 VERIFY3P(grp_only_mcip, !=, NULL);
3220 mac_set_group_state(group,
3221 MAC_GROUP_STATE_RESERVED);
3222 group_only_flent = grp_only_mcip->mci_flent;
3223
3224 /*
3225 * The only remaining client has exclusive
3226 * access on the group. Allow it to
3227 * dynamically poll the H/W rings etc.
3228 */
3229 mac_rx_srs_group_setup(grp_only_mcip,
3230 group_only_flent, SRST_LINK);
3231 mac_fanout_setup(grp_only_mcip,
3232 group_only_flent,
3233 MCIP_RESOURCE_PROPS(grp_only_mcip),
3234 mac_rx_deliver, grp_only_mcip, NULL, NULL);
3235 mac_rx_group_unmark(group, MR_INCIPIENT);
3236 mac_set_rings_effective(grp_only_mcip);
3237 } else if (next_state == MAC_GROUP_STATE_REGISTERED) {
3238 /*
3239 * This is a non-default group being freed up.
3240 * We need to reevaluate the default group
3241 * to see if the primary client can get
3242 * exclusive access to the default group.
3243 */
3244 VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip));
3245 if (mrp->mrp_mask & MRP_RX_RINGS) {
3246 MAC_RX_GRP_RELEASED(mip);
3247 if (mip->mi_rx_group_type ==
3248 MAC_GROUP_TYPE_DYNAMIC) {
3249 MAC_RX_RING_RELEASED(mip,
3250 group->mrg_cur_count);
3251 }
3252 }
3253 mac_release_rx_group(mcip, group);
3254 mac_set_group_state(group,
3255 MAC_GROUP_STATE_REGISTERED);
3256 check_default_group = B_TRUE;
3257 } else {
3258 VERIFY3S(next_state, ==,
3259 MAC_GROUP_STATE_SHARED);
3260 mac_set_group_state(group,
3261 MAC_GROUP_STATE_SHARED);
3262 mac_rx_group_unmark(group, MR_CONDEMNED);
3263 }
3264 flent->fe_rx_ring_group = NULL;
3265 }
3266 /*
3267 * Remove the client from the TX group. Additionally, if
3268 * this a non-default group, then we also need to release
3269 * the group.
3270 */
3271 group = flent->fe_tx_ring_group;
3272 default_group = MAC_DEFAULT_TX_GROUP(mip);
3273 if (group != NULL) {
3274 mac_group_remove_client(group, mcip);
3275 next_state = mac_group_next_state(group,
3276 &grp_only_mcip, default_group, B_FALSE);
3277 if (next_state == MAC_GROUP_STATE_REGISTERED) {
3278 if (group != default_group) {
3279 if (mrp->mrp_mask & MRP_TX_RINGS) {
3280 MAC_TX_GRP_RELEASED(mip);
3281 if (mip->mi_tx_group_type ==
3282 MAC_GROUP_TYPE_DYNAMIC) {
3283 MAC_TX_RING_RELEASED(
3284 mip, group->
3285 mrg_cur_count);
3286 }
3287 }
3288 mac_release_tx_group(mcip, group);
3289 /*
3290 * If the default group is reserved,
3291 * then we need to set the effective
3292 * rings as we would have given
3293 * back some rings when the group
3294 * was released
3295 */
3296 if (mip->mi_tx_group_type ==
3297 MAC_GROUP_TYPE_DYNAMIC &&
3298 default_group->mrg_state ==
3299 MAC_GROUP_STATE_RESERVED) {
3300 grp_only_mcip =
3301 MAC_GROUP_ONLY_CLIENT
3302 (default_group);
3303 mac_set_rings_effective(
3304 grp_only_mcip);
3305 }
3306 } else {
3307 mac_ring_t *ring;
3308 int cnt;
3309 int ringcnt;
3310
3311 /*
3312 * Stop all the rings except the
3313 * default ring.
3314 */
3315 ringcnt = group->mrg_cur_count;
3316 ring = group->mrg_rings;
3317 for (cnt = 0; cnt < ringcnt; cnt++) {
3318 if (ring->mr_state ==
3319 MR_INUSE && ring !=
3320 (mac_ring_t *)
3321 mip->mi_default_tx_ring) {
3322 mac_stop_ring(ring);
3323 ring->mr_flag = 0;
3324 }
3325 ring = ring->mr_next;
3326 }
3327 }
3328 } else if (next_state == MAC_GROUP_STATE_RESERVED) {
3329 mac_set_rings_effective(grp_only_mcip);
3330 }
3331 flent->fe_tx_ring_group = NULL;
3332 group->mrg_state = next_state;
3333 }
3334 break;
3335 default:
3336 ASSERT(B_FALSE);
3337 break;
3338 }
3339
3340 /*
3341 * The mac client using the default group gets exclusive access to the
3342 * default group if and only if it is the sole client on the entire
3343 * mip. If so set the group state to reserved, and set up the SRSes
3344 * over the default group.
3345 */
3346 if (check_default_group) {
3347 default_group = MAC_DEFAULT_RX_GROUP(mip);
3348 VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED);
3349 next_state = mac_group_next_state(default_group,
3350 &grp_only_mcip, default_group, B_TRUE);
3351 if (next_state == MAC_GROUP_STATE_RESERVED) {
3352 VERIFY3P(grp_only_mcip, !=, NULL);
3353 VERIFY3U(mip->mi_nactiveclients, ==, 1);
3354 mac_set_group_state(default_group,
3355 MAC_GROUP_STATE_RESERVED);
3356 mac_rx_srs_group_setup(grp_only_mcip,
3357 grp_only_mcip->mci_flent, SRST_LINK);
3358 mac_fanout_setup(grp_only_mcip,
3359 grp_only_mcip->mci_flent,
3360 MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver,
3361 grp_only_mcip, NULL, NULL);
3362 mac_rx_group_unmark(default_group, MR_INCIPIENT);
3363 mac_set_rings_effective(grp_only_mcip);
3364 }
3365 }
3366
3367 /*
3368 * If the primary is the only one left and the MAC supports
3369 * dynamic grouping, we need to see if the primary needs to
3370 * be moved to the default group so that it can use all the
3371 * H/W rings.
3372 */
3373 if (!(flent->fe_type & FLOW_PRIMARY_MAC) &&
3374 mip->mi_nactiveclients == 1 &&
3375 mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) {
3376 default_group = MAC_DEFAULT_RX_GROUP(mip);
3377 grp_only_mcip = mac_primary_client_handle(mip);
3378 if (grp_only_mcip == NULL)
3379 return;
3380 group_only_flent = grp_only_mcip->mci_flent;
3381 mrp = MCIP_RESOURCE_PROPS(grp_only_mcip);
3382 /*
3383 * If the primary has an explicit property set, leave it
3384 * alone.
3385 */
3386 if (mrp->mrp_mask & MRP_RX_RINGS)
3387 return;
3388 /*
3389 * Switch the primary to the default group.
3390 */
3391 (void) mac_rx_switch_group(grp_only_mcip,
3392 group_only_flent->fe_rx_ring_group, default_group);
3393 }
3394 }
3395
3396 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */
3397
3398 static void
3399 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs)
3400 {
3401 if (mac_srs->srs_type & SRST_TX) {
3402 mac_srs_tx_t *tx;
3403
3404 ASSERT(mac_srs->srs_tcp_soft_rings == NULL);
3405 ASSERT(mac_srs->srs_udp_soft_rings == NULL);
3406 ASSERT(mac_srs->srs_oth_soft_rings == NULL);
3407 ASSERT(mac_srs->srs_tx_soft_rings != NULL);
3408 kmem_free(mac_srs->srs_tx_soft_rings,
3409 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP);
3410 mac_srs->srs_tx_soft_rings = NULL;
3411 tx = &mac_srs->srs_tx;
3412 if (tx->st_soft_rings != NULL) {
3413 kmem_free(tx->st_soft_rings,
3414 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP);
3415 }
3416 } else {
3417 ASSERT(mac_srs->srs_tx_soft_rings == NULL);
3418 ASSERT(mac_srs->srs_tcp_soft_rings != NULL);
3419 kmem_free(mac_srs->srs_tcp_soft_rings,
3420 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3421 mac_srs->srs_tcp_soft_rings = NULL;
3422 ASSERT(mac_srs->srs_udp_soft_rings != NULL);
3423 kmem_free(mac_srs->srs_udp_soft_rings,
3424 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3425 mac_srs->srs_udp_soft_rings = NULL;
3426 ASSERT(mac_srs->srs_oth_soft_rings != NULL);
3427 kmem_free(mac_srs->srs_oth_soft_rings,
3428 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3429 mac_srs->srs_oth_soft_rings = NULL;
3430 }
3431 }
3432
3433 /*
3434 * An RX SRS is attached to at most one mac_ring.
3435 * A TX SRS has no rings.
3436 */
3437 static void
3438 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs)
3439 {
3440 mac_client_impl_t *mcip;
3441 mac_ring_t *ring;
3442 flow_entry_t *flent;
3443
3444 ring = mac_srs->srs_ring;
3445 if (mac_srs->srs_type & SRST_TX) {
3446 ASSERT(ring == NULL);
3447 return;
3448 }
3449
3450 if (ring == NULL)
3451 return;
3452
3453 /*
3454 * Broadcast flows don't have a client impl association, but they
3455 * use only soft rings.
3456 */
3457 flent = mac_srs->srs_flent;
3458 mcip = flent->fe_mcip;
3459 ASSERT(mcip != NULL);
3460
3461 ring->mr_classify_type = MAC_NO_CLASSIFIER;
3462 ring->mr_srs = NULL;
3463 }
3464
3465 /*
3466 * Physical unlink and free of the data structures happen below. This is
3467 * driven from mac_flow_destroy(), on the last refrele of a flow.
3468 *
3469 * Assumes Rx srs is 1-1 mapped with an ring.
3470 */
3471 void
3472 mac_srs_free(mac_soft_ring_set_t *mac_srs)
3473 {
3474 ASSERT(mac_srs->srs_mcip == NULL ||
3475 MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
3476 ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE |
3477 SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE));
3478
3479 mac_pkt_drop(NULL, NULL, mac_srs->srs_first, B_FALSE);
3480 mac_srs_ring_free(mac_srs);
3481 mac_srs_soft_rings_free(mac_srs);
3482 mac_srs_fanout_list_free(mac_srs);
3483
3484 mac_srs->srs_bw = NULL;
3485 mac_srs_stat_delete(mac_srs);
3486 kmem_cache_free(mac_srs_cache, mac_srs);
3487 }
3488
3489 static void
3490 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag)
3491 {
3492 mac_soft_ring_t *softring;
3493
3494 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3495
3496 mac_srs_soft_rings_signal(mac_srs, s_ring_flag);
3497 if (s_ring_flag == S_RING_CONDEMNED) {
3498 while (mac_srs->srs_soft_ring_condemned_count !=
3499 mac_srs->srs_soft_ring_count)
3500 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3501 } else {
3502 while (mac_srs->srs_soft_ring_quiesced_count !=
3503 mac_srs->srs_soft_ring_count)
3504 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3505 }
3506 mutex_exit(&mac_srs->srs_lock);
3507
3508 for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3509 softring = softring->s_ring_next) {
3510 (void) untimeout(softring->s_ring_tid);
3511 softring->s_ring_tid = NULL;
3512 }
3513
3514 (void) untimeout(mac_srs->srs_tid);
3515 mac_srs->srs_tid = NULL;
3516
3517 mutex_enter(&mac_srs->srs_lock);
3518 }
3519
3520 /*
3521 * The block comment above mac_rx_classify_flow_state_change explains the
3522 * background. At this point upcalls from the driver (both hardware classified
3523 * and software classified) have been cut off. We now need to quiesce the
3524 * SRS worker, poll, and softring threads. The SRS worker thread serves as
3525 * the master controller. The steps involved are described below in the function
3526 */
3527 void
3528 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs)
3529 {
3530 uint_t s_ring_flag;
3531 uint_t srs_poll_wait_flag;
3532
3533 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3534 ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE));
3535
3536 if (mac_srs->srs_state & SRS_CONDEMNED) {
3537 s_ring_flag = S_RING_CONDEMNED;
3538 srs_poll_wait_flag = SRS_POLL_THR_EXITED;
3539 } else {
3540 s_ring_flag = S_RING_QUIESCE;
3541 srs_poll_wait_flag = SRS_POLL_THR_QUIESCED;
3542 }
3543
3544 /*
3545 * In the case of Rx SRS wait till the poll thread is done.
3546 */
3547 if ((mac_srs->srs_type & SRST_TX) == 0 &&
3548 mac_srs->srs_poll_thr != NULL) {
3549 while (!(mac_srs->srs_state & srs_poll_wait_flag))
3550 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3551
3552 /*
3553 * Turn off polling as part of the quiesce operation.
3554 */
3555 MAC_SRS_POLLING_OFF(mac_srs);
3556 mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS);
3557 }
3558
3559 /*
3560 * Then signal the soft ring worker threads to quiesce or quit
3561 * as needed and then wait till that happens.
3562 */
3563 mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag);
3564
3565 if (mac_srs->srs_state & SRS_CONDEMNED)
3566 mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE);
3567 else
3568 mac_srs->srs_state |= SRS_QUIESCE_DONE;
3569 cv_signal(&mac_srs->srs_quiesce_done_cv);
3570 }
3571
3572 /*
3573 * Signal an SRS to start a temporary quiesce, or permanent removal, or restart
3574 * a quiesced SRS by setting the appropriate flags and signaling the SRS worker
3575 * or poll thread. This function is internal to the quiescing logic and is
3576 * called internally from the SRS quiesce or flow quiesce or client quiesce
3577 * higher level functions.
3578 */
3579 void
3580 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag)
3581 {
3582 mac_ring_t *ring;
3583
3584 ring = mac_srs->srs_ring;
3585 ASSERT(ring == NULL || ring->mr_refcnt == 0);
3586
3587 if (srs_flag == SRS_CONDEMNED) {
3588 /*
3589 * The SRS is going away. We need to unbind the SRS and SR
3590 * threads before removing from the global SRS list. Otherwise
3591 * there is a small window where the cpu reconfig callbacks
3592 * may miss the SRS in the list walk and DR could fail since
3593 * there are still bound threads.
3594 */
3595 mac_srs_threads_unbind(mac_srs);
3596 mac_srs_remove_glist(mac_srs);
3597 }
3598 /*
3599 * Wakeup the SRS worker and poll threads.
3600 */
3601 mutex_enter(&mac_srs->srs_lock);
3602 mac_srs->srs_state |= srs_flag;
3603 cv_signal(&mac_srs->srs_async);
3604 cv_signal(&mac_srs->srs_cv);
3605 mutex_exit(&mac_srs->srs_lock);
3606 }
3607
3608 /*
3609 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls
3610 * from the driver are done, then the Rx SRS is quiesced and only then can
3611 * we signal the soft rings. Thus this function can't be called arbitrarily
3612 * without satisfying the prerequisites. On the Tx side, the threads from
3613 * top need to quiesced, then the Tx SRS and only then can we signal the
3614 * Tx soft rings.
3615 */
3616 static void
3617 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag)
3618 {
3619 mac_soft_ring_t *softring;
3620
3621 for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3622 softring = softring->s_ring_next)
3623 mac_soft_ring_signal(softring, sr_flag);
3624 }
3625
3626 /*
3627 * The block comment above mac_rx_classify_flow_state_change explains the
3628 * background. At this point the SRS is quiesced and we need to restart the
3629 * SRS worker, poll, and softring threads. The SRS worker thread serves as
3630 * the master controller. The steps involved are described below in the function
3631 */
3632 void
3633 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs)
3634 {
3635 boolean_t iam_rx_srs;
3636 mac_soft_ring_t *softring;
3637
3638 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3639 if ((mac_srs->srs_type & SRST_TX) != 0) {
3640 iam_rx_srs = B_FALSE;
3641 ASSERT((mac_srs->srs_state &
3642 (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) ==
3643 (SRS_QUIESCE_DONE | SRS_QUIESCE));
3644 } else {
3645 iam_rx_srs = B_TRUE;
3646 ASSERT((mac_srs->srs_state &
3647 (SRS_QUIESCE_DONE | SRS_QUIESCE)) ==
3648 (SRS_QUIESCE_DONE | SRS_QUIESCE));
3649 if (mac_srs->srs_poll_thr != NULL) {
3650 ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) ==
3651 SRS_POLL_THR_QUIESCED);
3652 }
3653 }
3654
3655 /*
3656 * Signal any quiesced soft ring workers to restart and wait for the
3657 * soft ring down count to come down to zero.
3658 */
3659 if (mac_srs->srs_soft_ring_quiesced_count != 0) {
3660 for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3661 softring = softring->s_ring_next) {
3662 if (!(softring->s_ring_state & S_RING_QUIESCE))
3663 continue;
3664 mac_soft_ring_signal(softring, S_RING_RESTART);
3665 }
3666 while (mac_srs->srs_soft_ring_quiesced_count != 0)
3667 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3668 }
3669
3670 mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART);
3671 if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) {
3672 /*
3673 * Signal the poll thread and ask it to restart. Wait till it
3674 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets
3675 * cleared.
3676 */
3677 mac_srs->srs_state |= SRS_POLL_THR_RESTART;
3678 cv_signal(&mac_srs->srs_cv);
3679 while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED)
3680 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3681 ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART));
3682 }
3683 /* Wake up any waiter waiting for the restart to complete */
3684 mac_srs->srs_state |= SRS_RESTART_DONE;
3685 cv_signal(&mac_srs->srs_quiesce_done_cv);
3686 }
3687
3688 static void
3689 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs)
3690 {
3691 mutex_enter(&mac_srs->srs_lock);
3692 if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) {
3693 ASSERT(mac_srs->srs_worker_cpuid == -1);
3694 mutex_exit(&mac_srs->srs_lock);
3695 return;
3696 }
3697
3698 mac_srs->srs_worker_cpuid = -1;
3699 mac_srs->srs_state &= ~SRS_WORKER_BOUND;
3700 thread_affinity_clear(mac_srs->srs_worker);
3701 mutex_exit(&mac_srs->srs_lock);
3702 }
3703
3704 static void
3705 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs)
3706 {
3707 mutex_enter(&mac_srs->srs_lock);
3708 if (mac_srs->srs_poll_thr == NULL ||
3709 (mac_srs->srs_state & SRS_POLL_BOUND) == 0) {
3710 ASSERT(mac_srs->srs_poll_cpuid == -1);
3711 mutex_exit(&mac_srs->srs_lock);
3712 return;
3713 }
3714
3715 mac_srs->srs_poll_cpuid = -1;
3716 mac_srs->srs_state &= ~SRS_POLL_BOUND;
3717 thread_affinity_clear(mac_srs->srs_poll_thr);
3718 mutex_exit(&mac_srs->srs_lock);
3719 }
3720
3721 static void
3722 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs)
3723 {
3724 mac_soft_ring_t *soft_ring;
3725
3726 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
3727
3728 mutex_enter(&cpu_lock);
3729 mac_srs_worker_unbind(mac_srs);
3730 if (!(mac_srs->srs_type & SRST_TX))
3731 mac_srs_poll_unbind(mac_srs);
3732
3733 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
3734 soft_ring = soft_ring->s_ring_next) {
3735 mac_soft_ring_unbind(soft_ring);
3736 }
3737 mutex_exit(&cpu_lock);
3738 }
3739
3740 /*
3741 * When a CPU is going away, unbind all MAC threads which are bound
3742 * to that CPU. The affinity of the thread to the CPU is saved to allow
3743 * the thread to be rebound to the CPU if it comes back online.
3744 */
3745 static void
3746 mac_walk_srs_and_unbind(int cpuid)
3747 {
3748 mac_soft_ring_set_t *mac_srs;
3749 mac_soft_ring_t *soft_ring;
3750
3751 rw_enter(&mac_srs_g_lock, RW_READER);
3752
3753 if ((mac_srs = mac_srs_g_list) == NULL)
3754 goto done;
3755
3756 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) {
3757 if (mac_srs->srs_worker_cpuid == cpuid) {
3758 mac_srs->srs_worker_cpuid_save = cpuid;
3759 mac_srs_worker_unbind(mac_srs);
3760 }
3761
3762 if (!(mac_srs->srs_type & SRST_TX)) {
3763 if (mac_srs->srs_poll_cpuid == cpuid) {
3764 mac_srs->srs_poll_cpuid_save = cpuid;
3765 mac_srs_poll_unbind(mac_srs);
3766 }
3767 }
3768
3769 /* Next tackle the soft rings associated with the srs */
3770 mutex_enter(&mac_srs->srs_lock);
3771 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
3772 soft_ring = soft_ring->s_ring_next) {
3773 if (soft_ring->s_ring_cpuid == cpuid) {
3774 soft_ring->s_ring_cpuid_save = cpuid;
3775 mac_soft_ring_unbind(soft_ring);
3776 }
3777 }
3778 mutex_exit(&mac_srs->srs_lock);
3779 }
3780 done:
3781 rw_exit(&mac_srs_g_lock);
3782 }
3783
3784 /* TX SETUP and TEARDOWN ROUTINES */
3785
3786 /*
3787 * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring()
3788 * handle the case where the number of rings is one. I.e. there is
3789 * a ring pointed to by mac_srs->srs_tx_arg2.
3790 */
3791 void
3792 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring)
3793 {
3794 mac_client_impl_t *mcip = mac_srs->srs_mcip;
3795 mac_soft_ring_t *soft_ring;
3796 int count = mac_srs->srs_tx_ring_count;
3797 uint32_t soft_ring_type = ST_RING_TX;
3798 uint_t ring_info;
3799
3800 ASSERT(mac_srs->srs_state & SRS_QUIESCE);
3801 ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring);
3802 if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE))
3803 soft_ring_type |= ST_RING_WORKER_ONLY;
3804 soft_ring = mac_soft_ring_create(count, 0,
3805 soft_ring_type, maxclsyspri, mcip, mac_srs, -1,
3806 NULL, mcip, (mac_resource_handle_t)tx_ring);
3807 mac_srs->srs_tx_ring_count++;
3808 mac_srs_update_fanout_list(mac_srs);
3809 /*
3810 * put this soft ring in quiesce mode too so when we restart
3811 * all soft rings in the srs are in the same state.
3812 */
3813 mac_soft_ring_signal(soft_ring, S_RING_QUIESCE);
3814 }
3815
3816 static void
3817 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring)
3818 {
3819 int sringcnt;
3820
3821 mutex_enter(&mac_srs->srs_lock);
3822 sringcnt = mac_srs->srs_soft_ring_count;
3823 ASSERT(sringcnt > 0);
3824 mac_soft_ring_signal(softring, S_RING_CONDEMNED);
3825
3826 ASSERT(mac_srs->srs_soft_ring_condemned_count == 0);
3827 while (mac_srs->srs_soft_ring_condemned_count != 1)
3828 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3829
3830 if (softring == mac_srs->srs_soft_ring_head) {
3831 mac_srs->srs_soft_ring_head = softring->s_ring_next;
3832 if (mac_srs->srs_soft_ring_head != NULL) {
3833 mac_srs->srs_soft_ring_head->s_ring_prev = NULL;
3834 } else {
3835 mac_srs->srs_soft_ring_tail = NULL;
3836 }
3837 } else {
3838 softring->s_ring_prev->s_ring_next =
3839 softring->s_ring_next;
3840 if (softring->s_ring_next != NULL) {
3841 softring->s_ring_next->s_ring_prev =
3842 softring->s_ring_prev;
3843 } else {
3844 mac_srs->srs_soft_ring_tail =
3845 softring->s_ring_prev;
3846 }
3847 }
3848 mac_srs->srs_soft_ring_count--;
3849
3850 mac_srs->srs_soft_ring_condemned_count--;
3851 mutex_exit(&mac_srs->srs_lock);
3852
3853 mac_soft_ring_free(softring);
3854 }
3855
3856 void
3857 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring)
3858 {
3859 int i;
3860 mac_soft_ring_t *soft_ring, *remove_sring;
3861 mac_client_impl_t *mcip = mac_srs->srs_mcip;
3862
3863 mutex_enter(&mac_srs->srs_lock);
3864 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
3865 soft_ring = mac_srs->srs_tx_soft_rings[i];
3866 if (soft_ring->s_ring_tx_arg2 == tx_ring)
3867 break;
3868 }
3869 mutex_exit(&mac_srs->srs_lock);
3870 ASSERT(i < mac_srs->srs_tx_ring_count);
3871 remove_sring = soft_ring;
3872 /*
3873 * In the case of aggr, the soft ring associated with a Tx ring
3874 * is also stored in st_soft_rings[] array. That entry should
3875 * be removed.
3876 */
3877 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) {
3878 mac_srs_tx_t *tx = &mac_srs->srs_tx;
3879
3880 ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring);
3881 tx->st_soft_rings[tx_ring->mr_index] = NULL;
3882 }
3883 mac_soft_ring_remove(mac_srs, remove_sring);
3884 mac_srs_update_fanout_list(mac_srs);
3885 }
3886
3887 /*
3888 * mac_tx_srs_setup():
3889 * Used to setup Tx rings. If no free Tx ring is available, then default
3890 * Tx ring is used.
3891 */
3892 void
3893 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent)
3894 {
3895 mac_impl_t *mip = mcip->mci_mip;
3896 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs;
3897 int i;
3898 int tx_ring_count = 0;
3899 uint32_t soft_ring_type;
3900 mac_group_t *grp = NULL;
3901 mac_ring_t *ring;
3902 mac_srs_tx_t *tx = &tx_srs->srs_tx;
3903 boolean_t is_aggr;
3904 uint_t ring_info = 0;
3905
3906 is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0;
3907 grp = flent->fe_tx_ring_group;
3908 if (grp == NULL) {
3909 ring = (mac_ring_t *)mip->mi_default_tx_ring;
3910 goto no_group;
3911 }
3912 tx_ring_count = grp->mrg_cur_count;
3913 ring = grp->mrg_rings;
3914 /*
3915 * An attempt is made to reserve 'tx_ring_count' number
3916 * of Tx rings. If tx_ring_count is 0, default Tx ring
3917 * is used. If it is 1, an attempt is made to reserve one
3918 * Tx ring. In both the cases, the ring information is
3919 * stored in Tx SRS. If multiple Tx rings are specified,
3920 * then each Tx ring will have a Tx-side soft ring. All
3921 * these soft rings will be hang off Tx SRS.
3922 */
3923 switch (grp->mrg_state) {
3924 case MAC_GROUP_STATE_SHARED:
3925 case MAC_GROUP_STATE_RESERVED:
3926 if (tx_ring_count <= 1 && !is_aggr) {
3927 no_group:
3928 if (ring != NULL &&
3929 ring->mr_state != MR_INUSE) {
3930 (void) mac_start_ring(ring);
3931 ring_info = mac_hwring_getinfo(
3932 (mac_ring_handle_t)ring);
3933 }
3934 tx->st_arg2 = (void *)ring;
3935 mac_tx_srs_stat_recreate(tx_srs, B_FALSE);
3936 if (tx_srs->srs_type & SRST_BW_CONTROL) {
3937 tx->st_mode = SRS_TX_BW;
3938 } else if (mac_tx_serialize ||
3939 (ring_info & MAC_RING_TX_SERIALIZE)) {
3940 tx->st_mode = SRS_TX_SERIALIZE;
3941 } else {
3942 tx->st_mode = SRS_TX_DEFAULT;
3943 }
3944 break;
3945 }
3946 soft_ring_type = ST_RING_TX;
3947 if (tx_srs->srs_type & SRST_BW_CONTROL) {
3948 tx->st_mode = is_aggr ?
3949 SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT;
3950 } else {
3951 tx->st_mode = is_aggr ? SRS_TX_AGGR :
3952 SRS_TX_FANOUT;
3953 }
3954 for (i = 0; i < tx_ring_count; i++) {
3955 ASSERT(ring != NULL);
3956 switch (ring->mr_state) {
3957 case MR_INUSE:
3958 case MR_FREE:
3959 ASSERT(ring->mr_srs == NULL);
3960
3961 if (ring->mr_state != MR_INUSE)
3962 (void) mac_start_ring(ring);
3963 ring_info = mac_hwring_getinfo(
3964 (mac_ring_handle_t)ring);
3965 if (mac_tx_serialize || (ring_info &
3966 MAC_RING_TX_SERIALIZE)) {
3967 soft_ring_type |=
3968 ST_RING_WORKER_ONLY;
3969 }
3970 (void) mac_soft_ring_create(i, 0,
3971 soft_ring_type, maxclsyspri,
3972 mcip, tx_srs, -1, NULL, mcip,
3973 (mac_resource_handle_t)ring);
3974 break;
3975 default:
3976 cmn_err(CE_PANIC,
3977 "srs_setup: mcip = %p "
3978 "trying to add UNKNOWN ring = %p\n",
3979 (void *)mcip, (void *)ring);
3980 break;
3981 }
3982 ring = ring->mr_next;
3983 }
3984 mac_srs_update_fanout_list(tx_srs);
3985 break;
3986 default:
3987 ASSERT(B_FALSE);
3988 break;
3989 }
3990 tx->st_func = mac_tx_get_func(tx->st_mode);
3991 if (is_aggr) {
3992 VERIFY(i_mac_capab_get((mac_handle_t)mip,
3993 MAC_CAPAB_AGGR, &tx->st_capab_aggr));
3994 }
3995 DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs,
3996 int, tx->st_mode, int, tx_srs->srs_tx_ring_count);
3997 }
3998
3999 /*
4000 * Update the fanout of a client if its recorded link speed doesn't match
4001 * its current link speed.
4002 */
4003 void
4004 mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart)
4005 {
4006 uint64_t link_speed;
4007 mac_resource_props_t *mcip_mrp;
4008 flow_entry_t *flent = mcip->mci_flent;
4009 mac_soft_ring_set_t *rx_srs;
4010 mac_cpus_t *srs_cpu;
4011 int soft_ring_count, maxcpus;
4012
4013 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
4014
4015 link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip,
4016 MAC_STAT_IFSPEED);
4017
4018 if ((link_speed != 0) &&
4019 (link_speed != mcip->mci_flent->fe_nic_speed)) {
4020 mcip_mrp = MCIP_RESOURCE_PROPS(mcip);
4021 /*
4022 * Before calling mac_fanout_setup(), check to see if
4023 * the SRSes already have the right number of soft
4024 * rings. mac_fanout_setup() is a heavy duty operation
4025 * where new cpu bindings are done for SRS and soft
4026 * ring threads and interrupts re-targeted.
4027 */
4028 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus;
4029 soft_ring_count = mac_compute_soft_ring_count(flent,
4030 flent->fe_rx_srs_cnt - 1, maxcpus);
4031 /*
4032 * If soft_ring_count returned by
4033 * mac_compute_soft_ring_count() is 0, bump it
4034 * up by 1 because we always have atleast one
4035 * TCP, UDP, and OTH soft ring associated with
4036 * an SRS.
4037 */
4038 soft_ring_count = (soft_ring_count == 0) ?
4039 1 : soft_ring_count;
4040 rx_srs = flent->fe_rx_srs[0];
4041 srs_cpu = &rx_srs->srs_cpu;
4042 if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) {
4043 mac_fanout_setup(mcip, flent, mcip_mrp,
4044 mac_rx_deliver, mcip, NULL, cpupart);
4045 }
4046 }
4047 }
4048
4049 /*
4050 * Walk through the list of MAC clients for the MAC.
4051 * For each active MAC client, recompute the number of soft rings
4052 * associated with every client, only if current speed is different
4053 * from the speed that was previously used for soft ring computation.
4054 * If the cable is disconnected whlie the NIC is started, we would get
4055 * notification with speed set to 0. We do not recompute in that case.
4056 */
4057 void
4058 mac_fanout_recompute(mac_impl_t *mip)
4059 {
4060 mac_client_impl_t *mcip;
4061 cpupart_t *cpupart;
4062 boolean_t use_default;
4063 mac_resource_props_t *mrp, *emrp;
4064
4065 i_mac_perim_enter(mip);
4066 if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 ||
4067 mip->mi_linkstate != LINK_STATE_UP) {
4068 i_mac_perim_exit(mip);
4069 return;
4070 }
4071
4072 for (mcip = mip->mi_clients_list; mcip != NULL;
4073 mcip = mcip->mci_client_next) {
4074 /* Aggr port clients don't have SRSes. */
4075 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0)
4076 continue;
4077
4078 if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 ||
4079 !MCIP_DATAPATH_SETUP(mcip))
4080 continue;
4081 mrp = MCIP_RESOURCE_PROPS(mcip);
4082 emrp = MCIP_EFFECTIVE_PROPS(mcip);
4083 use_default = B_FALSE;
4084 pool_lock();
4085 cpupart = mac_pset_find(mrp, &use_default);
4086 mac_fanout_recompute_client(mcip, cpupart);
4087 mac_set_pool_effective(use_default, cpupart, mrp, emrp);
4088 pool_unlock();
4089 }
4090
4091 i_mac_perim_exit(mip);
4092 }
4093
4094 /*
4095 * Given a MAC, change the polling state for all its MAC clients. 'enable' is
4096 * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by
4097 * default.
4098 */
4099 void
4100 mac_poll_state_change(mac_handle_t mh, boolean_t enable)
4101 {
4102 mac_impl_t *mip = (mac_impl_t *)mh;
4103 mac_client_impl_t *mcip;
4104
4105 i_mac_perim_enter(mip);
4106 if (enable)
4107 mip->mi_state_flags &= ~MIS_POLL_DISABLE;
4108 else
4109 mip->mi_state_flags |= MIS_POLL_DISABLE;
4110 for (mcip = mip->mi_clients_list; mcip != NULL;
4111 mcip = mcip->mci_client_next)
4112 mac_client_update_classifier(mcip, B_TRUE);
4113 i_mac_perim_exit(mip);
4114 }