Print this page
11490 SRS ring polling disabled for VLANs
11491 Want DLS bypass for VLAN traffic
11492 add VLVF bypass to ixgbe core
2869 duplicate packets with vnics over aggrs
11489 DLS stat delete and aggr kstat can deadlock
Portions contributed by: Theo Schlossnagle <jesus@omniti.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Dan McDonald <danmcd@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/mac/mac_soft_ring.c
+++ new/usr/src/uts/common/io/mac/mac_soft_ring.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 - * Copyright 2017 Joyent, Inc.
24 + * Copyright 2018 Joyent, Inc.
25 25 */
26 26
27 27 /*
28 28 * General Soft rings - Simulating Rx rings in S/W.
29 29 *
30 30 * Soft ring is a data abstraction containing a queue and a worker
31 31 * thread and represents a hardware Rx ring in software. Each soft
32 32 * ring set can have a collection of soft rings for separating
33 33 * L3/L4 specific traffic (IPv4 from IPv6 or TCP from UDP) or for
34 34 * allowing a higher degree of parallelism by sending traffic to
35 35 * one of the soft rings for a SRS (using a hash on src IP or port).
36 36 * Each soft ring worker thread can be bound to a different CPU
37 37 * allowing the processing for each soft ring to happen in parallel
38 38 * and independent from each other.
39 39 *
40 40 * Protocol soft rings:
41 41 *
42 42 * Each SRS has at an minimum 3 softrings. One each for IPv4 TCP,
43 43 * IPv4 UDP and rest (OTH - for IPv6 and everything else). The
44 44 * SRS does dynamic polling and enforces link level bandwidth but
45 45 * it does so for all traffic (IPv4 and IPv6 and all protocols) on
46 46 * that link. However, each protocol layer wants a different
47 47 * behaviour. For instance IPv4 TCP has per CPU squeues which
48 48 * enforce their own polling and flow control so IPv4 TCP traffic
49 49 * needs to go to a separate soft ring which can be polled by the
50 50 * TCP squeue. It also allows TCP squeue to push back flow control
51 51 * all the way to NIC hardware (if it puts its corresponding soft
52 52 * ring in the poll mode and soft ring queue builds up, the
53 53 * shared srs_poll_pkt_cnt goes up and SRS automatically stops
54 54 * more packets from entering the system).
55 55 *
56 56 * Similarly, the UDP benefits from a DLS bypass and packet chaining
57 57 * so sending it to a separate soft ring is desired. All the rest of
58 58 * the traffic (including IPv6 is sent to OTH softring). The IPv6
59 59 * traffic current goes through OTH softring and via DLS because
60 60 * it need more processing to be done. Irrespective of the sap
61 61 * (IPv4 or IPv6) or the transport, the dynamic polling, B/W enforcement,
62 62 * cpu assignment, fanout, etc apply to all traffic since they
63 63 * are implement by the SRS which is agnostic to sap or transport.
64 64 *
65 65 * Fanout soft rings:
66 66 *
67 67 * On a multithreaded system, we can assign more CPU and multi thread
68 68 * the stack by creating a soft ring per CPU and spreading traffic
69 69 * based on a hash computed on src IP etc. Since we still need to
70 70 * keep the protocol separation, we create a set of 3 soft ring per
71 71 * CPU (specified by cpu list or degree of fanout).
72 72 *
73 73 * NOTE: See the block level comment on top of mac_sched.c
74 74 */
75 75
76 76 #include <sys/types.h>
77 77 #include <sys/callb.h>
78 78 #include <sys/sdt.h>
79 79 #include <sys/strsubr.h>
80 80 #include <sys/strsun.h>
81 81 #include <sys/vlan.h>
82 82 #include <inet/ipsec_impl.h>
83 83 #include <inet/ip_impl.h>
84 84 #include <inet/sadb.h>
85 85 #include <inet/ipsecesp.h>
86 86 #include <inet/ipsecah.h>
87 87
88 88 #include <sys/mac_impl.h>
89 89 #include <sys/mac_client_impl.h>
90 90 #include <sys/mac_soft_ring.h>
91 91 #include <sys/mac_flow_impl.h>
92 92 #include <sys/mac_stat.h>
93 93
94 94 static void mac_rx_soft_ring_drain(mac_soft_ring_t *);
95 95 static void mac_soft_ring_fire(void *);
96 96 static void mac_soft_ring_worker(mac_soft_ring_t *);
97 97 static void mac_tx_soft_ring_drain(mac_soft_ring_t *);
98 98
99 99 uint32_t mac_tx_soft_ring_max_q_cnt = 100000;
100 100 uint32_t mac_tx_soft_ring_hiwat = 1000;
101 101
102 102 extern kmem_cache_t *mac_soft_ring_cache;
103 103
104 104 #define ADD_SOFTRING_TO_SET(mac_srs, softring) { \
105 105 if (mac_srs->srs_soft_ring_head == NULL) { \
106 106 mac_srs->srs_soft_ring_head = softring; \
107 107 mac_srs->srs_soft_ring_tail = softring; \
108 108 } else { \
109 109 /* ADD to the list */ \
110 110 softring->s_ring_prev = \
111 111 mac_srs->srs_soft_ring_tail; \
112 112 mac_srs->srs_soft_ring_tail->s_ring_next = softring; \
113 113 mac_srs->srs_soft_ring_tail = softring; \
114 114 } \
115 115 mac_srs->srs_soft_ring_count++; \
116 116 }
117 117
118 118 /*
119 119 * mac_soft_ring_worker_wakeup
120 120 *
121 121 * Wake up the soft ring worker thread to process the queue as long
122 122 * as no one else is processing it and upper layer (client) is still
123 123 * ready to receive packets.
124 124 */
125 125 void
126 126 mac_soft_ring_worker_wakeup(mac_soft_ring_t *ringp)
127 127 {
128 128 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
129 129 if (!(ringp->s_ring_state & S_RING_PROC) &&
130 130 !(ringp->s_ring_state & S_RING_BLANK) &&
131 131 (ringp->s_ring_tid == NULL)) {
132 132 if (ringp->s_ring_wait != 0) {
133 133 ringp->s_ring_tid =
134 134 timeout(mac_soft_ring_fire, ringp,
135 135 ringp->s_ring_wait);
136 136 } else {
137 137 /* Schedule the worker thread. */
138 138 cv_signal(&ringp->s_ring_async);
139 139 }
140 140 }
141 141 }
142 142
143 143 /*
144 144 * mac_soft_ring_create
145 145 *
146 146 * Create a soft ring, do the necessary setup and bind the worker
147 147 * thread to the assigned CPU.
148 148 */
149 149 mac_soft_ring_t *
150 150 mac_soft_ring_create(int id, clock_t wait, uint16_t type,
151 151 pri_t pri, mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs,
152 152 processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1,
153 153 mac_resource_handle_t x_arg2)
154 154 {
155 155 mac_soft_ring_t *ringp;
156 156 char name[S_RING_NAMELEN];
157 157
158 158 bzero(name, 64);
159 159 ringp = kmem_cache_alloc(mac_soft_ring_cache, KM_SLEEP);
160 160
161 161 if (type & ST_RING_TCP) {
162 162 (void) snprintf(name, sizeof (name),
163 163 "mac_tcp_soft_ring_%d_%p", id, (void *)mac_srs);
164 164 } else if (type & ST_RING_UDP) {
165 165 (void) snprintf(name, sizeof (name),
166 166 "mac_udp_soft_ring_%d_%p", id, (void *)mac_srs);
167 167 } else if (type & ST_RING_OTH) {
168 168 (void) snprintf(name, sizeof (name),
169 169 "mac_oth_soft_ring_%d_%p", id, (void *)mac_srs);
170 170 } else {
171 171 ASSERT(type & ST_RING_TX);
172 172 (void) snprintf(name, sizeof (name),
173 173 "mac_tx_soft_ring_%d_%p", id, (void *)mac_srs);
174 174 }
175 175
176 176 bzero(ringp, sizeof (mac_soft_ring_t));
177 177 (void) strncpy(ringp->s_ring_name, name, S_RING_NAMELEN + 1);
178 178 ringp->s_ring_name[S_RING_NAMELEN] = '\0';
179 179 mutex_init(&ringp->s_ring_lock, NULL, MUTEX_DEFAULT, NULL);
180 180 ringp->s_ring_notify_cb_info.mcbi_lockp = &ringp->s_ring_lock;
181 181
182 182 ringp->s_ring_type = type;
183 183 ringp->s_ring_wait = MSEC_TO_TICK(wait);
184 184 ringp->s_ring_mcip = mcip;
185 185 ringp->s_ring_set = mac_srs;
186 186
187 187 /*
188 188 * Protect against access from DR callbacks (mac_walk_srs_bind/unbind)
189 189 * which can't grab the mac perimeter
190 190 */
191 191 mutex_enter(&mac_srs->srs_lock);
192 192 ADD_SOFTRING_TO_SET(mac_srs, ringp);
193 193 mutex_exit(&mac_srs->srs_lock);
194 194
195 195 /*
196 196 * set the bind CPU to -1 to indicate
197 197 * no thread affinity set
198 198 */
199 199 ringp->s_ring_cpuid = ringp->s_ring_cpuid_save = -1;
|
↓ open down ↓ |
165 lines elided |
↑ open up ↑ |
200 200 ringp->s_ring_worker = thread_create(NULL, 0,
201 201 mac_soft_ring_worker, ringp, 0, &p0, TS_RUN, pri);
202 202 if (type & ST_RING_TX) {
203 203 ringp->s_ring_drain_func = mac_tx_soft_ring_drain;
204 204 ringp->s_ring_tx_arg1 = x_arg1;
205 205 ringp->s_ring_tx_arg2 = x_arg2;
206 206 ringp->s_ring_tx_max_q_cnt = mac_tx_soft_ring_max_q_cnt;
207 207 ringp->s_ring_tx_hiwat =
208 208 (mac_tx_soft_ring_hiwat > mac_tx_soft_ring_max_q_cnt) ?
209 209 mac_tx_soft_ring_max_q_cnt : mac_tx_soft_ring_hiwat;
210 - if (mcip->mci_state_flags & MCIS_IS_AGGR) {
210 + if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) {
211 211 mac_srs_tx_t *tx = &mac_srs->srs_tx;
212 212
213 213 ASSERT(tx->st_soft_rings[
214 214 ((mac_ring_t *)x_arg2)->mr_index] == NULL);
215 215 tx->st_soft_rings[((mac_ring_t *)x_arg2)->mr_index] =
216 216 ringp;
217 217 }
218 218 } else {
219 219 ringp->s_ring_drain_func = mac_rx_soft_ring_drain;
220 220 ringp->s_ring_rx_func = rx_func;
221 221 ringp->s_ring_rx_arg1 = x_arg1;
222 222 ringp->s_ring_rx_arg2 = x_arg2;
223 223 if (mac_srs->srs_state & SRS_SOFTRING_QUEUE)
224 224 ringp->s_ring_type |= ST_RING_WORKER_ONLY;
225 225 }
226 226 if (cpuid != -1)
227 227 (void) mac_soft_ring_bind(ringp, cpuid);
228 228
229 229 mac_soft_ring_stat_create(ringp);
230 230
231 231 return (ringp);
232 232 }
233 233
234 234 /*
235 235 * mac_soft_ring_free
236 236 *
237 237 * Free the soft ring once we are done with it.
238 238 */
239 239 void
240 240 mac_soft_ring_free(mac_soft_ring_t *softring)
241 241 {
242 242 ASSERT((softring->s_ring_state &
243 243 (S_RING_CONDEMNED | S_RING_CONDEMNED_DONE | S_RING_PROC)) ==
244 244 (S_RING_CONDEMNED | S_RING_CONDEMNED_DONE));
245 245 mac_pkt_drop(NULL, NULL, softring->s_ring_first, B_FALSE);
246 246 softring->s_ring_tx_arg2 = NULL;
247 247 mac_soft_ring_stat_delete(softring);
248 248 mac_callback_free(softring->s_ring_notify_cb_list);
249 249 kmem_cache_free(mac_soft_ring_cache, softring);
250 250 }
251 251
252 252 int mac_soft_ring_thread_bind = 1;
253 253
254 254 /*
255 255 * mac_soft_ring_bind
256 256 *
257 257 * Bind a soft ring worker thread to supplied CPU.
258 258 */
259 259 cpu_t *
260 260 mac_soft_ring_bind(mac_soft_ring_t *ringp, processorid_t cpuid)
261 261 {
262 262 cpu_t *cp;
263 263 boolean_t clear = B_FALSE;
264 264
265 265 ASSERT(MUTEX_HELD(&cpu_lock));
266 266
267 267 if (mac_soft_ring_thread_bind == 0) {
268 268 DTRACE_PROBE1(mac__soft__ring__no__cpu__bound,
269 269 mac_soft_ring_t *, ringp);
270 270 return (NULL);
271 271 }
272 272
273 273 cp = cpu_get(cpuid);
274 274 if (cp == NULL || !cpu_is_online(cp))
275 275 return (NULL);
276 276
277 277 mutex_enter(&ringp->s_ring_lock);
278 278 ringp->s_ring_state |= S_RING_BOUND;
279 279 if (ringp->s_ring_cpuid != -1)
280 280 clear = B_TRUE;
281 281 ringp->s_ring_cpuid = cpuid;
282 282 mutex_exit(&ringp->s_ring_lock);
283 283
284 284 if (clear)
285 285 thread_affinity_clear(ringp->s_ring_worker);
286 286
287 287 DTRACE_PROBE2(mac__soft__ring__cpu__bound, mac_soft_ring_t *,
288 288 ringp, processorid_t, cpuid);
289 289
290 290 thread_affinity_set(ringp->s_ring_worker, cpuid);
291 291
292 292 return (cp);
293 293 }
294 294
295 295 /*
296 296 * mac_soft_ring_unbind
297 297 *
298 298 * Un Bind a soft ring worker thread.
299 299 */
300 300 void
301 301 mac_soft_ring_unbind(mac_soft_ring_t *ringp)
302 302 {
303 303 ASSERT(MUTEX_HELD(&cpu_lock));
304 304
305 305 mutex_enter(&ringp->s_ring_lock);
306 306 if (!(ringp->s_ring_state & S_RING_BOUND)) {
307 307 ASSERT(ringp->s_ring_cpuid == -1);
308 308 mutex_exit(&ringp->s_ring_lock);
309 309 return;
310 310 }
311 311
312 312 ringp->s_ring_cpuid = -1;
313 313 ringp->s_ring_state &= ~S_RING_BOUND;
314 314 thread_affinity_clear(ringp->s_ring_worker);
315 315 mutex_exit(&ringp->s_ring_lock);
316 316 }
317 317
318 318 /*
319 319 * PRIVATE FUNCTIONS
320 320 */
321 321
322 322 static void
323 323 mac_soft_ring_fire(void *arg)
324 324 {
325 325 mac_soft_ring_t *ringp = arg;
326 326
327 327 mutex_enter(&ringp->s_ring_lock);
328 328 if (ringp->s_ring_tid == NULL) {
329 329 mutex_exit(&ringp->s_ring_lock);
330 330 return;
331 331 }
|
↓ open down ↓ |
111 lines elided |
↑ open up ↑ |
332 332
333 333 ringp->s_ring_tid = NULL;
334 334
335 335 if (!(ringp->s_ring_state & S_RING_PROC)) {
336 336 cv_signal(&ringp->s_ring_async);
337 337 }
338 338 mutex_exit(&ringp->s_ring_lock);
339 339 }
340 340
341 341 /*
342 - * mac_rx_soft_ring_drain
342 + * Drain the soft ring pointed to by ringp.
343 343 *
344 - * Called when worker thread model (ST_RING_WORKER_ONLY) of processing
345 - * incoming packets is used. s_ring_first contain the queued packets.
346 - * s_ring_rx_func contains the upper level (client) routine where the
347 - * packets are destined and s_ring_rx_arg1/s_ring_rx_arg2 are the
348 - * cookie meant for the client.
344 + * o s_ring_first: pointer to the queued packet chain.
345 + *
346 + * o s_ring_rx_func: pointer to to the client's Rx routine.
347 + *
348 + * o s_ring_rx_{arg1,arg2}: opaque values specific to the client.
349 349 */
350 -/* ARGSUSED */
351 350 static void
352 351 mac_rx_soft_ring_drain(mac_soft_ring_t *ringp)
353 352 {
354 353 mblk_t *mp;
355 354 void *arg1;
356 355 mac_resource_handle_t arg2;
357 356 timeout_id_t tid;
358 357 mac_direct_rx_t proc;
359 358 size_t sz;
360 359 int cnt;
361 360 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
362 361
363 362 ringp->s_ring_run = curthread;
364 363 ASSERT(mutex_owned(&ringp->s_ring_lock));
365 364 ASSERT(!(ringp->s_ring_state & S_RING_PROC));
366 365
367 366 if ((tid = ringp->s_ring_tid) != NULL)
368 367 ringp->s_ring_tid = NULL;
369 368
370 369 ringp->s_ring_state |= S_RING_PROC;
371 370
372 371 proc = ringp->s_ring_rx_func;
373 372 arg1 = ringp->s_ring_rx_arg1;
374 373 arg2 = ringp->s_ring_rx_arg2;
375 374
376 375 while ((ringp->s_ring_first != NULL) &&
377 376 !(ringp->s_ring_state & S_RING_PAUSE)) {
378 377 mp = ringp->s_ring_first;
379 378 ringp->s_ring_first = NULL;
380 379 ringp->s_ring_last = NULL;
381 380 cnt = ringp->s_ring_count;
382 381 ringp->s_ring_count = 0;
383 382 sz = ringp->s_ring_size;
384 383 ringp->s_ring_size = 0;
|
↓ open down ↓ |
24 lines elided |
↑ open up ↑ |
385 384 mutex_exit(&ringp->s_ring_lock);
386 385
387 386 if (tid != NULL) {
388 387 (void) untimeout(tid);
389 388 tid = NULL;
390 389 }
391 390
392 391 (*proc)(arg1, arg2, mp, NULL);
393 392
394 393 /*
395 - * If we have a soft ring set which is doing
396 - * bandwidth control, we need to decrement its
397 - * srs_size so it can have a accurate idea of
398 - * what is the real data queued between SRS and
399 - * its soft rings. We decrement the size for a
400 - * packet only when it gets processed by both
401 - * SRS and the soft ring.
394 + * If we have an SRS performing bandwidth control, then
395 + * we need to decrement the size and count so the SRS
396 + * has an accurate measure of the data queued between
397 + * the SRS and its soft rings. We decrement the
398 + * counters only when the packet is processed by both
399 + * the SRS and the soft ring.
402 400 */
403 401 mutex_enter(&mac_srs->srs_lock);
404 402 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
405 403 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
406 404 mutex_exit(&mac_srs->srs_lock);
407 405
408 406 mutex_enter(&ringp->s_ring_lock);
409 407 }
410 408 ringp->s_ring_state &= ~S_RING_PROC;
411 409 if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
412 410 cv_signal(&ringp->s_ring_client_cv);
413 411 ringp->s_ring_run = NULL;
414 412 }
415 413
416 414 /*
417 - * mac_soft_ring_worker
418 - *
419 415 * The soft ring worker routine to process any queued packets. In
420 - * normal case, the worker thread is bound to a CPU. It the soft
421 - * ring is dealing with TCP packets, then the worker thread will
422 - * be bound to the same CPU as the TCP squeue.
416 + * normal case, the worker thread is bound to a CPU. If the soft ring
417 + * handles TCP packets then the worker thread is bound to the same CPU
418 + * as the TCP squeue.
423 419 */
424 420 static void
425 421 mac_soft_ring_worker(mac_soft_ring_t *ringp)
426 422 {
427 423 kmutex_t *lock = &ringp->s_ring_lock;
428 424 kcondvar_t *async = &ringp->s_ring_async;
429 425 mac_soft_ring_set_t *srs = ringp->s_ring_set;
430 426 callb_cpr_t cprinfo;
431 427
432 428 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_soft_ring");
433 429 mutex_enter(lock);
434 430 start:
435 431 for (;;) {
436 432 while (((ringp->s_ring_first == NULL ||
437 433 (ringp->s_ring_state & (S_RING_BLOCK|S_RING_BLANK))) &&
438 434 !(ringp->s_ring_state & S_RING_PAUSE)) ||
439 435 (ringp->s_ring_state & S_RING_PROC)) {
440 436
441 437 CALLB_CPR_SAFE_BEGIN(&cprinfo);
442 438 cv_wait(async, lock);
443 439 CALLB_CPR_SAFE_END(&cprinfo, lock);
444 440 }
445 441
446 442 /*
447 443 * Either we have work to do, or we have been asked to
448 444 * shutdown temporarily or permanently
449 445 */
450 446 if (ringp->s_ring_state & S_RING_PAUSE)
451 447 goto done;
452 448
453 449 ringp->s_ring_drain_func(ringp);
454 450 }
455 451 done:
456 452 mutex_exit(lock);
457 453 mutex_enter(&srs->srs_lock);
458 454 mutex_enter(lock);
459 455
460 456 ringp->s_ring_state |= S_RING_QUIESCE_DONE;
461 457 if (!(ringp->s_ring_state & S_RING_CONDEMNED)) {
462 458 srs->srs_soft_ring_quiesced_count++;
463 459 cv_broadcast(&srs->srs_async);
464 460 mutex_exit(&srs->srs_lock);
465 461 while (!(ringp->s_ring_state &
466 462 (S_RING_RESTART | S_RING_CONDEMNED)))
467 463 cv_wait(&ringp->s_ring_async, &ringp->s_ring_lock);
468 464 mutex_exit(lock);
469 465 mutex_enter(&srs->srs_lock);
470 466 mutex_enter(lock);
471 467 srs->srs_soft_ring_quiesced_count--;
472 468 if (ringp->s_ring_state & S_RING_RESTART) {
473 469 ASSERT(!(ringp->s_ring_state & S_RING_CONDEMNED));
474 470 ringp->s_ring_state &= ~(S_RING_RESTART |
475 471 S_RING_QUIESCE | S_RING_QUIESCE_DONE);
476 472 cv_broadcast(&srs->srs_async);
477 473 mutex_exit(&srs->srs_lock);
478 474 goto start;
479 475 }
480 476 }
481 477 ASSERT(ringp->s_ring_state & S_RING_CONDEMNED);
482 478 ringp->s_ring_state |= S_RING_CONDEMNED_DONE;
483 479 CALLB_CPR_EXIT(&cprinfo);
484 480 srs->srs_soft_ring_condemned_count++;
485 481 cv_broadcast(&srs->srs_async);
486 482 mutex_exit(&srs->srs_lock);
487 483 thread_exit();
488 484 }
489 485
490 486 /*
491 487 * mac_soft_ring_intr_enable and mac_soft_ring_intr_disable
492 488 *
493 489 * these functions are called to toggle the sending of packets to the
494 490 * client. They are called by the client. the client gets the name
495 491 * of these routine and corresponding cookie (pointing to softring)
496 492 * during capability negotiation at setup time.
497 493 *
498 494 * Enabling is allow the processing thread to send packets to the
499 495 * client while disabling does the opposite.
500 496 */
501 497 int
502 498 mac_soft_ring_intr_enable(void *arg)
503 499 {
504 500 mac_soft_ring_t *ringp = (mac_soft_ring_t *)arg;
505 501 mutex_enter(&ringp->s_ring_lock);
506 502 ringp->s_ring_state &= ~S_RING_BLANK;
507 503 if (ringp->s_ring_first != NULL)
508 504 mac_soft_ring_worker_wakeup(ringp);
509 505 mutex_exit(&ringp->s_ring_lock);
510 506 return (0);
511 507 }
512 508
513 509 boolean_t
514 510 mac_soft_ring_intr_disable(void *arg)
515 511 {
516 512 mac_soft_ring_t *ringp = (mac_soft_ring_t *)arg;
517 513 boolean_t sring_blanked = B_FALSE;
518 514 /*
519 515 * Stop worker thread from sending packets above.
520 516 * Squeue will poll soft ring when it needs packets.
521 517 */
522 518 mutex_enter(&ringp->s_ring_lock);
523 519 if (!(ringp->s_ring_state & S_RING_PROC)) {
524 520 ringp->s_ring_state |= S_RING_BLANK;
525 521 sring_blanked = B_TRUE;
526 522 }
527 523 mutex_exit(&ringp->s_ring_lock);
528 524 return (sring_blanked);
529 525 }
530 526
531 527 /*
532 528 * mac_soft_ring_poll
533 529 *
534 530 * This routine is called by the client to poll for packets from
535 531 * the soft ring. The function name and cookie corresponding to
536 532 * the soft ring is exchanged during capability negotiation during
537 533 * setup.
538 534 */
539 535 mblk_t *
540 536 mac_soft_ring_poll(mac_soft_ring_t *ringp, size_t bytes_to_pickup)
541 537 {
542 538 mblk_t *head, *tail;
543 539 mblk_t *mp;
544 540 size_t sz = 0;
545 541 int cnt = 0;
546 542 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
547 543
548 544 ASSERT(mac_srs != NULL);
549 545
550 546 mutex_enter(&ringp->s_ring_lock);
551 547 head = tail = mp = ringp->s_ring_first;
552 548 if (head == NULL) {
553 549 mutex_exit(&ringp->s_ring_lock);
554 550 return (NULL);
555 551 }
556 552
557 553 if (ringp->s_ring_size <= bytes_to_pickup) {
558 554 head = ringp->s_ring_first;
559 555 ringp->s_ring_first = NULL;
560 556 ringp->s_ring_last = NULL;
561 557 cnt = ringp->s_ring_count;
562 558 ringp->s_ring_count = 0;
563 559 sz = ringp->s_ring_size;
564 560 ringp->s_ring_size = 0;
565 561 } else {
566 562 while (mp && sz <= bytes_to_pickup) {
567 563 sz += msgdsize(mp);
568 564 cnt++;
569 565 tail = mp;
570 566 mp = mp->b_next;
571 567 }
572 568 ringp->s_ring_count -= cnt;
573 569 ringp->s_ring_size -= sz;
574 570 tail->b_next = NULL;
575 571 if (mp == NULL) {
576 572 ringp->s_ring_first = NULL;
577 573 ringp->s_ring_last = NULL;
578 574 ASSERT(ringp->s_ring_count == 0);
579 575 } else {
580 576 ringp->s_ring_first = mp;
581 577 }
582 578 }
583 579
584 580 mutex_exit(&ringp->s_ring_lock);
585 581 /*
586 582 * Update the shared count and size counters so
587 583 * that SRS has a accurate idea of queued packets.
588 584 */
589 585 mutex_enter(&mac_srs->srs_lock);
590 586 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
591 587 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
592 588 mutex_exit(&mac_srs->srs_lock);
593 589 return (head);
594 590 }
595 591
596 592 /*
597 593 * mac_soft_ring_dls_bypass
|
↓ open down ↓ |
165 lines elided |
↑ open up ↑ |
598 594 *
599 595 * Enable direct client (IP) callback function from the softrings.
600 596 * Callers need to make sure they don't need any DLS layer processing
601 597 */
602 598 void
603 599 mac_soft_ring_dls_bypass(void *arg, mac_direct_rx_t rx_func, void *rx_arg1)
604 600 {
605 601 mac_soft_ring_t *softring = arg;
606 602 mac_soft_ring_set_t *srs;
607 603
608 - ASSERT(rx_func != NULL);
604 + VERIFY3P(rx_func, !=, NULL);
609 605
610 606 mutex_enter(&softring->s_ring_lock);
611 607 softring->s_ring_rx_func = rx_func;
612 608 softring->s_ring_rx_arg1 = rx_arg1;
613 609 mutex_exit(&softring->s_ring_lock);
614 610
615 611 srs = softring->s_ring_set;
616 612 mutex_enter(&srs->srs_lock);
617 613 srs->srs_type |= SRST_DLS_BYPASS;
618 614 mutex_exit(&srs->srs_lock);
619 615 }
620 616
621 617 /*
622 618 * mac_soft_ring_signal
623 619 *
624 620 * Typically used to set the soft ring state to QUIESCE, CONDEMNED, or
625 621 * RESTART.
626 622 *
627 623 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls
628 624 * from the driver are done, then the Rx SRS is quiesced and only then can
629 625 * we signal the soft rings. Thus this function can't be called arbitrarily
630 626 * without satisfying the prerequisites. On the Tx side, the threads from
631 627 * top need to quiesced, then the Tx SRS and only then can we signal the
632 628 * Tx soft rings.
633 629 */
634 630 void
635 631 mac_soft_ring_signal(mac_soft_ring_t *softring, uint_t sr_flag)
636 632 {
637 633 mutex_enter(&softring->s_ring_lock);
638 634 softring->s_ring_state |= sr_flag;
639 635 cv_signal(&softring->s_ring_async);
640 636 mutex_exit(&softring->s_ring_lock);
641 637 }
642 638
643 639 /*
644 640 * mac_tx_soft_ring_drain
645 641 *
646 642 * The transmit side drain routine in case the soft ring was being
647 643 * used to transmit packets.
648 644 */
649 645 static void
650 646 mac_tx_soft_ring_drain(mac_soft_ring_t *ringp)
651 647 {
652 648 mblk_t *mp;
653 649 void *arg1;
654 650 void *arg2;
655 651 mblk_t *tail;
656 652 uint_t saved_pkt_count, saved_size;
657 653 mac_tx_stats_t stats;
658 654 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
659 655
660 656 saved_pkt_count = saved_size = 0;
661 657 ringp->s_ring_run = curthread;
662 658 ASSERT(mutex_owned(&ringp->s_ring_lock));
663 659 ASSERT(!(ringp->s_ring_state & S_RING_PROC));
664 660
665 661 ringp->s_ring_state |= S_RING_PROC;
666 662 arg1 = ringp->s_ring_tx_arg1;
667 663 arg2 = ringp->s_ring_tx_arg2;
668 664
669 665 while (ringp->s_ring_first != NULL) {
670 666 mp = ringp->s_ring_first;
671 667 tail = ringp->s_ring_last;
672 668 saved_pkt_count = ringp->s_ring_count;
673 669 saved_size = ringp->s_ring_size;
674 670 ringp->s_ring_first = NULL;
675 671 ringp->s_ring_last = NULL;
676 672 ringp->s_ring_count = 0;
677 673 ringp->s_ring_size = 0;
678 674 mutex_exit(&ringp->s_ring_lock);
679 675
680 676 mp = mac_tx_send(arg1, arg2, mp, &stats);
681 677
682 678 mutex_enter(&ringp->s_ring_lock);
683 679 if (mp != NULL) {
684 680 /* Device out of tx desc, set block */
685 681 tail->b_next = ringp->s_ring_first;
686 682 ringp->s_ring_first = mp;
687 683 ringp->s_ring_count +=
688 684 (saved_pkt_count - stats.mts_opackets);
689 685 ringp->s_ring_size += (saved_size - stats.mts_obytes);
690 686 if (ringp->s_ring_last == NULL)
691 687 ringp->s_ring_last = tail;
692 688
693 689 if (ringp->s_ring_tx_woken_up) {
694 690 ringp->s_ring_tx_woken_up = B_FALSE;
695 691 } else {
696 692 ringp->s_ring_state |= S_RING_BLOCK;
697 693 ringp->s_st_stat.mts_blockcnt++;
698 694 }
699 695
700 696 ringp->s_ring_state &= ~S_RING_PROC;
701 697 ringp->s_ring_run = NULL;
702 698 return;
703 699 } else {
704 700 ringp->s_ring_tx_woken_up = B_FALSE;
705 701 SRS_TX_STATS_UPDATE(mac_srs, &stats);
706 702 SOFTRING_TX_STATS_UPDATE(ringp, &stats);
707 703 }
708 704 }
709 705
710 706 if (ringp->s_ring_count == 0 && ringp->s_ring_state &
711 707 (S_RING_TX_HIWAT | S_RING_WAKEUP_CLIENT | S_RING_ENQUEUED)) {
712 708 mac_client_impl_t *mcip = ringp->s_ring_mcip;
713 709 boolean_t wakeup_required = B_FALSE;
714 710
715 711 if (ringp->s_ring_state &
716 712 (S_RING_TX_HIWAT|S_RING_WAKEUP_CLIENT)) {
717 713 wakeup_required = B_TRUE;
718 714 }
719 715 ringp->s_ring_state &=
720 716 ~(S_RING_TX_HIWAT | S_RING_WAKEUP_CLIENT | S_RING_ENQUEUED);
721 717 mutex_exit(&ringp->s_ring_lock);
722 718 if (wakeup_required) {
723 719 mac_tx_invoke_callbacks(mcip, (mac_tx_cookie_t)ringp);
724 720 /*
725 721 * If the client is not the primary MAC client, then we
726 722 * need to send the notification to the clients upper
727 723 * MAC, i.e. mci_upper_mip.
728 724 */
729 725 mac_tx_notify(mcip->mci_upper_mip != NULL ?
730 726 mcip->mci_upper_mip : mcip->mci_mip);
731 727 }
732 728 mutex_enter(&ringp->s_ring_lock);
733 729 }
734 730 ringp->s_ring_state &= ~S_RING_PROC;
735 731 ringp->s_ring_run = NULL;
736 732 }
|
↓ open down ↓ |
118 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX