4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2017 Joyent, Inc.
25 */
26
27 /*
28 * General Soft rings - Simulating Rx rings in S/W.
29 *
30 * Soft ring is a data abstraction containing a queue and a worker
31 * thread and represents a hardware Rx ring in software. Each soft
32 * ring set can have a collection of soft rings for separating
33 * L3/L4 specific traffic (IPv4 from IPv6 or TCP from UDP) or for
34 * allowing a higher degree of parallelism by sending traffic to
35 * one of the soft rings for a SRS (using a hash on src IP or port).
36 * Each soft ring worker thread can be bound to a different CPU
37 * allowing the processing for each soft ring to happen in parallel
38 * and independent from each other.
39 *
40 * Protocol soft rings:
41 *
42 * Each SRS has at an minimum 3 softrings. One each for IPv4 TCP,
43 * IPv4 UDP and rest (OTH - for IPv6 and everything else). The
44 * SRS does dynamic polling and enforces link level bandwidth but
190 */
191 mutex_enter(&mac_srs->srs_lock);
192 ADD_SOFTRING_TO_SET(mac_srs, ringp);
193 mutex_exit(&mac_srs->srs_lock);
194
195 /*
196 * set the bind CPU to -1 to indicate
197 * no thread affinity set
198 */
199 ringp->s_ring_cpuid = ringp->s_ring_cpuid_save = -1;
200 ringp->s_ring_worker = thread_create(NULL, 0,
201 mac_soft_ring_worker, ringp, 0, &p0, TS_RUN, pri);
202 if (type & ST_RING_TX) {
203 ringp->s_ring_drain_func = mac_tx_soft_ring_drain;
204 ringp->s_ring_tx_arg1 = x_arg1;
205 ringp->s_ring_tx_arg2 = x_arg2;
206 ringp->s_ring_tx_max_q_cnt = mac_tx_soft_ring_max_q_cnt;
207 ringp->s_ring_tx_hiwat =
208 (mac_tx_soft_ring_hiwat > mac_tx_soft_ring_max_q_cnt) ?
209 mac_tx_soft_ring_max_q_cnt : mac_tx_soft_ring_hiwat;
210 if (mcip->mci_state_flags & MCIS_IS_AGGR) {
211 mac_srs_tx_t *tx = &mac_srs->srs_tx;
212
213 ASSERT(tx->st_soft_rings[
214 ((mac_ring_t *)x_arg2)->mr_index] == NULL);
215 tx->st_soft_rings[((mac_ring_t *)x_arg2)->mr_index] =
216 ringp;
217 }
218 } else {
219 ringp->s_ring_drain_func = mac_rx_soft_ring_drain;
220 ringp->s_ring_rx_func = rx_func;
221 ringp->s_ring_rx_arg1 = x_arg1;
222 ringp->s_ring_rx_arg2 = x_arg2;
223 if (mac_srs->srs_state & SRS_SOFTRING_QUEUE)
224 ringp->s_ring_type |= ST_RING_WORKER_ONLY;
225 }
226 if (cpuid != -1)
227 (void) mac_soft_ring_bind(ringp, cpuid);
228
229 mac_soft_ring_stat_create(ringp);
230
322 static void
323 mac_soft_ring_fire(void *arg)
324 {
325 mac_soft_ring_t *ringp = arg;
326
327 mutex_enter(&ringp->s_ring_lock);
328 if (ringp->s_ring_tid == NULL) {
329 mutex_exit(&ringp->s_ring_lock);
330 return;
331 }
332
333 ringp->s_ring_tid = NULL;
334
335 if (!(ringp->s_ring_state & S_RING_PROC)) {
336 cv_signal(&ringp->s_ring_async);
337 }
338 mutex_exit(&ringp->s_ring_lock);
339 }
340
341 /*
342 * mac_rx_soft_ring_drain
343 *
344 * Called when worker thread model (ST_RING_WORKER_ONLY) of processing
345 * incoming packets is used. s_ring_first contain the queued packets.
346 * s_ring_rx_func contains the upper level (client) routine where the
347 * packets are destined and s_ring_rx_arg1/s_ring_rx_arg2 are the
348 * cookie meant for the client.
349 */
350 /* ARGSUSED */
351 static void
352 mac_rx_soft_ring_drain(mac_soft_ring_t *ringp)
353 {
354 mblk_t *mp;
355 void *arg1;
356 mac_resource_handle_t arg2;
357 timeout_id_t tid;
358 mac_direct_rx_t proc;
359 size_t sz;
360 int cnt;
361 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
362
363 ringp->s_ring_run = curthread;
364 ASSERT(mutex_owned(&ringp->s_ring_lock));
365 ASSERT(!(ringp->s_ring_state & S_RING_PROC));
366
367 if ((tid = ringp->s_ring_tid) != NULL)
368 ringp->s_ring_tid = NULL;
369
370 ringp->s_ring_state |= S_RING_PROC;
375
376 while ((ringp->s_ring_first != NULL) &&
377 !(ringp->s_ring_state & S_RING_PAUSE)) {
378 mp = ringp->s_ring_first;
379 ringp->s_ring_first = NULL;
380 ringp->s_ring_last = NULL;
381 cnt = ringp->s_ring_count;
382 ringp->s_ring_count = 0;
383 sz = ringp->s_ring_size;
384 ringp->s_ring_size = 0;
385 mutex_exit(&ringp->s_ring_lock);
386
387 if (tid != NULL) {
388 (void) untimeout(tid);
389 tid = NULL;
390 }
391
392 (*proc)(arg1, arg2, mp, NULL);
393
394 /*
395 * If we have a soft ring set which is doing
396 * bandwidth control, we need to decrement its
397 * srs_size so it can have a accurate idea of
398 * what is the real data queued between SRS and
399 * its soft rings. We decrement the size for a
400 * packet only when it gets processed by both
401 * SRS and the soft ring.
402 */
403 mutex_enter(&mac_srs->srs_lock);
404 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
405 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
406 mutex_exit(&mac_srs->srs_lock);
407
408 mutex_enter(&ringp->s_ring_lock);
409 }
410 ringp->s_ring_state &= ~S_RING_PROC;
411 if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
412 cv_signal(&ringp->s_ring_client_cv);
413 ringp->s_ring_run = NULL;
414 }
415
416 /*
417 * mac_soft_ring_worker
418 *
419 * The soft ring worker routine to process any queued packets. In
420 * normal case, the worker thread is bound to a CPU. It the soft
421 * ring is dealing with TCP packets, then the worker thread will
422 * be bound to the same CPU as the TCP squeue.
423 */
424 static void
425 mac_soft_ring_worker(mac_soft_ring_t *ringp)
426 {
427 kmutex_t *lock = &ringp->s_ring_lock;
428 kcondvar_t *async = &ringp->s_ring_async;
429 mac_soft_ring_set_t *srs = ringp->s_ring_set;
430 callb_cpr_t cprinfo;
431
432 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_soft_ring");
433 mutex_enter(lock);
434 start:
435 for (;;) {
436 while (((ringp->s_ring_first == NULL ||
437 (ringp->s_ring_state & (S_RING_BLOCK|S_RING_BLANK))) &&
438 !(ringp->s_ring_state & S_RING_PAUSE)) ||
439 (ringp->s_ring_state & S_RING_PROC)) {
440
441 CALLB_CPR_SAFE_BEGIN(&cprinfo);
442 cv_wait(async, lock);
588 */
589 mutex_enter(&mac_srs->srs_lock);
590 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
591 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
592 mutex_exit(&mac_srs->srs_lock);
593 return (head);
594 }
595
596 /*
597 * mac_soft_ring_dls_bypass
598 *
599 * Enable direct client (IP) callback function from the softrings.
600 * Callers need to make sure they don't need any DLS layer processing
601 */
602 void
603 mac_soft_ring_dls_bypass(void *arg, mac_direct_rx_t rx_func, void *rx_arg1)
604 {
605 mac_soft_ring_t *softring = arg;
606 mac_soft_ring_set_t *srs;
607
608 ASSERT(rx_func != NULL);
609
610 mutex_enter(&softring->s_ring_lock);
611 softring->s_ring_rx_func = rx_func;
612 softring->s_ring_rx_arg1 = rx_arg1;
613 mutex_exit(&softring->s_ring_lock);
614
615 srs = softring->s_ring_set;
616 mutex_enter(&srs->srs_lock);
617 srs->srs_type |= SRST_DLS_BYPASS;
618 mutex_exit(&srs->srs_lock);
619 }
620
621 /*
622 * mac_soft_ring_signal
623 *
624 * Typically used to set the soft ring state to QUIESCE, CONDEMNED, or
625 * RESTART.
626 *
627 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls
628 * from the driver are done, then the Rx SRS is quiesced and only then can
|
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2018 Joyent, Inc.
25 */
26
27 /*
28 * General Soft rings - Simulating Rx rings in S/W.
29 *
30 * Soft ring is a data abstraction containing a queue and a worker
31 * thread and represents a hardware Rx ring in software. Each soft
32 * ring set can have a collection of soft rings for separating
33 * L3/L4 specific traffic (IPv4 from IPv6 or TCP from UDP) or for
34 * allowing a higher degree of parallelism by sending traffic to
35 * one of the soft rings for a SRS (using a hash on src IP or port).
36 * Each soft ring worker thread can be bound to a different CPU
37 * allowing the processing for each soft ring to happen in parallel
38 * and independent from each other.
39 *
40 * Protocol soft rings:
41 *
42 * Each SRS has at an minimum 3 softrings. One each for IPv4 TCP,
43 * IPv4 UDP and rest (OTH - for IPv6 and everything else). The
44 * SRS does dynamic polling and enforces link level bandwidth but
190 */
191 mutex_enter(&mac_srs->srs_lock);
192 ADD_SOFTRING_TO_SET(mac_srs, ringp);
193 mutex_exit(&mac_srs->srs_lock);
194
195 /*
196 * set the bind CPU to -1 to indicate
197 * no thread affinity set
198 */
199 ringp->s_ring_cpuid = ringp->s_ring_cpuid_save = -1;
200 ringp->s_ring_worker = thread_create(NULL, 0,
201 mac_soft_ring_worker, ringp, 0, &p0, TS_RUN, pri);
202 if (type & ST_RING_TX) {
203 ringp->s_ring_drain_func = mac_tx_soft_ring_drain;
204 ringp->s_ring_tx_arg1 = x_arg1;
205 ringp->s_ring_tx_arg2 = x_arg2;
206 ringp->s_ring_tx_max_q_cnt = mac_tx_soft_ring_max_q_cnt;
207 ringp->s_ring_tx_hiwat =
208 (mac_tx_soft_ring_hiwat > mac_tx_soft_ring_max_q_cnt) ?
209 mac_tx_soft_ring_max_q_cnt : mac_tx_soft_ring_hiwat;
210 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) {
211 mac_srs_tx_t *tx = &mac_srs->srs_tx;
212
213 ASSERT(tx->st_soft_rings[
214 ((mac_ring_t *)x_arg2)->mr_index] == NULL);
215 tx->st_soft_rings[((mac_ring_t *)x_arg2)->mr_index] =
216 ringp;
217 }
218 } else {
219 ringp->s_ring_drain_func = mac_rx_soft_ring_drain;
220 ringp->s_ring_rx_func = rx_func;
221 ringp->s_ring_rx_arg1 = x_arg1;
222 ringp->s_ring_rx_arg2 = x_arg2;
223 if (mac_srs->srs_state & SRS_SOFTRING_QUEUE)
224 ringp->s_ring_type |= ST_RING_WORKER_ONLY;
225 }
226 if (cpuid != -1)
227 (void) mac_soft_ring_bind(ringp, cpuid);
228
229 mac_soft_ring_stat_create(ringp);
230
322 static void
323 mac_soft_ring_fire(void *arg)
324 {
325 mac_soft_ring_t *ringp = arg;
326
327 mutex_enter(&ringp->s_ring_lock);
328 if (ringp->s_ring_tid == NULL) {
329 mutex_exit(&ringp->s_ring_lock);
330 return;
331 }
332
333 ringp->s_ring_tid = NULL;
334
335 if (!(ringp->s_ring_state & S_RING_PROC)) {
336 cv_signal(&ringp->s_ring_async);
337 }
338 mutex_exit(&ringp->s_ring_lock);
339 }
340
341 /*
342 * Drain the soft ring pointed to by ringp.
343 *
344 * o s_ring_first: pointer to the queued packet chain.
345 *
346 * o s_ring_rx_func: pointer to to the client's Rx routine.
347 *
348 * o s_ring_rx_{arg1,arg2}: opaque values specific to the client.
349 */
350 static void
351 mac_rx_soft_ring_drain(mac_soft_ring_t *ringp)
352 {
353 mblk_t *mp;
354 void *arg1;
355 mac_resource_handle_t arg2;
356 timeout_id_t tid;
357 mac_direct_rx_t proc;
358 size_t sz;
359 int cnt;
360 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
361
362 ringp->s_ring_run = curthread;
363 ASSERT(mutex_owned(&ringp->s_ring_lock));
364 ASSERT(!(ringp->s_ring_state & S_RING_PROC));
365
366 if ((tid = ringp->s_ring_tid) != NULL)
367 ringp->s_ring_tid = NULL;
368
369 ringp->s_ring_state |= S_RING_PROC;
374
375 while ((ringp->s_ring_first != NULL) &&
376 !(ringp->s_ring_state & S_RING_PAUSE)) {
377 mp = ringp->s_ring_first;
378 ringp->s_ring_first = NULL;
379 ringp->s_ring_last = NULL;
380 cnt = ringp->s_ring_count;
381 ringp->s_ring_count = 0;
382 sz = ringp->s_ring_size;
383 ringp->s_ring_size = 0;
384 mutex_exit(&ringp->s_ring_lock);
385
386 if (tid != NULL) {
387 (void) untimeout(tid);
388 tid = NULL;
389 }
390
391 (*proc)(arg1, arg2, mp, NULL);
392
393 /*
394 * If we have an SRS performing bandwidth control, then
395 * we need to decrement the size and count so the SRS
396 * has an accurate measure of the data queued between
397 * the SRS and its soft rings. We decrement the
398 * counters only when the packet is processed by both
399 * the SRS and the soft ring.
400 */
401 mutex_enter(&mac_srs->srs_lock);
402 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
403 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
404 mutex_exit(&mac_srs->srs_lock);
405
406 mutex_enter(&ringp->s_ring_lock);
407 }
408 ringp->s_ring_state &= ~S_RING_PROC;
409 if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
410 cv_signal(&ringp->s_ring_client_cv);
411 ringp->s_ring_run = NULL;
412 }
413
414 /*
415 * The soft ring worker routine to process any queued packets. In
416 * normal case, the worker thread is bound to a CPU. If the soft ring
417 * handles TCP packets then the worker thread is bound to the same CPU
418 * as the TCP squeue.
419 */
420 static void
421 mac_soft_ring_worker(mac_soft_ring_t *ringp)
422 {
423 kmutex_t *lock = &ringp->s_ring_lock;
424 kcondvar_t *async = &ringp->s_ring_async;
425 mac_soft_ring_set_t *srs = ringp->s_ring_set;
426 callb_cpr_t cprinfo;
427
428 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_soft_ring");
429 mutex_enter(lock);
430 start:
431 for (;;) {
432 while (((ringp->s_ring_first == NULL ||
433 (ringp->s_ring_state & (S_RING_BLOCK|S_RING_BLANK))) &&
434 !(ringp->s_ring_state & S_RING_PAUSE)) ||
435 (ringp->s_ring_state & S_RING_PROC)) {
436
437 CALLB_CPR_SAFE_BEGIN(&cprinfo);
438 cv_wait(async, lock);
584 */
585 mutex_enter(&mac_srs->srs_lock);
586 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
587 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
588 mutex_exit(&mac_srs->srs_lock);
589 return (head);
590 }
591
592 /*
593 * mac_soft_ring_dls_bypass
594 *
595 * Enable direct client (IP) callback function from the softrings.
596 * Callers need to make sure they don't need any DLS layer processing
597 */
598 void
599 mac_soft_ring_dls_bypass(void *arg, mac_direct_rx_t rx_func, void *rx_arg1)
600 {
601 mac_soft_ring_t *softring = arg;
602 mac_soft_ring_set_t *srs;
603
604 VERIFY3P(rx_func, !=, NULL);
605
606 mutex_enter(&softring->s_ring_lock);
607 softring->s_ring_rx_func = rx_func;
608 softring->s_ring_rx_arg1 = rx_arg1;
609 mutex_exit(&softring->s_ring_lock);
610
611 srs = softring->s_ring_set;
612 mutex_enter(&srs->srs_lock);
613 srs->srs_type |= SRST_DLS_BYPASS;
614 mutex_exit(&srs->srs_lock);
615 }
616
617 /*
618 * mac_soft_ring_signal
619 *
620 * Typically used to set the soft ring state to QUIESCE, CONDEMNED, or
621 * RESTART.
622 *
623 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls
624 * from the driver are done, then the Rx SRS is quiesced and only then can
|