1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2018 Joyent, Inc.
25 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 /*
29 * MAC data path
30 *
31 * The MAC data path is concerned with the flow of traffic from mac clients --
32 * DLS, IP, etc. -- to various GLDv3 device drivers -- e1000g, vnic, aggr,
33 * ixgbe, etc. -- and from the GLDv3 device drivers back to clients.
34 *
35 * -----------
36 * Terminology
37 * -----------
38 *
39 * MAC uses a lot of different, but related terms that are associated with the
40 * design and structure of the data path. Before we cover other aspects, first
41 * let's review the terminology that MAC uses.
42 *
43 * MAC
44 *
45 * This driver. It interfaces with device drivers and provides abstractions
46 * that the rest of the system consumes. All data links -- things managed
47 * with dladm(1M), are accessed through MAC.
48 *
49 * GLDv3 DEVICE DRIVER
50 *
51 * A GLDv3 device driver refers to a driver, both for pseudo-devices and
52 * real devices, which implement the GLDv3 driver API. Common examples of
53 * these are igb and ixgbe, which are drivers for various Intel networking
54 * cards. These devices may or may not have various features, such as
55 * hardware rings and checksum offloading. For MAC, a GLDv3 device is the
56 * final point for the transmission of a packet and the starting point for
57 * the receipt of a packet.
58 *
59 * FLOWS
60 *
61 * At a high level, a flow refers to a series of packets that are related.
62 * Often times the term is used in the context of TCP to indicate a unique
63 * TCP connection and the traffic over it. However, a flow can exist at
64 * other levels of the system as well. MAC has a notion of a default flow
65 * which is used for all unicast traffic addressed to the address of a MAC
66 * device. For example, when a VNIC is created, a default flow is created
67 * for the VNIC's MAC address. In addition, flows are created for broadcast
68 * groups and a user may create a flow with flowadm(1M).
69 *
70 * CLASSIFICATION
71 *
72 * Classification refers to the notion of identifying an incoming frame
73 * based on its destination address and optionally its source addresses and
74 * doing different processing based on that information. Classification can
75 * be done in both hardware and software. In general, we usually only
76 * classify based on the layer two destination, eg. for Ethernet, the
77 * destination MAC address.
78 *
79 * The system also will do classification based on layer three and layer
80 * four properties. This is used to support things like flowadm(1M), which
81 * allows setting QoS and other properties on a per-flow basis.
82 *
83 * RING
84 *
85 * Conceptually, a ring represents a series of framed messages, often in a
86 * contiguous chunk of memory that acts as a circular buffer. Rings come in
87 * a couple of forms. Generally they are either a hardware construct (hw
88 * ring) or they are a software construct (sw ring) maintained by MAC.
89 *
90 * HW RING
91 *
92 * A hardware ring is a set of resources provided by a GLDv3 device driver
93 * (even if it is a pseudo-device). A hardware ring comes in two different
94 * forms: receive (rx) rings and transmit (tx) rings. An rx hw ring is
95 * something that has a unique DMA (direct memory access) region and
96 * generally supports some form of classification (though it isn't always
97 * used), as well as a means of generating an interrupt specific to that
98 * ring. For example, the device may generate a specific MSI-X for a PCI
99 * express device. A tx ring is similar, except that it is dedicated to
100 * transmission. It may also be a vector for enabling features such as VLAN
101 * tagging and large transmit offloading. It usually has its own dedicated
102 * interrupts for transmit being completed.
103 *
104 * SW RING
105 *
106 * A software ring is a construction of MAC. It represents the same thing
107 * that a hardware ring generally does, a collection of frames. However,
108 * instead of being in a contiguous ring of memory, they're instead linked
109 * by using the mblk_t's b_next pointer. Each frame may itself be multiple
110 * mblk_t's linked together by the b_cont pointer. A software ring always
111 * represents a collection of classified packets; however, it varies as to
112 * whether it uses only layer two information, or a combination of that and
113 * additional layer three and layer four data.
114 *
115 * FANOUT
116 *
117 * Fanout is the idea of spreading out the load of processing frames based
118 * on the source and destination information contained in the layer two,
119 * three, and four headers, such that the data can then be processed in
120 * parallel using multiple hardware threads.
121 *
122 * A fanout algorithm hashes the headers and uses that to place different
123 * flows into a bucket. The most important thing is that packets that are
124 * in the same flow end up in the same bucket. If they do not, performance
125 * can be adversely affected. Consider the case of TCP. TCP severely
126 * penalizes a connection if the data arrives out of order. If a given flow
127 * is processed on different CPUs, then the data will appear out of order,
128 * hence the invariant that fanout always hash a given flow to the same
129 * bucket and thus get processed on the same CPU.
130 *
131 * RECEIVE SIDE SCALING (RSS)
132 *
133 *
134 * Receive side scaling is a term that isn't common in illumos, but is used
135 * by vendors and was popularized by Microsoft. It refers to the idea of
136 * spreading the incoming receive load out across multiple interrupts which
137 * can be directed to different CPUs. This allows a device to leverage
138 * hardware rings even when it doesn't support hardware classification. The
139 * hardware uses an algorithm to perform fanout that ensures the flow
140 * invariant is maintained.
141 *
142 * SOFT RING SET
143 *
144 * A soft ring set, commonly abbreviated SRS, is a collection of rings and
145 * is used for both transmitting and receiving. It is maintained in the
146 * structure mac_soft_ring_set_t. A soft ring set is usually associated
147 * with flows, and coordinates both the use of hardware and software rings.
148 * Because the use of hardware rings can change as devices such as VNICs
149 * come and go, we always ensure that the set has software classification
150 * rules that correspond to the hardware classification rules from rings.
151 *
152 * Soft ring sets are also used for the enforcement of various QoS
153 * properties. For example, if a bandwidth limit has been placed on a
154 * specific flow or device, then that will be enforced by the soft ring
155 * set.
156 *
157 * SERVICE ATTACHMENT POINT (SAP)
158 *
159 * The service attachment point is a DLPI (Data Link Provider Interface)
160 * concept; however, it comes up quite often in MAC. Most MAC devices speak
161 * a protocol that has some notion of different channels or message type
162 * identifiers. For example, Ethernet defines an EtherType which is a part
163 * of the Ethernet header and defines the particular protocol of the data
164 * payload. If the EtherType is set to 0x0800, then it defines that the
165 * contents of that Ethernet frame is IPv4 traffic. For Ethernet, the
166 * EtherType is the SAP.
167 *
168 * In DLPI, a given consumer attaches to a specific SAP. In illumos, the ip
169 * and arp drivers attach to the EtherTypes for IPv4, IPv6, and ARP. Using
170 * libdlpi(3LIB) user software can attach to arbitrary SAPs. With the
171 * exception of 802.1Q VLAN tagged traffic, MAC itself does not directly
172 * consume the SAP; however, it uses that information as part of hashing
173 * and it may be used as part of the construction of flows.
174 *
175 * PRIMARY MAC CLIENT
176 *
177 * The primary mac client refers to a mac client whose unicast address
178 * matches the address of the device itself. For example, if the system has
179 * instance of the e1000g driver such as e1000g0, e1000g1, etc., the
180 * primary mac client is the one named after the device itself. VNICs that
181 * are created on top of such devices are not the primary client.
182 *
183 * TRANSMIT DESCRIPTORS
184 *
185 * Transmit descriptors are a resource that most GLDv3 device drivers have.
186 * Generally, a GLDv3 device driver takes a frame that's meant to be output
187 * and puts a copy of it into a region of memory. Each region of memory
188 * usually has an associated descriptor that the device uses to manage
189 * properties of the frames. Devices have a limited number of such
190 * descriptors. They get reclaimed once the device finishes putting the
191 * frame on the wire.
192 *
193 * If the driver runs out of transmit descriptors, for example, the OS is
194 * generating more frames than it can put on the wire, then it will return
195 * them back to the MAC layer.
196 *
197 * ---------------------------------
198 * Rings, Classification, and Fanout
199 * ---------------------------------
200 *
201 * The heart of MAC is made up of rings, and not those that Elven-kings wear.
202 * When receiving a packet, MAC breaks the work into two different, though
203 * interrelated phases. The first phase is generally classification and then the
204 * second phase is generally fanout. When a frame comes in from a GLDv3 Device,
205 * MAC needs to determine where that frame should be delivered. If it's a
206 * unicast frame (say a normal TCP/IP packet), then it will be delivered to a
207 * single MAC client; however, if it's a broadcast or multicast frame, then MAC
208 * may need to deliver it to multiple MAC clients.
209 *
210 * On transmit, classification isn't quite as important, but may still be used.
211 * Unlike with the receive path, the classification is not used to determine
212 * devices that should transmit something, but rather is used for special
213 * properties of a flow, eg. bandwidth limits for a given IP address, device, or
214 * connection.
215 *
216 * MAC employs a software classifier and leverages hardware classification as
217 * well. The software classifier can leverage the full layer two information,
218 * source, destination, VLAN, and SAP. If the SAP indicates that IP traffic is
219 * being sent, it can classify based on the IP header, and finally, it also
220 * knows how to classify based on the local and remote ports of TCP, UDP, and
221 * SCTP.
222 *
223 * Hardware classifiers vary in capability. Generally all hardware classifiers
224 * provide the capability to classify based on the destination MAC address. Some
225 * hardware has additional filters built in for performing more in-depth
226 * classification; however, it often has much more limited resources for these
227 * activities as compared to the layer two destination address classification.
228 *
229 * The modus operandi in MAC is to always ensure that we have software-based
230 * capabilities and rules in place and then to supplement that with hardware
231 * resources when available. In general, simple layer two classification is
232 * sufficient and nothing else is used, unless a specific flow is created with
233 * tools such as flowadm(1M) or bandwidth limits are set on a device with
234 * dladm(1M).
235 *
236 * RINGS AND GROUPS
237 *
238 * To get into how rings and classification play together, it's first important
239 * to understand how hardware devices commonly associate rings and allow them to
240 * be programmed. Recall that a hardware ring should be thought of as a DMA
241 * buffer and an interrupt resource. Rings are then collected into groups. A
242 * group itself has a series of classification rules. One or more MAC addresses
243 * are assigned to a group.
244 *
245 * Hardware devices vary in terms of what capabilities they provide. Sometimes
246 * they allow for a dynamic assignment of rings to a group and sometimes they
247 * have a static assignment of rings to a group. For example, the ixgbe driver
248 * has a static assignment of rings to groups such that every group has exactly
249 * one ring and the number of groups is equal to the number of rings.
250 *
251 * Classification and receive side scaling both come into play with how a device
252 * advertises itself to MAC and how MAC uses it. If a device supports layer two
253 * classification of frames, then MAC will assign MAC addresses to a group as a
254 * form of primary classification. If a single MAC address is assigned to a
255 * group, a common case, then MAC will consider packets that come in from rings
256 * on that group to be fully classified and will not need to do any software
257 * classification unless a specific flow has been created.
258 *
259 * If a device supports receive side scaling, then it may advertise or support
260 * groups with multiple rings. In those cases, then receive side scaling will
261 * come into play and MAC will use that as a means of fanning out received
262 * frames across multiple CPUs. This can also be combined with groups that
263 * support layer two classification.
264 *
265 * If a device supports dynamic assignments of rings to groups, then MAC will
266 * change around the way that rings are assigned to various groups as devices
267 * come and go from the system. For example, when a VNIC is created, a new flow
268 * will be created for the VNIC's MAC address. If a hardware ring is available,
269 * MAC may opt to reassign it from one group to another.
270 *
271 * ASSIGNMENT OF HARDWARE RINGS
272 *
273 * This is a bit of a complicated subject that varies depending on the device,
274 * the use of aggregations, the special nature of the primary mac client. This
275 * section deserves being fleshed out.
276 *
277 * FANOUT
278 *
279 * illumos uses fanout to help spread out the incoming processing load of chains
280 * of frames away from a single CPU. If a device supports receive side scaling,
281 * then that provides an initial form of fanout; however, what we're concerned
282 * with all happens after the context of a given set of frames being classified
283 * to a soft ring set.
284 *
285 * After frames reach a soft ring set and account for any potential bandwidth
286 * related accounting, they may be fanned out based on one of the following
287 * three modes:
288 *
289 * o No Fanout
290 * o Protocol level fanout
291 * o Full software ring protocol fanout
292 *
293 * MAC makes the determination as to which of these modes a given soft ring set
294 * obtains based on parameters such as whether or not it's the primary mac
295 * client, whether it's on a 10 GbE or faster device, user controlled dladm(1M)
296 * properties, and the nature of the hardware and the resources that it has.
297 *
298 * When there is no fanout, MAC does not create any soft rings for a device and
299 * the device has frames delivered directly to the MAC client.
300 *
301 * Otherwise, all fanout is performed by software. MAC divides incoming frames
302 * into one of three buckets -- IPv4 TCP traffic, IPv4 UDP traffic, and
303 * everything else. Regardless of the type of fanout, these three categories
304 * or buckets are always used.
305 *
306 * The difference between protocol level fanout and full software ring protocol
307 * fanout is the number of software rings that end up getting created. The
308 * system always uses the same number of software rings per protocol bucket. So
309 * in the first case when we're just doing protocol level fanout, we just create
310 * one software ring each for IPv4 TCP traffic, IPv4 UDP traffic, and everything
311 * else.
312 *
313 * In the case where we do full software ring protocol fanout, we generally use
314 * mac_compute_soft_ring_count() to determine the number of rings. There are
315 * other combinations of properties and devices that may send us down other
316 * paths, but this is a common starting point. If it's a non-bandwidth enforced
317 * device and we're on at least a 10 GbE link, then we'll use eight soft rings
318 * per protocol bucket as a starting point. See mac_compute_soft_ring_count()
319 * for more information on the total number.
320 *
321 * For each of these rings, we create a mac_soft_ring_t and an associated worker
322 * thread. Particularly when doing full software ring protocol fanout, we bind
323 * each of the worker threads to individual CPUs.
324 *
325 * The other advantage of these software rings is that it allows upper layers to
326 * optionally poll on them. For example, TCP can leverage an squeue to poll on
327 * the software ring, see squeue.c for more information.
328 *
329 * DLS BYPASS
330 *
331 * DLS is the data link services module. It interfaces with DLPI, which is the
332 * primary way that other parts of the system such as IP interface with the MAC
333 * layer. While DLS is traditionally a STREAMS-based interface, it allows for
334 * certain modules such as IP to negotiate various more modern interfaces to be
335 * used, which are useful for higher performance and allow it to use direct
336 * function calls to DLS instead of using STREAMS.
337 *
338 * When we have IPv4 TCP or UDP software rings, then traffic on those rings is
339 * eligible for what we call the dls bypass. In those cases, rather than going
340 * out mac_rx_deliver() to DLS, DLS instead registers them to go directly via
341 * the direct callback registered with DLS, generally ip_input().
342 *
343 * HARDWARE RING POLLING
344 *
345 * GLDv3 devices with hardware rings generally deliver chains of messages
346 * (mblk_t chain) during the context of a single interrupt. However, interrupts
347 * are not the only way that these devices may be used. As part of implementing
348 * ring support, a GLDv3 device driver must have a way to disable the generation
349 * of that interrupt and allow for the operating system to poll on that ring.
350 *
351 * To implement this, every soft ring set has a worker thread and a polling
352 * thread. If a sufficient packet rate comes into the system, MAC will 'blank'
353 * (disable) interrupts on that specific ring and the polling thread will start
354 * consuming packets from the hardware device and deliver them to the soft ring
355 * set, where the worker thread will take over.
356 *
357 * Once the rate of packet intake drops down below a certain threshold, then
358 * polling on the hardware ring will be quiesced and interrupts will be
359 * re-enabled for the given ring. This effectively allows the system to shift
360 * how it handles a ring based on its load. At high packet rates, polling on the
361 * device as opposed to relying on interrupts can actually reduce overall system
362 * load due to the minimization of interrupt activity.
363 *
364 * Note the importance of each ring having its own interrupt source. The whole
365 * idea here is that we do not disable interrupts on the device as a whole, but
366 * rather each ring can be independently toggled.
367 *
368 * USE OF WORKER THREADS
369 *
370 * Both the soft ring set and individual soft rings have a worker thread
371 * associated with them that may be bound to a specific CPU in the system. Any
372 * such assignment will get reassessed as part of dynamic reconfiguration events
373 * in the system such as the onlining and offlining of CPUs and the creation of
374 * CPU partitions.
375 *
376 * In many cases, while in an interrupt, we try to deliver a frame all the way
377 * through the stack in the context of the interrupt itself. However, if the
378 * amount of queued frames has exceeded a threshold, then we instead defer to
379 * the worker thread to do this work and signal it. This is particularly useful
380 * when you have the soft ring set delivering frames into multiple software
381 * rings. If it was only delivering frames into a single software ring then
382 * there'd be no need to have another thread take over. However, if it's
383 * delivering chains of frames to multiple rings, then it's worthwhile to have
384 * the worker for the software ring take over so that the different software
385 * rings can be processed in parallel.
386 *
387 * In a similar fashion to the hardware polling thread, if we don't have a
388 * backlog or there's nothing to do, then the worker thread will go back to
389 * sleep and frames can be delivered all the way from an interrupt. This
390 * behavior is useful as it's designed to minimize latency and the default
391 * disposition of MAC is to optimize for latency.
392 *
393 * MAINTAINING CHAINS
394 *
395 * Another useful idea that MAC uses is to try and maintain frames in chains for
396 * as long as possible. The idea is that all of MAC can handle chains of frames
397 * structured as a series of mblk_t structures linked with the b_next pointer.
398 * When performing software classification and software fanout, MAC does not
399 * simply determine the destination and send the frame along. Instead, in the
400 * case of classification, it tries to maintain a chain for as long as possible
401 * before passing it along and performing additional processing.
402 *
403 * In the case of fanout, MAC first determines what the target software ring is
404 * for every frame in the original chain and constructs a new chain for each
405 * target. MAC then delivers the new chain to each software ring in succession.
406 *
407 * The whole rationale for doing this is that we want to try and maintain the
408 * pipe as much as possible and deliver as many frames through the stack at once
409 * that we can, rather than just pushing a single frame through. This can often
410 * help bring down latency and allows MAC to get a better sense of the overall
411 * activity in the system and properly engage worker threads.
412 *
413 * --------------------
414 * Bandwidth Management
415 * --------------------
416 *
417 * Bandwidth management is something that's built into the soft ring set itself.
418 * When bandwidth limits are placed on a flow, a corresponding soft ring set is
419 * toggled into bandwidth mode. This changes how we transmit and receive the
420 * frames in question.
421 *
422 * Bandwidth management is done on a per-tick basis. We translate the user's
423 * requested bandwidth from a quantity per-second into a quantity per-tick. MAC
424 * cannot process a frame across more than one tick, thus it sets a lower bound
425 * for the bandwidth cap to be a single MTU. This also means that when
426 * hires ticks are enabled (hz is set to 1000), that the minimum amount of
427 * bandwidth is higher, because the number of ticks has increased and MAC has to
428 * go from accepting 100 packets / sec to 1000 / sec.
429 *
430 * The bandwidth counter is reset by either the soft ring set's worker thread or
431 * a thread that is doing an inline transmit or receive if they discover that
432 * the current tick is in the future from the recorded tick.
433 *
434 * Whenever we're receiving or transmitting data, we end up leaving most of the
435 * work to the soft ring set's worker thread. This forces data inserted into the
436 * soft ring set to be effectively serialized and allows us to exhume bandwidth
437 * at a reasonable rate. If there is nothing in the soft ring set at the moment
438 * and the set has available bandwidth, then it may processed inline.
439 * Otherwise, the worker is responsible for taking care of the soft ring set.
440 *
441 * ---------------------
442 * The Receive Data Path
443 * ---------------------
444 *
445 * The following series of ASCII art images breaks apart the way that a frame
446 * comes in and is processed in MAC.
447 *
448 * Part 1 -- Initial frame receipt, SRS classification
449 *
450 * Here, a frame is received by a GLDv3 driver, generally in the context of an
451 * interrupt, and it ends up in mac_rx_common(). A driver calls either mac_rx or
452 * mac_rx_ring, depending on whether or not it supports rings and can identify
453 * the interrupt as having come from a specific ring. Here we determine whether
454 * or not it's fully classified and perform software classification as
455 * appropriate. From here, everything always ends up going to either entry [A]
456 * or entry [B] based on whether or not they have subflow processing needed. We
457 * leave via fanout or delivery.
458 *
459 * +===========+
460 * v hardware v
461 * v interrupt v
462 * +===========+
463 * |
464 * * . . appropriate
465 * | upcall made
466 * | by GLDv3 driver . . always
467 * | .
468 * +--------+ | +----------+ . +---------------+
469 * | GLDv3 | +---->| mac_rx |-----*--->| mac_rx_common |
470 * | Driver |-->--+ +----------+ +---------------+
471 * +--------+ | ^ |
472 * | | ^ v
473 * ^ | * . . always +----------------------+
474 * | | | | mac_promisc_dispatch |
475 * | | +-------------+ +----------------------+
476 * | +--->| mac_rx_ring | |
477 * | +-------------+ * . . hw classified
478 * | v or single flow?
479 * | |
480 * | +--------++--------------+
481 * | | | * hw class,
482 * | | * hw classified | subflows
483 * | no hw class and . * | or single | exist
484 * | subflows | | flow |
485 * | | v v
486 * | | +-----------+ +-----------+
487 * | | | goto | | goto |
488 * | | | entry [A] | | entry [B] |
489 * | | +-----------+ +-----------+
490 * | v ^
491 * | +-------------+ |
492 * | | mac_rx_flow | * SRS and flow found,
493 * | +-------------+ | call flow cb
494 * | | +------+
495 * | v |
496 * v +==========+ +-----------------+
497 * | v For each v--->| mac_rx_classify |
498 * +----------+ v mblk_t v +-----------------+
499 * | srs | +==========+
500 * | pollling |
501 * | thread |->------------------------------------------+
502 * +----------+ |
503 * v . inline
504 * +--------------------+ +----------+ +---------+ .
505 * [A]---->| mac_rx_srs_process |-->| check bw |-->| enqueue |--*---------+
506 * +--------------------+ | limits | | frames | |
507 * ^ +----------+ | to SRS | |
508 * | +---------+ |
509 * | send chain +--------+ | |
510 * * when clasified | signal | * BW limits, |
511 * | flow changes | srs |<---+ loopback, |
512 * | | worker | stack too |
513 * | +--------+ deep |
514 * +-----------------+ +--------+ |
515 * | mac_flow_lookup | | srs | +---------------------+ |
516 * +-----------------+ | worker |---->| mac_rx_srs_drain |<---+
517 * ^ | thread | | mac_rx_srs_drain_bw |
518 * | +--------+ +---------------------+
519 * | |
520 * +----------------------------+ * software rings
521 * [B]-->| mac_rx_srs_subflow_process | | for fanout?
522 * +----------------------------+ |
523 * +----------+-----------+
524 * | |
525 * v v
526 * +--------+ +--------+
527 * | goto | | goto |
528 * | Part 2 | | Part 3 |
529 * +--------+ +--------+
530 *
531 * Part 2 -- Fanout
532 *
533 * This part is concerned with using software fanout to assign frames to
534 * software rings and then deliver them to MAC clients or allow those rings to
535 * be polled upon. While there are two different primary fanout entry points,
536 * mac_rx_fanout and mac_rx_proto_fanout, they behave in similar ways, and aside
537 * from some of the individual hashing techniques used, most of the general
538 * flow is the same.
539 *
540 * +--------+ +-------------------+
541 * | From |---+--------->| mac_rx_srs_fanout |----+
542 * | Part 1 | | +-------------------+ | +=================+
543 * +--------+ | | v for each mblk_t v
544 * * . . protocol only +--->v assign to new v
545 * | fanout | v chain based on v
546 * | | v hash % nrings v
547 * | +-------------------------+ | +=================+
548 * +--->| mac_rx_srs_proto_fanout |----+ |
549 * +-------------------------+ |
550 * v
551 * +------------+ +--------------------------+ +================+
552 * | enqueue in |<---| mac_rx_soft_ring_process |<------v for each chain v
553 * | soft ring | +--------------------------+ +================+
554 * +------------+
555 * | +-----------+
556 * * soft ring set | soft ring |
557 * | empty and no | worker |
558 * | worker? | thread |
559 * | +-----------+
560 * +------*----------------+ |
561 * | . | v
562 * No . * . Yes | +------------------------+
563 * | +----<--| mac_rx_soft_ring_drain |
564 * | | +------------------------+
565 * v |
566 * +-----------+ v
567 * | signal | +---------------+
568 * | soft ring | | Deliver chain |
569 * | worker | | goto Part 3 |
570 * +-----------+ +---------------+
571 *
572 *
573 * Part 3 -- Packet Delivery
574 *
575 * Here, we go through and deliver the mblk_t chain directly to a given
576 * processing function. In a lot of cases this is mac_rx_deliver(). In the case
577 * of DLS bypass being used, then instead we end up going ahead and deliver it
578 * to the direct callback registered with DLS, generally ip_input.
579 *
580 *
581 * +---------+ +----------------+ +------------------+
582 * | From |---+------->| mac_rx_deliver |--->| Off to DLS, or |
583 * | Parts 1 | | +----------------+ | other MAC client |
584 * | and 2 | * DLS bypass +------------------+
585 * +---------+ | enabled +----------+ +-------------+
586 * +---------->| ip_input |--->| To IP |
587 * +----------+ | and beyond! |
588 * +-------------+
589 *
590 * ----------------------
591 * The Transmit Data Path
592 * ----------------------
593 *
594 * Before we go into the images, it's worth talking about a problem that is a
595 * bit different from the receive data path. GLDv3 device drivers have a finite
596 * amount of transmit descriptors. When they run out, they return unused frames
597 * back to MAC. MAC, at this point has several options about what it will do,
598 * which vary based upon the settings that the client uses.
599 *
600 * When a device runs out of descriptors, the next thing that MAC does is
601 * enqueue them off of the soft ring set or a software ring, depending on the
602 * configuration of the soft ring set. MAC will enqueue up to a high watermark
603 * of mblk_t chains, at which point it will indicate flow control back to the
604 * client. Once this condition is reached, any mblk_t chains that were not
605 * enqueued will be returned to the caller and they will have to decide what to
606 * do with them. There are various flags that control this behavior that a
607 * client may pass, which are discussed below.
608 *
609 * When this condition is hit, MAC also returns a cookie to the client in
610 * addition to unconsumed frames. Clients can poll on that cookie and register a
611 * callback with MAC to be notified when they are no longer subject to flow
612 * control, at which point they may continue to call mac_tx(). This flow control
613 * actually manages to work itself all the way up the stack, back through dls,
614 * to ip, through the various protocols, and to sockfs.
615 *
616 * While the behavior described above is the default, this behavior can be
617 * modified. There are two alternate modes, described below, which are
618 * controlled with flags.
619 *
620 * DROP MODE
621 *
622 * This mode is controlled by having the client pass the MAC_DROP_ON_NO_DESC
623 * flag. When this is passed, if a device driver runs out of transmit
624 * descriptors, then the MAC layer will drop any unsent traffic. The client in
625 * this case will never have any frames returned to it.
626 *
627 * DON'T ENQUEUE
628 *
629 * This mode is controlled by having the client pass the MAC_TX_NO_ENQUEUE flag.
630 * If the MAC_DROP_ON_NO_DESC flag is also passed, it takes precedence. In this
631 * mode, when we hit a case where a driver runs out of transmit descriptors,
632 * then instead of enqueuing packets in a soft ring set or software ring, we
633 * instead return the mblk_t chain back to the caller and immediately put the
634 * soft ring set into flow control mode.
635 *
636 * The following series of ASCII art images describe the transmit data path that
637 * MAC clients enter into based on calling into mac_tx(). A soft ring set has a
638 * transmission function associated with it. There are seven possible
639 * transmission modes, some of which share function entry points. The one that a
640 * soft ring set gets depends on properties such as whether there are
641 * transmission rings for fanout, whether the device involves aggregations,
642 * whether any bandwidth limits exist, etc.
643 *
644 *
645 * Part 1 -- Initial checks
646 *
647 * * . called by
648 * | MAC clients
649 * v . . No
650 * +--------+ +-----------+ . +-------------------+ +====================+
651 * | mac_tx |->| device |-*-->| mac_protect_check |->v Is this the simple v
652 * +--------+ | quiesced? | +-------------------+ v case? See [1] v
653 * +-----------+ | +====================+
654 * * . Yes * failed |
655 * v | frames |
656 * +--------------+ | +-------+---------+
657 * | freemsgchain |<---------+ Yes . * No . *
658 * +--------------+ v v
659 * +-----------+ +--------+
660 * | goto | | goto |
661 * | Part 2 | | SRS TX |
662 * | Entry [A] | | func |
663 * +-----------+ +--------+
664 * | |
665 * | v
666 * | +--------+
667 * +---------->| return |
668 * | cookie |
669 * +--------+
670 *
671 * [1] The simple case refers to the SRS being configured with the
672 * SRS_TX_DEFAULT transmission mode, having a single mblk_t (not a chain), their
673 * being only a single active client, and not having a backlog in the srs.
674 *
675 *
676 * Part 2 -- The SRS transmission functions
677 *
678 * This part is a bit more complicated. The different transmission paths often
679 * leverage one another. In this case, we'll draw out the more common ones
680 * before the parts that depend upon them. Here, we're going to start with the
681 * workings of mac_tx_send() a common function that most of the others end up
682 * calling.
683 *
684 * +-------------+
685 * | mac_tx_send |
686 * +-------------+
687 * |
688 * v
689 * +=============+ +==============+
690 * v more than v--->v check v
691 * v one client? v v VLAN and add v
692 * +=============+ v VLAN tags v
693 * | +==============+
694 * | |
695 * +------------------+
696 * |
697 * | [A]
698 * v |
699 * +============+ . No v
700 * v more than v . +==========+ +--------------------------+
701 * v one active v-*---->v for each v---->| mac_promisc_dispatch_one |---+
702 * v client? v v mblk_t v +--------------------------+ |
703 * +============+ +==========+ ^ |
704 * | | +==========+ |
705 * * . Yes | v hardware v<-------+
706 * v +------------+ v rings? v
707 * +==========+ | +==========+
708 * v for each v No . . . * |
709 * v mblk_t v specific | |
710 * +==========+ flow | +-----+-----+
711 * | | | |
712 * v | v v
713 * +-----------------+ | +-------+ +---------+
714 * | mac_tx_classify |------------+ | GLDv3 | | GLDv3 |
715 * +-----------------+ |TX func| | ring tx |
716 * | +-------+ | func |
717 * * Specific flow, generally | +---------+
718 * | bcast, mcast, loopback | |
719 * v +-----+-----+
720 * +==========+ +---------+ |
721 * v valid L2 v--*--->| freemsg | v
722 * v header v . No +---------+ +-------------------+
723 * +==========+ | return unconsumed |
724 * * . Yes | frames to the |
725 * v | caller |
726 * +===========+ +-------------------+
727 * v braodcast v +----------------+ ^
728 * v flow? v--*-->| mac_bcast_send |------------------+
729 * +===========+ . +----------------+ |
730 * | . . Yes |
731 * No . * v
732 * | +---------------------+ +---------------+ +----------+
733 * +->|mac_promisc_dispatch |->| mac_fix_cksum |->| flow |
734 * +---------------------+ +---------------+ | callback |
735 * +----------+
736 *
737 *
738 * In addition, many but not all of the routines, all rely on
739 * mac_tx_softring_process as an entry point.
740 *
741 *
742 * . No . No
743 * +--------------------------+ +========+ . +===========+ . +-------------+
744 * | mac_tx_soft_ring_process |-->v worker v-*->v out of tx v-*->| goto |
745 * +--------------------------+ v only? v v descr.? v | mac_tx_send |
746 * +========+ +===========+ +-------------+
747 * Yes . * * . Yes |
748 * . No v | v
749 * v=========+ . +===========+ . Yes | Yes . +==========+
750 * v apppend v<--*----------v out of tx v-*-------+---------*--v returned v
751 * v mblk_t v v descr.? v | v frames? v
752 * v chain v +===========+ | +==========+
753 * +=========+ | *. No
754 * | | v
755 * v v +------------+
756 * +===================+ +----------------------+ | done |
757 * v worker scheduled? v | mac_tx_sring_enqueue | | processing |
758 * v Out of tx descr? v +----------------------+ +------------+
759 * +===================+ |
760 * | | . Yes v
761 * * Yes * No . +============+
762 * | v +-*---------v drop on no v
763 * | +========+ v v TX desc? v
764 * | v wake v +----------+ +============+
765 * | v worker v | mac_pkt_ | * . No
766 * | +========+ | drop | | . Yes . No
767 * | | +----------+ v . .
768 * | | v ^ +===============+ . +========+ .
769 * +--+--------+---------+ | v Don't enqueue v-*->v ring v-*----+
770 * | | v Set? v v empty? v |
771 * | +---------------+ +===============+ +========+ |
772 * | | | | |
773 * | | +-------------------+ | |
774 * | *. Yes | +---------+ |
775 * | | v v v
776 * | | +===========+ +========+ +--------------+
777 * | +<-v At hiwat? v v append v | return |
778 * | +===========+ v mblk_t v | mblk_t chain |
779 * | * No v chain v | and flow |
780 * | v +========+ | control |
781 * | +=========+ | | cookie |
782 * | v append v v +--------------+
783 * | v mblk_t v +========+
784 * | v chain v v wake v +------------+
785 * | +=========+ v worker v-->| done |
786 * | | +========+ | processing |
787 * | v .. Yes +------------+
788 * | +=========+ . +========+
789 * | v first v--*-->v wake v
790 * | v append? v v worker v
791 * | +=========+ +========+
792 * | | |
793 * | No . * |
794 * | v |
795 * | +--------------+ |
796 * +------>| Return | |
797 * | flow control |<------------+
798 * | cookie |
799 * +--------------+
800 *
801 *
802 * The remaining images are all specific to each of the different transmission
803 * modes.
804 *
805 * SRS TX DEFAULT
806 *
807 * [ From Part 1 ]
808 * |
809 * v
810 * +-------------------------+
811 * | mac_tx_single_ring_mode |
812 * +-------------------------+
813 * |
814 * | . Yes
815 * v .
816 * +==========+ . +============+
817 * v SRS v-*->v Try to v---->---------------------+
818 * v backlog? v v enqueue in v |
819 * +==========+ v SRS v-->------+ * . . Queue too
820 * | +============+ * don't enqueue | deep or
821 * * . No ^ | | flag or at | drop flag
822 * | | v | hiwat, |
823 * v | | | return +---------+
824 * +-------------+ | | | cookie | freemsg |
825 * | goto |-*-----+ | | +---------+
826 * | mac_tx_send | . returned | | |
827 * +-------------+ mblk_t | | |
828 * | | | |
829 * | | | |
830 * * . . all mblk_t * queued, | |
831 * v consumed | may return | |
832 * +-------------+ | tx cookie | |
833 * | SRS TX func |<------------+------------+----------------+
834 * | completed |
835 * +-------------+
836 *
837 * SRS_TX_SERIALIZE
838 *
839 * +------------------------+
840 * | mac_tx_serializer_mode |
841 * +------------------------+
842 * |
843 * | . No
844 * v .
845 * +============+ . +============+ +-------------+ +============+
846 * v srs being v-*->v set SRS v--->| goto |-->v remove SRS v
847 * v processed? v v proc flags v | mac_tx_send | v proc flag v
848 * +============+ +============+ +-------------+ +============+
849 * | |
850 * * Yes |
851 * v . No v
852 * +--------------------+ . +==========+
853 * | mac_tx_srs_enqueue | +------------------------*-----<--v returned v
854 * +--------------------+ | v frames? v
855 * | | . Yes +==========+
856 * | | . |
857 * | | . +=========+ v
858 * v +-<-*-v queued v +--------------------+
859 * +-------------+ | v frames? v<----| mac_tx_srs_enqueue |
860 * | SRS TX func | | +=========+ +--------------------+
861 * | completed, |<------+ * . Yes
862 * | may return | | v
863 * | cookie | | +========+
864 * +-------------+ +-<---v wake v
865 * v worker v
866 * +========+
867 *
868 *
869 * SRS_TX_FANOUT
870 *
871 * . Yes
872 * +--------------------+ +=============+ . +--------------------------+
873 * | mac_tx_fanout_mode |--->v Have fanout v-*-->| goto |
874 * +--------------------+ v hint? v | mac_rx_soft_ring_process |
875 * +=============+ +--------------------------+
876 * * . No |
877 * v ^
878 * +===========+ |
879 * +--->v for each v +===============+
880 * | v mblk_t v v pick softring v
881 * same * +===========+ v from hash v
882 * hash | | +===============+
883 * | v |
884 * | +--------------+ |
885 * +---| mac_pkt_hash |--->*------------+
886 * +--------------+ . different
887 * hash or
888 * done proc.
889 * SRS_TX_AGGR chain
890 *
891 * +------------------+ +================================+
892 * | mac_tx_aggr_mode |--->v Use aggr capab function to v
893 * +------------------+ v find appropriate tx ring. v
894 * v Applies hash based on aggr v
895 * v policy, see mac_tx_aggr_mode() v
896 * +================================+
897 * |
898 * v
899 * +-------------------------------+
900 * | goto |
901 * | mac_rx_srs_soft_ring_process |
902 * +-------------------------------+
903 *
904 *
905 * SRS_TX_BW, SRS_TX_BW_FANOUT, SRS_TX_BW_AGGR
906 *
907 * Note, all three of these tx functions start from the same place --
908 * mac_tx_bw_mode().
909 *
910 * +----------------+
911 * | mac_tx_bw_mode |
912 * +----------------+
913 * |
914 * v . No . No . Yes
915 * +==============+ . +============+ . +=============+ . +=========+
916 * v Out of BW? v--*->v SRS empty? v--*->v reset BW v-*->v Bump BW v
917 * +==============+ +============+ v tick count? v v Usage v
918 * | | +=============+ +=========+
919 * | +---------+ | |
920 * | | +--------------------+ |
921 * | | | +----------------------+
922 * v | v v
923 * +===============+ | +==========+ +==========+ +------------------+
924 * v Don't enqueue v | v set bw v v Is aggr? v--*-->| goto |
925 * v flag set? v | v enforced v +==========+ . | mac_tx_aggr_mode |-+
926 * +===============+ | +==========+ | . +------------------+ |
927 * | Yes .* | | No . * . |
928 * | | | | | . Yes |
929 * * . No | | v | |
930 * | +---------+ | +========+ v +======+ |
931 * | | freemsg | | v append v +============+ . Yes v pick v |
932 * | +---------+ | v mblk_t v v Is fanout? v--*---->v ring v |
933 * | | | v chain v +============+ +======+ |
934 * +------+ | +========+ | | |
935 * v | | v v |
936 * +---------+ | v +-------------+ +--------------------+ |
937 * | return | | +========+ | goto | | goto | |
938 * | flow | | v wakeup v | mac_tx_send | | mac_tx_fanout_mode | |
939 * | control | | v worker v +-------------+ +--------------------+ |
940 * | cookie | | +========+ | | |
941 * +---------+ | | | +------+------+
942 * | v | |
943 * | +---------+ | v
944 * | | return | +============+ +------------+
945 * | | flow | v unconsumed v-------+ | done |
946 * | | control | v frames? v | | processing |
947 * | | cookie | +============+ | +------------+
948 * | +---------+ | |
949 * | Yes * |
950 * | | |
951 * | +===========+ |
952 * | v subtract v |
953 * | v unused bw v |
954 * | +===========+ |
955 * | | |
956 * | v |
957 * | +--------------------+ |
958 * +------------->| mac_tx_srs_enqueue | |
959 * +--------------------+ |
960 * | |
961 * | |
962 * +------------+ |
963 * | return fc | |
964 * | cookie and |<------+
965 * | mblk_t |
966 * +------------+
967 */
968
969 #include <sys/types.h>
970 #include <sys/callb.h>
971 #include <sys/sdt.h>
972 #include <sys/strsubr.h>
973 #include <sys/strsun.h>
974 #include <sys/vlan.h>
975 #include <sys/stack.h>
976 #include <sys/archsystm.h>
977 #include <inet/ipsec_impl.h>
978 #include <inet/ip_impl.h>
979 #include <inet/sadb.h>
980 #include <inet/ipsecesp.h>
981 #include <inet/ipsecah.h>
982 #include <inet/ip6.h>
983
984 #include <sys/mac_impl.h>
985 #include <sys/mac_client_impl.h>
986 #include <sys/mac_client_priv.h>
987 #include <sys/mac_soft_ring.h>
988 #include <sys/mac_flow_impl.h>
989
990 static mac_tx_cookie_t mac_tx_single_ring_mode(mac_soft_ring_set_t *, mblk_t *,
991 uintptr_t, uint16_t, mblk_t **);
992 static mac_tx_cookie_t mac_tx_serializer_mode(mac_soft_ring_set_t *, mblk_t *,
993 uintptr_t, uint16_t, mblk_t **);
994 static mac_tx_cookie_t mac_tx_fanout_mode(mac_soft_ring_set_t *, mblk_t *,
995 uintptr_t, uint16_t, mblk_t **);
996 static mac_tx_cookie_t mac_tx_bw_mode(mac_soft_ring_set_t *, mblk_t *,
997 uintptr_t, uint16_t, mblk_t **);
998 static mac_tx_cookie_t mac_tx_aggr_mode(mac_soft_ring_set_t *, mblk_t *,
999 uintptr_t, uint16_t, mblk_t **);
1000
1001 typedef struct mac_tx_mode_s {
1002 mac_tx_srs_mode_t mac_tx_mode;
1003 mac_tx_func_t mac_tx_func;
1004 } mac_tx_mode_t;
1005
1006 /*
1007 * There are seven modes of operation on the Tx side. These modes get set
1008 * in mac_tx_srs_setup(). Except for the experimental TX_SERIALIZE mode,
1009 * none of the other modes are user configurable. They get selected by
1010 * the system depending upon whether the link (or flow) has multiple Tx
1011 * rings or a bandwidth configured, or if the link is an aggr, etc.
1012 *
1013 * When the Tx SRS is operating in aggr mode (st_mode) or if there are
1014 * multiple Tx rings owned by Tx SRS, then each Tx ring (pseudo or
1015 * otherwise) will have a soft ring associated with it. These soft rings
1016 * are stored in srs_tx_soft_rings[] array.
1017 *
1018 * Additionally in the case of aggr, there is the st_soft_rings[] array
1019 * in the mac_srs_tx_t structure. This array is used to store the same
1020 * set of soft rings that are present in srs_tx_soft_rings[] array but
1021 * in a different manner. The soft ring associated with the pseudo Tx
1022 * ring is saved at mr_index (of the pseudo ring) in st_soft_rings[]
1023 * array. This helps in quickly getting the soft ring associated with the
1024 * Tx ring when aggr_find_tx_ring() returns the pseudo Tx ring that is to
1025 * be used for transmit.
1026 */
1027 mac_tx_mode_t mac_tx_mode_list[] = {
1028 {SRS_TX_DEFAULT, mac_tx_single_ring_mode},
1029 {SRS_TX_SERIALIZE, mac_tx_serializer_mode},
1030 {SRS_TX_FANOUT, mac_tx_fanout_mode},
1031 {SRS_TX_BW, mac_tx_bw_mode},
1032 {SRS_TX_BW_FANOUT, mac_tx_bw_mode},
1033 {SRS_TX_AGGR, mac_tx_aggr_mode},
1034 {SRS_TX_BW_AGGR, mac_tx_bw_mode}
1035 };
1036
1037 /*
1038 * Soft Ring Set (SRS) - The Run time code that deals with
1039 * dynamic polling from the hardware, bandwidth enforcement,
1040 * fanout etc.
1041 *
1042 * We try to use H/W classification on NIC and assign traffic for
1043 * a MAC address to a particular Rx ring or ring group. There is a
1044 * 1-1 mapping between a SRS and a Rx ring. The SRS dynamically
1045 * switches the underlying Rx ring between interrupt and
1046 * polling mode and enforces any specified B/W control.
1047 *
1048 * There is always a SRS created and tied to each H/W and S/W rule.
1049 * Whenever we create a H/W rule, we always add the the same rule to
1050 * S/W classifier and tie a SRS to it.
1051 *
1052 * In case a B/W control is specified, it is broken into bytes
1053 * per ticks and as soon as the quota for a tick is exhausted,
1054 * the underlying Rx ring is forced into poll mode for remainder of
1055 * the tick. The SRS poll thread only polls for bytes that are
1056 * allowed to come in the SRS. We typically let 4x the configured
1057 * B/W worth of packets to come in the SRS (to prevent unnecessary
1058 * drops due to bursts) but only process the specified amount.
1059 *
1060 * A MAC client (e.g. a VNIC or aggr) can have 1 or more
1061 * Rx rings (and corresponding SRSs) assigned to it. The SRS
1062 * in turn can have softrings to do protocol level fanout or
1063 * softrings to do S/W based fanout or both. In case the NIC
1064 * has no Rx rings, we do S/W classification to respective SRS.
1065 * The S/W classification rule is always setup and ready. This
1066 * allows the MAC layer to reassign Rx rings whenever needed
1067 * but packets still continue to flow via the default path and
1068 * getting S/W classified to correct SRS.
1069 *
1070 * The SRS's are used on both Tx and Rx side. They use the same
1071 * data structure but the processing routines have slightly different
1072 * semantics due to the fact that Rx side needs to do dynamic
1073 * polling etc.
1074 *
1075 * Dynamic Polling Notes
1076 * =====================
1077 *
1078 * Each Soft ring set is capable of switching its Rx ring between
1079 * interrupt and poll mode and actively 'polls' for packets in
1080 * poll mode. If the SRS is implementing a B/W limit, it makes
1081 * sure that only Max allowed packets are pulled in poll mode
1082 * and goes to poll mode as soon as B/W limit is exceeded. As
1083 * such, there are no overheads to implement B/W limits.
1084 *
1085 * In poll mode, its better to keep the pipeline going where the
1086 * SRS worker thread keeps processing packets and poll thread
1087 * keeps bringing more packets (specially if they get to run
1088 * on different CPUs). This also prevents the overheads associated
1089 * by excessive signalling (on NUMA machines, this can be
1090 * pretty devastating). The exception is latency optimized case
1091 * where worker thread does no work and interrupt and poll thread
1092 * are allowed to do their own drain.
1093 *
1094 * We use the following policy to control Dynamic Polling:
1095 * 1) We switch to poll mode anytime the processing
1096 * thread causes a backlog to build up in SRS and
1097 * its associated Soft Rings (sr_poll_pkt_cnt > 0).
1098 * 2) As long as the backlog stays under the low water
1099 * mark (sr_lowat), we poll the H/W for more packets.
1100 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low
1101 * water mark, we stay in poll mode but don't poll
1102 * the H/W for more packets.
1103 * 4) Anytime in polling mode, if we poll the H/W for
1104 * packets and find nothing plus we have an existing
1105 * backlog (sr_poll_pkt_cnt > 0), we stay in polling
1106 * mode but don't poll the H/W for packets anymore
1107 * (let the polling thread go to sleep).
1108 * 5) Once the backlog is relived (packets are processed)
1109 * we reenable polling (by signalling the poll thread)
1110 * only when the backlog dips below sr_poll_thres.
1111 * 6) sr_hiwat is used exclusively when we are not
1112 * polling capable and is used to decide when to
1113 * drop packets so the SRS queue length doesn't grow
1114 * infinitely.
1115 *
1116 * NOTE: Also see the block level comment on top of mac_soft_ring.c
1117 */
1118
1119 /*
1120 * mac_latency_optimize
1121 *
1122 * Controls whether the poll thread can process the packets inline
1123 * or let the SRS worker thread do the processing. This applies if
1124 * the SRS was not being processed. For latency sensitive traffic,
1125 * this needs to be true to allow inline processing. For throughput
1126 * under load, this should be false.
1127 *
1128 * This (and other similar) tunable should be rolled into a link
1129 * or flow specific workload hint that can be set using dladm
1130 * linkprop (instead of multiple such tunables).
1131 */
1132 boolean_t mac_latency_optimize = B_TRUE;
1133
1134 /*
1135 * MAC_RX_SRS_ENQUEUE_CHAIN and MAC_TX_SRS_ENQUEUE_CHAIN
1136 *
1137 * queue a mp or chain in soft ring set and increment the
1138 * local count (srs_count) for the SRS and the shared counter
1139 * (srs_poll_pkt_cnt - shared between SRS and its soft rings
1140 * to track the total unprocessed packets for polling to work
1141 * correctly).
1142 *
1143 * The size (total bytes queued) counters are incremented only
1144 * if we are doing B/W control.
1145 */
1146 #define MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
1147 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1148 if ((mac_srs)->srs_last != NULL) \
1149 (mac_srs)->srs_last->b_next = (head); \
1150 else \
1151 (mac_srs)->srs_first = (head); \
1152 (mac_srs)->srs_last = (tail); \
1153 (mac_srs)->srs_count += count; \
1154 }
1155
1156 #define MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
1157 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
1158 \
1159 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \
1160 srs_rx->sr_poll_pkt_cnt += count; \
1161 ASSERT(srs_rx->sr_poll_pkt_cnt > 0); \
1162 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \
1163 (mac_srs)->srs_size += (sz); \
1164 mutex_enter(&(mac_srs)->srs_bw->mac_bw_lock); \
1165 (mac_srs)->srs_bw->mac_bw_sz += (sz); \
1166 mutex_exit(&(mac_srs)->srs_bw->mac_bw_lock); \
1167 } \
1168 }
1169
1170 #define MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) { \
1171 mac_srs->srs_state |= SRS_ENQUEUED; \
1172 MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz); \
1173 if ((mac_srs)->srs_type & SRST_BW_CONTROL) { \
1174 (mac_srs)->srs_size += (sz); \
1175 (mac_srs)->srs_bw->mac_bw_sz += (sz); \
1176 } \
1177 }
1178
1179 /*
1180 * Turn polling on routines
1181 */
1182 #define MAC_SRS_POLLING_ON(mac_srs) { \
1183 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1184 if (((mac_srs)->srs_state & \
1185 (SRS_POLLING_CAPAB|SRS_POLLING)) == SRS_POLLING_CAPAB) { \
1186 (mac_srs)->srs_state |= SRS_POLLING; \
1187 (void) mac_hwring_disable_intr((mac_ring_handle_t) \
1188 (mac_srs)->srs_ring); \
1189 (mac_srs)->srs_rx.sr_poll_on++; \
1190 } \
1191 }
1192
1193 #define MAC_SRS_WORKER_POLLING_ON(mac_srs) { \
1194 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1195 if (((mac_srs)->srs_state & \
1196 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_POLLING)) == \
1197 (SRS_POLLING_CAPAB|SRS_WORKER)) { \
1198 (mac_srs)->srs_state |= SRS_POLLING; \
1199 (void) mac_hwring_disable_intr((mac_ring_handle_t) \
1200 (mac_srs)->srs_ring); \
1201 (mac_srs)->srs_rx.sr_worker_poll_on++; \
1202 } \
1203 }
1204
1205 /*
1206 * MAC_SRS_POLL_RING
1207 *
1208 * Signal the SRS poll thread to poll the underlying H/W ring
1209 * provided it wasn't already polling (SRS_GET_PKTS was set).
1210 *
1211 * Poll thread gets to run only from mac_rx_srs_drain() and only
1212 * if the drain was being done by the worker thread.
1213 */
1214 #define MAC_SRS_POLL_RING(mac_srs) { \
1215 mac_srs_rx_t *srs_rx = &(mac_srs)->srs_rx; \
1216 \
1217 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1218 srs_rx->sr_poll_thr_sig++; \
1219 if (((mac_srs)->srs_state & \
1220 (SRS_POLLING_CAPAB|SRS_WORKER|SRS_GET_PKTS)) == \
1221 (SRS_WORKER|SRS_POLLING_CAPAB)) { \
1222 (mac_srs)->srs_state |= SRS_GET_PKTS; \
1223 cv_signal(&(mac_srs)->srs_cv); \
1224 } else { \
1225 srs_rx->sr_poll_thr_busy++; \
1226 } \
1227 }
1228
1229 /*
1230 * MAC_SRS_CHECK_BW_CONTROL
1231 *
1232 * Check to see if next tick has started so we can reset the
1233 * SRS_BW_ENFORCED flag and allow more packets to come in the
1234 * system.
1235 */
1236 #define MAC_SRS_CHECK_BW_CONTROL(mac_srs) { \
1237 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1238 ASSERT(((mac_srs)->srs_type & SRST_TX) || \
1239 MUTEX_HELD(&(mac_srs)->srs_bw->mac_bw_lock)); \
1240 clock_t now = ddi_get_lbolt(); \
1241 if ((mac_srs)->srs_bw->mac_bw_curr_time != now) { \
1242 (mac_srs)->srs_bw->mac_bw_curr_time = now; \
1243 (mac_srs)->srs_bw->mac_bw_used = 0; \
1244 if ((mac_srs)->srs_bw->mac_bw_state & SRS_BW_ENFORCED) \
1245 (mac_srs)->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED; \
1246 } \
1247 }
1248
1249 /*
1250 * MAC_SRS_WORKER_WAKEUP
1251 *
1252 * Wake up the SRS worker thread to process the queue as long as
1253 * no one else is processing the queue. If we are optimizing for
1254 * latency, we wake up the worker thread immediately or else we
1255 * wait mac_srs_worker_wakeup_ticks before worker thread gets
1256 * woken up.
1257 */
1258 int mac_srs_worker_wakeup_ticks = 0;
1259 #define MAC_SRS_WORKER_WAKEUP(mac_srs) { \
1260 ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock)); \
1261 if (!((mac_srs)->srs_state & SRS_PROC) && \
1262 (mac_srs)->srs_tid == NULL) { \
1263 if (((mac_srs)->srs_state & SRS_LATENCY_OPT) || \
1264 (mac_srs_worker_wakeup_ticks == 0)) \
1265 cv_signal(&(mac_srs)->srs_async); \
1266 else \
1267 (mac_srs)->srs_tid = \
1268 timeout(mac_srs_fire, (mac_srs), \
1269 mac_srs_worker_wakeup_ticks); \
1270 } \
1271 }
1272
1273 #define TX_BANDWIDTH_MODE(mac_srs) \
1274 ((mac_srs)->srs_tx.st_mode == SRS_TX_BW || \
1275 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_FANOUT || \
1276 (mac_srs)->srs_tx.st_mode == SRS_TX_BW_AGGR)
1277
1278 #define TX_SRS_TO_SOFT_RING(mac_srs, head, hint) { \
1279 if (tx_mode == SRS_TX_BW_FANOUT) \
1280 (void) mac_tx_fanout_mode(mac_srs, head, hint, 0, NULL);\
1281 else \
1282 (void) mac_tx_aggr_mode(mac_srs, head, hint, 0, NULL); \
1283 }
1284
1285 /*
1286 * MAC_TX_SRS_BLOCK
1287 *
1288 * Always called from mac_tx_srs_drain() function. SRS_TX_BLOCKED
1289 * will be set only if srs_tx_woken_up is FALSE. If
1290 * srs_tx_woken_up is TRUE, it indicates that the wakeup arrived
1291 * before we grabbed srs_lock to set SRS_TX_BLOCKED. We need to
1292 * attempt to transmit again and not setting SRS_TX_BLOCKED does
1293 * that.
1294 */
1295 #define MAC_TX_SRS_BLOCK(srs, mp) { \
1296 ASSERT(MUTEX_HELD(&(srs)->srs_lock)); \
1297 if ((srs)->srs_tx.st_woken_up) { \
1298 (srs)->srs_tx.st_woken_up = B_FALSE; \
1299 } else { \
1300 ASSERT(!((srs)->srs_state & SRS_TX_BLOCKED)); \
1301 (srs)->srs_state |= SRS_TX_BLOCKED; \
1302 (srs)->srs_tx.st_stat.mts_blockcnt++; \
1303 } \
1304 }
1305
1306 /*
1307 * MAC_TX_SRS_TEST_HIWAT
1308 *
1309 * Called before queueing a packet onto Tx SRS to test and set
1310 * SRS_TX_HIWAT if srs_count exceeds srs_tx_hiwat.
1311 */
1312 #define MAC_TX_SRS_TEST_HIWAT(srs, mp, tail, cnt, sz, cookie) { \
1313 boolean_t enqueue = 1; \
1314 \
1315 if ((srs)->srs_count > (srs)->srs_tx.st_hiwat) { \
1316 /* \
1317 * flow-controlled. Store srs in cookie so that it \
1318 * can be returned as mac_tx_cookie_t to client \
1319 */ \
1320 (srs)->srs_state |= SRS_TX_HIWAT; \
1321 cookie = (mac_tx_cookie_t)srs; \
1322 (srs)->srs_tx.st_hiwat_cnt++; \
1323 if ((srs)->srs_count > (srs)->srs_tx.st_max_q_cnt) { \
1324 /* increment freed stats */ \
1325 (srs)->srs_tx.st_stat.mts_sdrops += cnt; \
1326 /* \
1327 * b_prev may be set to the fanout hint \
1328 * hence can't use freemsg directly \
1329 */ \
1330 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE); \
1331 DTRACE_PROBE1(tx_queued_hiwat, \
1332 mac_soft_ring_set_t *, srs); \
1333 enqueue = 0; \
1334 } \
1335 } \
1336 if (enqueue) \
1337 MAC_TX_SRS_ENQUEUE_CHAIN(srs, mp, tail, cnt, sz); \
1338 }
1339
1340 /* Some utility macros */
1341 #define MAC_SRS_BW_LOCK(srs) \
1342 if (!(srs->srs_type & SRST_TX)) \
1343 mutex_enter(&srs->srs_bw->mac_bw_lock);
1344
1345 #define MAC_SRS_BW_UNLOCK(srs) \
1346 if (!(srs->srs_type & SRST_TX)) \
1347 mutex_exit(&srs->srs_bw->mac_bw_lock);
1348
1349 #define MAC_TX_SRS_DROP_MESSAGE(srs, mp, cookie) { \
1350 mac_pkt_drop(NULL, NULL, mp, B_FALSE); \
1351 /* increment freed stats */ \
1352 mac_srs->srs_tx.st_stat.mts_sdrops++; \
1353 cookie = (mac_tx_cookie_t)srs; \
1354 }
1355
1356 #define MAC_TX_SET_NO_ENQUEUE(srs, mp_chain, ret_mp, cookie) { \
1357 mac_srs->srs_state |= SRS_TX_WAKEUP_CLIENT; \
1358 cookie = (mac_tx_cookie_t)srs; \
1359 *ret_mp = mp_chain; \
1360 }
1361
1362 /*
1363 * MAC_RX_SRS_TOODEEP
1364 *
1365 * Macro called as part of receive-side processing to determine if handling
1366 * can occur in situ (in the interrupt thread) or if it should be left to a
1367 * worker thread. Note that the constant used to make this determination is
1368 * not entirely made-up, and is a result of some emprical validation. That
1369 * said, the constant is left as a static variable to allow it to be
1370 * dynamically tuned in the field if and as needed.
1371 */
1372 static uintptr_t mac_rx_srs_stack_needed = 10240;
1373 static uint_t mac_rx_srs_stack_toodeep;
1374
1375 #ifndef STACK_GROWTH_DOWN
1376 #error Downward stack growth assumed.
1377 #endif
1378
1379 #define MAC_RX_SRS_TOODEEP() (STACK_BIAS + (uintptr_t)getfp() - \
1380 (uintptr_t)curthread->t_stkbase < mac_rx_srs_stack_needed && \
1381 ++mac_rx_srs_stack_toodeep)
1382
1383
1384 /*
1385 * Drop the rx packet and advance to the next one in the chain.
1386 */
1387 static void
1388 mac_rx_drop_pkt(mac_soft_ring_set_t *srs, mblk_t *mp)
1389 {
1390 mac_srs_rx_t *srs_rx = &srs->srs_rx;
1391
1392 ASSERT(mp->b_next == NULL);
1393 mutex_enter(&srs->srs_lock);
1394 MAC_UPDATE_SRS_COUNT_LOCKED(srs, 1);
1395 MAC_UPDATE_SRS_SIZE_LOCKED(srs, msgdsize(mp));
1396 mutex_exit(&srs->srs_lock);
1397
1398 srs_rx->sr_stat.mrs_sdrops++;
1399 freemsg(mp);
1400 }
1401
1402 /* DATAPATH RUNTIME ROUTINES */
1403
1404 /*
1405 * mac_srs_fire
1406 *
1407 * Timer callback routine for waking up the SRS worker thread.
1408 */
1409 static void
1410 mac_srs_fire(void *arg)
1411 {
1412 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)arg;
1413
1414 mutex_enter(&mac_srs->srs_lock);
1415 if (mac_srs->srs_tid == NULL) {
1416 mutex_exit(&mac_srs->srs_lock);
1417 return;
1418 }
1419
1420 mac_srs->srs_tid = NULL;
1421 if (!(mac_srs->srs_state & SRS_PROC))
1422 cv_signal(&mac_srs->srs_async);
1423
1424 mutex_exit(&mac_srs->srs_lock);
1425 }
1426
1427 /*
1428 * 'hint' is fanout_hint (type of uint64_t) which is given by the TCP/IP stack,
1429 * and it is used on the TX path.
1430 */
1431 #define HASH_HINT(hint) \
1432 ((hint) ^ ((hint) >> 24) ^ ((hint) >> 16) ^ ((hint) >> 8))
1433
1434
1435 /*
1436 * hash based on the src address, dst address and the port information.
1437 */
1438 #define HASH_ADDR(src, dst, ports) \
1439 (ntohl((src) + (dst)) ^ ((ports) >> 24) ^ ((ports) >> 16) ^ \
1440 ((ports) >> 8) ^ (ports))
1441
1442 #define COMPUTE_INDEX(key, sz) (key % sz)
1443
1444 #define FANOUT_ENQUEUE_MP(head, tail, cnt, bw_ctl, sz, sz0, mp) { \
1445 if ((tail) != NULL) { \
1446 ASSERT((tail)->b_next == NULL); \
1447 (tail)->b_next = (mp); \
1448 } else { \
1449 ASSERT((head) == NULL); \
1450 (head) = (mp); \
1451 } \
1452 (tail) = (mp); \
1453 (cnt)++; \
1454 if ((bw_ctl)) \
1455 (sz) += (sz0); \
1456 }
1457
1458 #define MAC_FANOUT_DEFAULT 0
1459 #define MAC_FANOUT_RND_ROBIN 1
1460 int mac_fanout_type = MAC_FANOUT_DEFAULT;
1461
1462 #define MAX_SR_TYPES 3
1463 /* fanout types for port based hashing */
1464 enum pkt_type {
1465 V4_TCP = 0,
1466 V4_UDP,
1467 OTH,
1468 UNDEF
1469 };
1470
1471 /*
1472 * Pair of local and remote ports in the transport header
1473 */
1474 #define PORTS_SIZE 4
1475
1476 /*
1477 * This routine delivers packets destined for an SRS into one of the
1478 * protocol soft rings.
1479 *
1480 * Given a chain of packets we need to split it up into multiple sub
1481 * chains: TCP, UDP or OTH soft ring. Instead of entering the soft
1482 * ring one packet at a time, we want to enter it in the form of a
1483 * chain otherwise we get this start/stop behaviour where the worker
1484 * thread goes to sleep and then next packet comes in forcing it to
1485 * wake up.
1486 */
1487 static void
1488 mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
1489 {
1490 struct ether_header *ehp;
1491 struct ether_vlan_header *evhp;
1492 uint32_t sap;
1493 ipha_t *ipha;
1494 uint8_t *dstaddr;
1495 size_t hdrsize;
1496 mblk_t *mp;
1497 mblk_t *headmp[MAX_SR_TYPES];
1498 mblk_t *tailmp[MAX_SR_TYPES];
1499 int cnt[MAX_SR_TYPES];
1500 size_t sz[MAX_SR_TYPES];
1501 size_t sz1;
1502 boolean_t bw_ctl;
1503 boolean_t hw_classified;
1504 boolean_t dls_bypass;
1505 boolean_t is_ether;
1506 boolean_t is_unicast;
1507 enum pkt_type type;
1508 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1509
1510 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
1511 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
1512
1513 /*
1514 * If we don't have a Rx ring, S/W classification would have done
1515 * its job and its a packet meant for us. If we were polling on
1516 * the default ring (i.e. there was a ring assigned to this SRS),
1517 * then we need to make sure that the mac address really belongs
1518 * to us.
1519 */
1520 hw_classified = mac_srs->srs_ring != NULL &&
1521 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
1522
1523 /*
1524 * Some clients, such as non-ethernet, need DLS processing in
1525 * the Rx path. Such clients clear the SRST_DLS_BYPASS flag.
1526 * DLS bypass may also be disabled via the
1527 * MCIS_RX_BYPASS_DISABLE flag.
1528 */
1529 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
1530 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
1531
1532 bzero(headmp, MAX_SR_TYPES * sizeof (mblk_t *));
1533 bzero(tailmp, MAX_SR_TYPES * sizeof (mblk_t *));
1534 bzero(cnt, MAX_SR_TYPES * sizeof (int));
1535 bzero(sz, MAX_SR_TYPES * sizeof (size_t));
1536
1537 /*
1538 * We have a chain from SRS that we need to split across the
1539 * soft rings. The squeues for the TCP and IPv4 SAPs use their
1540 * own soft rings to allow polling from the squeue. The rest of
1541 * the packets are delivered on the OTH soft ring which cannot
1542 * be polled.
1543 */
1544 while (head != NULL) {
1545 mp = head;
1546 head = head->b_next;
1547 mp->b_next = NULL;
1548
1549 type = OTH;
1550 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1551
1552 if (is_ether) {
1553 /*
1554 * At this point we can be sure the packet at least
1555 * has an ether header.
1556 */
1557 if (sz1 < sizeof (struct ether_header)) {
1558 mac_rx_drop_pkt(mac_srs, mp);
1559 continue;
1560 }
1561 ehp = (struct ether_header *)mp->b_rptr;
1562
1563 /*
1564 * Determine if this is a VLAN or non-VLAN packet.
1565 */
1566 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
1567 evhp = (struct ether_vlan_header *)mp->b_rptr;
1568 sap = ntohs(evhp->ether_type);
1569 hdrsize = sizeof (struct ether_vlan_header);
1570
1571 /*
1572 * Check if the VID of the packet, if
1573 * any, belongs to this client.
1574 * Technically, if this packet came up
1575 * via a HW classified ring then we
1576 * don't need to perform this check.
1577 * Perhaps a future optimization.
1578 */
1579 if (!mac_client_check_flow_vid(mcip,
1580 VLAN_ID(ntohs(evhp->ether_tci)))) {
1581 mac_rx_drop_pkt(mac_srs, mp);
1582 continue;
1583 }
1584 } else {
1585 hdrsize = sizeof (struct ether_header);
1586 }
1587 is_unicast =
1588 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
1589 dstaddr = (uint8_t *)&ehp->ether_dhost;
1590 } else {
1591 mac_header_info_t mhi;
1592
1593 if (mac_header_info((mac_handle_t)mcip->mci_mip,
1594 mp, &mhi) != 0) {
1595 mac_rx_drop_pkt(mac_srs, mp);
1596 continue;
1597 }
1598 hdrsize = mhi.mhi_hdrsize;
1599 sap = mhi.mhi_bindsap;
1600 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
1601 dstaddr = (uint8_t *)mhi.mhi_daddr;
1602 }
1603
1604 if (!dls_bypass) {
1605 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
1606 cnt[type], bw_ctl, sz[type], sz1, mp);
1607 continue;
1608 }
1609
1610 if (sap == ETHERTYPE_IP) {
1611 /*
1612 * If we are H/W classified, but we have promisc
1613 * on, then we need to check for the unicast address.
1614 */
1615 if (hw_classified && mcip->mci_promisc_list != NULL) {
1616 mac_address_t *map;
1617
1618 rw_enter(&mcip->mci_rw_lock, RW_READER);
1619 map = mcip->mci_unicast;
1620 if (bcmp(dstaddr, map->ma_addr,
1621 map->ma_len) == 0)
1622 type = UNDEF;
1623 rw_exit(&mcip->mci_rw_lock);
1624 } else if (is_unicast) {
1625 type = UNDEF;
1626 }
1627 }
1628
1629 /*
1630 * This needs to become a contract with the driver for
1631 * the fast path.
1632 *
1633 * In the normal case the packet will have at least the L2
1634 * header and the IP + Transport header in the same mblk.
1635 * This is usually the case when the NIC driver sends up
1636 * the packet. This is also true when the stack generates
1637 * a packet that is looped back and when the stack uses the
1638 * fastpath mechanism. The normal case is optimized for
1639 * performance and may bypass DLS. All other cases go through
1640 * the 'OTH' type path without DLS bypass.
1641 */
1642 ipha = (ipha_t *)(mp->b_rptr + hdrsize);
1643 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha))
1644 type = OTH;
1645
1646 if (type == OTH) {
1647 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
1648 cnt[type], bw_ctl, sz[type], sz1, mp);
1649 continue;
1650 }
1651
1652 ASSERT(type == UNDEF);
1653
1654 /*
1655 * Determine the type from the IP protocol value. If
1656 * classified as TCP or UDP, then update the read
1657 * pointer to the beginning of the IP header.
1658 * Otherwise leave the message as is for further
1659 * processing by DLS.
1660 */
1661 switch (ipha->ipha_protocol) {
1662 case IPPROTO_TCP:
1663 type = V4_TCP;
1664 mp->b_rptr += hdrsize;
1665 break;
1666 case IPPROTO_UDP:
1667 type = V4_UDP;
1668 mp->b_rptr += hdrsize;
1669 break;
1670 default:
1671 type = OTH;
1672 break;
1673 }
1674
1675 FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], cnt[type],
1676 bw_ctl, sz[type], sz1, mp);
1677 }
1678
1679 for (type = V4_TCP; type < UNDEF; type++) {
1680 if (headmp[type] != NULL) {
1681 mac_soft_ring_t *softring;
1682
1683 ASSERT(tailmp[type]->b_next == NULL);
1684 switch (type) {
1685 case V4_TCP:
1686 softring = mac_srs->srs_tcp_soft_rings[0];
1687 break;
1688 case V4_UDP:
1689 softring = mac_srs->srs_udp_soft_rings[0];
1690 break;
1691 case OTH:
1692 softring = mac_srs->srs_oth_soft_rings[0];
1693 }
1694 mac_rx_soft_ring_process(mcip, softring,
1695 headmp[type], tailmp[type], cnt[type], sz[type]);
1696 }
1697 }
1698 }
1699
1700 int fanout_unaligned = 0;
1701
1702 /*
1703 * The fanout routine for any clients with DLS bypass disabled or for
1704 * traffic classified as "other". Returns -1 on an error (drop the
1705 * packet due to a malformed packet), 0 on success, with values
1706 * written in *indx and *type.
1707 */
1708 static int
1709 mac_rx_srs_long_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *mp,
1710 uint32_t sap, size_t hdrsize, enum pkt_type *type, uint_t *indx)
1711 {
1712 ip6_t *ip6h;
1713 ipha_t *ipha;
1714 uint8_t *whereptr;
1715 uint_t hash;
1716 uint16_t remlen;
1717 uint8_t nexthdr;
1718 uint16_t hdr_len;
1719 uint32_t src_val, dst_val;
1720 boolean_t modifiable = B_TRUE;
1721 boolean_t v6;
1722
1723 ASSERT(MBLKL(mp) >= hdrsize);
1724
1725 if (sap == ETHERTYPE_IPV6) {
1726 v6 = B_TRUE;
1727 hdr_len = IPV6_HDR_LEN;
1728 } else if (sap == ETHERTYPE_IP) {
1729 v6 = B_FALSE;
1730 hdr_len = IP_SIMPLE_HDR_LENGTH;
1731 } else {
1732 *indx = 0;
1733 *type = OTH;
1734 return (0);
1735 }
1736
1737 ip6h = (ip6_t *)(mp->b_rptr + hdrsize);
1738 ipha = (ipha_t *)ip6h;
1739
1740 if ((uint8_t *)ip6h == mp->b_wptr) {
1741 /*
1742 * The first mblk_t only includes the mac header.
1743 * Note that it is safe to change the mp pointer here,
1744 * as the subsequent operation does not assume mp
1745 * points to the start of the mac header.
1746 */
1747 mp = mp->b_cont;
1748
1749 /*
1750 * Make sure the IP header points to an entire one.
1751 */
1752 if (mp == NULL)
1753 return (-1);
1754
1755 if (MBLKL(mp) < hdr_len) {
1756 modifiable = (DB_REF(mp) == 1);
1757
1758 if (modifiable && !pullupmsg(mp, hdr_len))
1759 return (-1);
1760 }
1761
1762 ip6h = (ip6_t *)mp->b_rptr;
1763 ipha = (ipha_t *)ip6h;
1764 }
1765
1766 if (!modifiable || !(OK_32PTR((char *)ip6h)) ||
1767 ((uint8_t *)ip6h + hdr_len > mp->b_wptr)) {
1768 /*
1769 * If either the IP header is not aligned, or it does not hold
1770 * the complete simple structure (a pullupmsg() is not an
1771 * option since it would result in an unaligned IP header),
1772 * fanout to the default ring.
1773 *
1774 * Note that this may cause packet reordering.
1775 */
1776 *indx = 0;
1777 *type = OTH;
1778 fanout_unaligned++;
1779 return (0);
1780 }
1781
1782 /*
1783 * Extract next-header, full header length, and source-hash value
1784 * using v4/v6 specific fields.
1785 */
1786 if (v6) {
1787 remlen = ntohs(ip6h->ip6_plen);
1788 nexthdr = ip6h->ip6_nxt;
1789 src_val = V4_PART_OF_V6(ip6h->ip6_src);
1790 dst_val = V4_PART_OF_V6(ip6h->ip6_dst);
1791 /*
1792 * Do src based fanout if below tunable is set to B_TRUE or
1793 * when mac_ip_hdr_length_v6() fails because of malformed
1794 * packets or because mblks need to be concatenated using
1795 * pullupmsg().
1796 *
1797 * Perform a version check to prevent parsing weirdness...
1798 */
1799 if (IPH_HDR_VERSION(ip6h) != IPV6_VERSION ||
1800 !mac_ip_hdr_length_v6(ip6h, mp->b_wptr, &hdr_len, &nexthdr,
1801 NULL)) {
1802 goto src_dst_based_fanout;
1803 }
1804 } else {
1805 hdr_len = IPH_HDR_LENGTH(ipha);
1806 remlen = ntohs(ipha->ipha_length) - hdr_len;
1807 nexthdr = ipha->ipha_protocol;
1808 src_val = (uint32_t)ipha->ipha_src;
1809 dst_val = (uint32_t)ipha->ipha_dst;
1810 /*
1811 * Catch IPv4 fragment case here. IPv6 has nexthdr == FRAG
1812 * for its equivalent case.
1813 */
1814 if ((ntohs(ipha->ipha_fragment_offset_and_flags) &
1815 (IPH_MF | IPH_OFFSET)) != 0) {
1816 goto src_dst_based_fanout;
1817 }
1818 }
1819 if (remlen < MIN_EHDR_LEN)
1820 return (-1);
1821 whereptr = (uint8_t *)ip6h + hdr_len;
1822
1823 /* If the transport is one of below, we do port/SPI based fanout */
1824 switch (nexthdr) {
1825 case IPPROTO_TCP:
1826 case IPPROTO_UDP:
1827 case IPPROTO_SCTP:
1828 case IPPROTO_ESP:
1829 /*
1830 * If the ports or SPI in the transport header is not part of
1831 * the mblk, do src_based_fanout, instead of calling
1832 * pullupmsg().
1833 */
1834 if (mp->b_cont == NULL || whereptr + PORTS_SIZE <= mp->b_wptr)
1835 break; /* out of switch... */
1836 /* FALLTHRU */
1837 default:
1838 goto src_dst_based_fanout;
1839 }
1840
1841 switch (nexthdr) {
1842 case IPPROTO_TCP:
1843 hash = HASH_ADDR(src_val, dst_val, *(uint32_t *)whereptr);
1844 *indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
1845 *type = OTH;
1846 break;
1847 case IPPROTO_UDP:
1848 case IPPROTO_SCTP:
1849 case IPPROTO_ESP:
1850 if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
1851 hash = HASH_ADDR(src_val, dst_val,
1852 *(uint32_t *)whereptr);
1853 *indx = COMPUTE_INDEX(hash,
1854 mac_srs->srs_udp_ring_count);
1855 } else {
1856 *indx = mac_srs->srs_ind % mac_srs->srs_udp_ring_count;
1857 mac_srs->srs_ind++;
1858 }
1859 *type = OTH;
1860 break;
1861 }
1862 return (0);
1863
1864 src_dst_based_fanout:
1865 hash = HASH_ADDR(src_val, dst_val, (uint32_t)0);
1866 *indx = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count);
1867 *type = OTH;
1868 return (0);
1869 }
1870
1871 /*
1872 * This routine delivers packets destined for an SRS into a soft ring member
1873 * of the set.
1874 *
1875 * Given a chain of packets we need to split it up into multiple sub
1876 * chains: TCP, UDP or OTH soft ring. Instead of entering the soft
1877 * ring one packet at a time, we want to enter it in the form of a
1878 * chain otherwise we get this start/stop behaviour where the worker
1879 * thread goes to sleep and then next packet comes in forcing it to
1880 * wake up.
1881 *
1882 * Note:
1883 * Since we know what is the maximum fanout possible, we create a 2D array
1884 * of 'softring types * MAX_SR_FANOUT' for the head, tail, cnt and sz
1885 * variables so that we can enter the softrings with chain. We need the
1886 * MAX_SR_FANOUT so we can allocate the arrays on the stack (a kmem_alloc
1887 * for each packet would be expensive). If we ever want to have the
1888 * ability to have unlimited fanout, we should probably declare a head,
1889 * tail, cnt, sz with each soft ring (a data struct which contains a softring
1890 * along with these members) and create an array of this uber struct so we
1891 * don't have to do kmem_alloc.
1892 */
1893 int fanout_oth1 = 0;
1894 int fanout_oth2 = 0;
1895 int fanout_oth3 = 0;
1896 int fanout_oth4 = 0;
1897 int fanout_oth5 = 0;
1898
1899 static void
1900 mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
1901 {
1902 struct ether_header *ehp;
1903 struct ether_vlan_header *evhp;
1904 uint32_t sap;
1905 ipha_t *ipha;
1906 uint8_t *dstaddr;
1907 uint_t indx;
1908 size_t ports_offset;
1909 size_t ipha_len;
1910 size_t hdrsize;
1911 uint_t hash;
1912 mblk_t *mp;
1913 mblk_t *headmp[MAX_SR_TYPES][MAX_SR_FANOUT];
1914 mblk_t *tailmp[MAX_SR_TYPES][MAX_SR_FANOUT];
1915 int cnt[MAX_SR_TYPES][MAX_SR_FANOUT];
1916 size_t sz[MAX_SR_TYPES][MAX_SR_FANOUT];
1917 size_t sz1;
1918 boolean_t bw_ctl;
1919 boolean_t hw_classified;
1920 boolean_t dls_bypass;
1921 boolean_t is_ether;
1922 boolean_t is_unicast;
1923 int fanout_cnt;
1924 enum pkt_type type;
1925 mac_client_impl_t *mcip = mac_srs->srs_mcip;
1926
1927 is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
1928 bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
1929
1930 /*
1931 * If we don't have a Rx ring, S/W classification would have done
1932 * its job and its a packet meant for us. If we were polling on
1933 * the default ring (i.e. there was a ring assigned to this SRS),
1934 * then we need to make sure that the mac address really belongs
1935 * to us.
1936 */
1937 hw_classified = mac_srs->srs_ring != NULL &&
1938 mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
1939
1940 /*
1941 * Some clients, such as non Ethernet, need DLS processing in
1942 * the Rx path. Such clients clear the SRST_DLS_BYPASS flag.
1943 * DLS bypass may also be disabled via the
1944 * MCIS_RX_BYPASS_DISABLE flag, but this is only consumed by
1945 * sun4v vsw currently.
1946 */
1947 dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
1948 ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
1949
1950 /*
1951 * Since the softrings are never destroyed and we always
1952 * create equal number of softrings for TCP, UDP and rest,
1953 * its OK to check one of them for count and use it without
1954 * any lock. In future, if soft rings get destroyed because
1955 * of reduction in fanout, we will need to ensure that happens
1956 * behind the SRS_PROC.
1957 */
1958 fanout_cnt = mac_srs->srs_tcp_ring_count;
1959
1960 bzero(headmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
1961 bzero(tailmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
1962 bzero(cnt, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (int));
1963 bzero(sz, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (size_t));
1964
1965 /*
1966 * We got a chain from SRS that we need to send to the soft rings.
1967 * Since squeues for TCP & IPv4 SAP poll their soft rings (for
1968 * performance reasons), we need to separate out v4_tcp, v4_udp
1969 * and the rest goes in other.
1970 */
1971 while (head != NULL) {
1972 mp = head;
1973 head = head->b_next;
1974 mp->b_next = NULL;
1975
1976 type = OTH;
1977 sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1978
1979 if (is_ether) {
1980 /*
1981 * At this point we can be sure the packet at least
1982 * has an ether header.
1983 */
1984 if (sz1 < sizeof (struct ether_header)) {
1985 mac_rx_drop_pkt(mac_srs, mp);
1986 continue;
1987 }
1988 ehp = (struct ether_header *)mp->b_rptr;
1989
1990 /*
1991 * Determine if this is a VLAN or non-VLAN packet.
1992 */
1993 if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
1994 evhp = (struct ether_vlan_header *)mp->b_rptr;
1995 sap = ntohs(evhp->ether_type);
1996 hdrsize = sizeof (struct ether_vlan_header);
1997
1998 /*
1999 * Check if the VID of the packet, if
2000 * any, belongs to this client.
2001 * Technically, if this packet came up
2002 * via a HW classified ring then we
2003 * don't need to perform this check.
2004 * Perhaps a future optimization.
2005 */
2006 if (!mac_client_check_flow_vid(mcip,
2007 VLAN_ID(ntohs(evhp->ether_tci)))) {
2008 mac_rx_drop_pkt(mac_srs, mp);
2009 continue;
2010 }
2011 } else {
2012 hdrsize = sizeof (struct ether_header);
2013 }
2014 is_unicast =
2015 ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
2016 dstaddr = (uint8_t *)&ehp->ether_dhost;
2017 } else {
2018 mac_header_info_t mhi;
2019
2020 if (mac_header_info((mac_handle_t)mcip->mci_mip,
2021 mp, &mhi) != 0) {
2022 mac_rx_drop_pkt(mac_srs, mp);
2023 continue;
2024 }
2025 hdrsize = mhi.mhi_hdrsize;
2026 sap = mhi.mhi_bindsap;
2027 is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
2028 dstaddr = (uint8_t *)mhi.mhi_daddr;
2029 }
2030
2031 if (!dls_bypass) {
2032 if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
2033 hdrsize, &type, &indx) == -1) {
2034 mac_rx_drop_pkt(mac_srs, mp);
2035 continue;
2036 }
2037
2038 FANOUT_ENQUEUE_MP(headmp[type][indx],
2039 tailmp[type][indx], cnt[type][indx], bw_ctl,
2040 sz[type][indx], sz1, mp);
2041 continue;
2042 }
2043
2044 /*
2045 * If we are using the default Rx ring where H/W or S/W
2046 * classification has not happened, we need to verify if
2047 * this unicast packet really belongs to us.
2048 */
2049 if (sap == ETHERTYPE_IP) {
2050 /*
2051 * If we are H/W classified, but we have promisc
2052 * on, then we need to check for the unicast address.
2053 */
2054 if (hw_classified && mcip->mci_promisc_list != NULL) {
2055 mac_address_t *map;
2056
2057 rw_enter(&mcip->mci_rw_lock, RW_READER);
2058 map = mcip->mci_unicast;
2059 if (bcmp(dstaddr, map->ma_addr,
2060 map->ma_len) == 0)
2061 type = UNDEF;
2062 rw_exit(&mcip->mci_rw_lock);
2063 } else if (is_unicast) {
2064 type = UNDEF;
2065 }
2066 }
2067
2068 /*
2069 * This needs to become a contract with the driver for
2070 * the fast path.
2071 */
2072
2073 ipha = (ipha_t *)(mp->b_rptr + hdrsize);
2074 if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha)) {
2075 type = OTH;
2076 fanout_oth1++;
2077 }
2078
2079 if (type != OTH) {
2080 uint16_t frag_offset_flags;
2081
2082 switch (ipha->ipha_protocol) {
2083 case IPPROTO_TCP:
2084 case IPPROTO_UDP:
2085 case IPPROTO_SCTP:
2086 case IPPROTO_ESP:
2087 ipha_len = IPH_HDR_LENGTH(ipha);
2088 if ((uchar_t *)ipha + ipha_len + PORTS_SIZE >
2089 mp->b_wptr) {
2090 type = OTH;
2091 break;
2092 }
2093 frag_offset_flags =
2094 ntohs(ipha->ipha_fragment_offset_and_flags);
2095 if ((frag_offset_flags &
2096 (IPH_MF | IPH_OFFSET)) != 0) {
2097 type = OTH;
2098 fanout_oth3++;
2099 break;
2100 }
2101 ports_offset = hdrsize + ipha_len;
2102 break;
2103 default:
2104 type = OTH;
2105 fanout_oth4++;
2106 break;
2107 }
2108 }
2109
2110 if (type == OTH) {
2111 if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
2112 hdrsize, &type, &indx) == -1) {
2113 mac_rx_drop_pkt(mac_srs, mp);
2114 continue;
2115 }
2116
2117 FANOUT_ENQUEUE_MP(headmp[type][indx],
2118 tailmp[type][indx], cnt[type][indx], bw_ctl,
2119 sz[type][indx], sz1, mp);
2120 continue;
2121 }
2122
2123 ASSERT(type == UNDEF);
2124
2125 /*
2126 * XXX-Sunay: We should hold srs_lock since ring_count
2127 * below can change. But if we are always called from
2128 * mac_rx_srs_drain and SRS_PROC is set, then we can
2129 * enforce that ring_count can't be changed i.e.
2130 * to change fanout type or ring count, the calling
2131 * thread needs to be behind SRS_PROC.
2132 */
2133 switch (ipha->ipha_protocol) {
2134 case IPPROTO_TCP:
2135 /*
2136 * Note that for ESP, we fanout on SPI and it is at the
2137 * same offset as the 2x16-bit ports. So it is clumped
2138 * along with TCP, UDP and SCTP.
2139 */
2140 hash = HASH_ADDR(ipha->ipha_src, ipha->ipha_dst,
2141 *(uint32_t *)(mp->b_rptr + ports_offset));
2142 indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
2143 type = V4_TCP;
2144 mp->b_rptr += hdrsize;
2145 break;
2146 case IPPROTO_UDP:
2147 case IPPROTO_SCTP:
2148 case IPPROTO_ESP:
2149 if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
2150 hash = HASH_ADDR(ipha->ipha_src, ipha->ipha_dst,
2151 *(uint32_t *)(mp->b_rptr + ports_offset));
2152 indx = COMPUTE_INDEX(hash,
2153 mac_srs->srs_udp_ring_count);
2154 } else {
2155 indx = mac_srs->srs_ind %
2156 mac_srs->srs_udp_ring_count;
2157 mac_srs->srs_ind++;
2158 }
2159 type = V4_UDP;
2160 mp->b_rptr += hdrsize;
2161 break;
2162 default:
2163 indx = 0;
2164 type = OTH;
2165 }
2166
2167 FANOUT_ENQUEUE_MP(headmp[type][indx], tailmp[type][indx],
2168 cnt[type][indx], bw_ctl, sz[type][indx], sz1, mp);
2169 }
2170
2171 for (type = V4_TCP; type < UNDEF; type++) {
2172 int i;
2173
2174 for (i = 0; i < fanout_cnt; i++) {
2175 if (headmp[type][i] != NULL) {
2176 mac_soft_ring_t *softring;
2177
2178 ASSERT(tailmp[type][i]->b_next == NULL);
2179 switch (type) {
2180 case V4_TCP:
2181 softring =
2182 mac_srs->srs_tcp_soft_rings[i];
2183 break;
2184 case V4_UDP:
2185 softring =
2186 mac_srs->srs_udp_soft_rings[i];
2187 break;
2188 case OTH:
2189 softring =
2190 mac_srs->srs_oth_soft_rings[i];
2191 break;
2192 }
2193 mac_rx_soft_ring_process(mcip,
2194 softring, headmp[type][i], tailmp[type][i],
2195 cnt[type][i], sz[type][i]);
2196 }
2197 }
2198 }
2199 }
2200
2201 #define SRS_BYTES_TO_PICKUP 150000
2202 ssize_t max_bytes_to_pickup = SRS_BYTES_TO_PICKUP;
2203
2204 /*
2205 * mac_rx_srs_poll_ring
2206 *
2207 * This SRS Poll thread uses this routine to poll the underlying hardware
2208 * Rx ring to get a chain of packets. It can inline process that chain
2209 * if mac_latency_optimize is set (default) or signal the SRS worker thread
2210 * to do the remaining processing.
2211 *
2212 * Since packets come in the system via interrupt or poll path, we also
2213 * update the stats and deal with promiscous clients here.
2214 */
2215 void
2216 mac_rx_srs_poll_ring(mac_soft_ring_set_t *mac_srs)
2217 {
2218 kmutex_t *lock = &mac_srs->srs_lock;
2219 kcondvar_t *async = &mac_srs->srs_cv;
2220 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
2221 mblk_t *head, *tail, *mp;
2222 callb_cpr_t cprinfo;
2223 ssize_t bytes_to_pickup;
2224 size_t sz;
2225 int count;
2226 mac_client_impl_t *smcip;
2227
2228 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_srs_poll");
2229 mutex_enter(lock);
2230
2231 start:
2232 for (;;) {
2233 if (mac_srs->srs_state & SRS_PAUSE)
2234 goto done;
2235
2236 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2237 cv_wait(async, lock);
2238 CALLB_CPR_SAFE_END(&cprinfo, lock);
2239
2240 if (mac_srs->srs_state & SRS_PAUSE)
2241 goto done;
2242
2243 check_again:
2244 if (mac_srs->srs_type & SRST_BW_CONTROL) {
2245 /*
2246 * We pick as many bytes as we are allowed to queue.
2247 * Its possible that we will exceed the total
2248 * packets queued in case this SRS is part of the
2249 * Rx ring group since > 1 poll thread can be pulling
2250 * upto the max allowed packets at the same time
2251 * but that should be OK.
2252 */
2253 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2254 bytes_to_pickup =
2255 mac_srs->srs_bw->mac_bw_drop_threshold -
2256 mac_srs->srs_bw->mac_bw_sz;
2257 /*
2258 * We shouldn't have been signalled if we
2259 * have 0 or less bytes to pick but since
2260 * some of the bytes accounting is driver
2261 * dependant, we do the safety check.
2262 */
2263 if (bytes_to_pickup < 0)
2264 bytes_to_pickup = 0;
2265 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2266 } else {
2267 /*
2268 * ToDO: Need to change the polling API
2269 * to add a packet count and a flag which
2270 * tells the driver whether we want packets
2271 * based on a count, or bytes, or all the
2272 * packets queued in the driver/HW. This
2273 * way, we never have to check the limits
2274 * on poll path. We truly let only as many
2275 * packets enter the system as we are willing
2276 * to process or queue.
2277 *
2278 * Something along the lines of
2279 * pkts_to_pickup = mac_soft_ring_max_q_cnt -
2280 * mac_srs->srs_poll_pkt_cnt
2281 */
2282
2283 /*
2284 * Since we are not doing B/W control, pick
2285 * as many packets as allowed.
2286 */
2287 bytes_to_pickup = max_bytes_to_pickup;
2288 }
2289
2290 /* Poll the underlying Hardware */
2291 mutex_exit(lock);
2292 head = MAC_HWRING_POLL(mac_srs->srs_ring, (int)bytes_to_pickup);
2293 mutex_enter(lock);
2294
2295 ASSERT((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
2296 SRS_POLL_THR_OWNER);
2297
2298 mp = tail = head;
2299 count = 0;
2300 sz = 0;
2301 while (mp != NULL) {
2302 tail = mp;
2303 sz += msgdsize(mp);
2304 mp = mp->b_next;
2305 count++;
2306 }
2307
2308 if (head != NULL) {
2309 tail->b_next = NULL;
2310 smcip = mac_srs->srs_mcip;
2311
2312 SRS_RX_STAT_UPDATE(mac_srs, pollbytes, sz);
2313 SRS_RX_STAT_UPDATE(mac_srs, pollcnt, count);
2314
2315 /*
2316 * If there are any promiscuous mode callbacks
2317 * defined for this MAC client, pass them a copy
2318 * if appropriate and also update the counters.
2319 */
2320 if (smcip != NULL) {
2321 if (smcip->mci_mip->mi_promisc_list != NULL) {
2322 mutex_exit(lock);
2323 mac_promisc_dispatch(smcip->mci_mip,
2324 head, NULL);
2325 mutex_enter(lock);
2326 }
2327 }
2328 if (mac_srs->srs_type & SRST_BW_CONTROL) {
2329 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2330 mac_srs->srs_bw->mac_bw_polled += sz;
2331 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2332 }
2333 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail,
2334 count, sz);
2335 if (count <= 10)
2336 srs_rx->sr_stat.mrs_chaincntundr10++;
2337 else if (count > 10 && count <= 50)
2338 srs_rx->sr_stat.mrs_chaincnt10to50++;
2339 else
2340 srs_rx->sr_stat.mrs_chaincntover50++;
2341 }
2342
2343 /*
2344 * We are guaranteed that SRS_PROC will be set if we
2345 * are here. Also, poll thread gets to run only if
2346 * the drain was being done by a worker thread although
2347 * its possible that worker thread is still running
2348 * and poll thread was sent down to keep the pipeline
2349 * going instead of doing a complete drain and then
2350 * trying to poll the NIC.
2351 *
2352 * So we need to check SRS_WORKER flag to make sure
2353 * that the worker thread is not processing the queue
2354 * in parallel to us. The flags and conditions are
2355 * protected by the srs_lock to prevent any race. We
2356 * ensure that we don't drop the srs_lock from now
2357 * till the end and similarly we don't drop the srs_lock
2358 * in mac_rx_srs_drain() till similar condition check
2359 * are complete. The mac_rx_srs_drain() needs to ensure
2360 * that SRS_WORKER flag remains set as long as its
2361 * processing the queue.
2362 */
2363 if (!(mac_srs->srs_state & SRS_WORKER) &&
2364 (mac_srs->srs_first != NULL)) {
2365 /*
2366 * We have packets to process and worker thread
2367 * is not running. Check to see if poll thread is
2368 * allowed to process.
2369 */
2370 if (mac_srs->srs_state & SRS_LATENCY_OPT) {
2371 mac_srs->srs_drain_func(mac_srs, SRS_POLL_PROC);
2372 if (!(mac_srs->srs_state & SRS_PAUSE) &&
2373 srs_rx->sr_poll_pkt_cnt <=
2374 srs_rx->sr_lowat) {
2375 srs_rx->sr_poll_again++;
2376 goto check_again;
2377 }
2378 /*
2379 * We are already above low water mark
2380 * so stay in the polling mode but no
2381 * need to poll. Once we dip below
2382 * the polling threshold, the processing
2383 * thread (soft ring) will signal us
2384 * to poll again (MAC_UPDATE_SRS_COUNT)
2385 */
2386 srs_rx->sr_poll_drain_no_poll++;
2387 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
2388 /*
2389 * In B/W control case, its possible
2390 * that the backlog built up due to
2391 * B/W limit being reached and packets
2392 * are queued only in SRS. In this case,
2393 * we should schedule worker thread
2394 * since no one else will wake us up.
2395 */
2396 if ((mac_srs->srs_type & SRST_BW_CONTROL) &&
2397 (mac_srs->srs_tid == NULL)) {
2398 mac_srs->srs_tid =
2399 timeout(mac_srs_fire, mac_srs, 1);
2400 srs_rx->sr_poll_worker_wakeup++;
2401 }
2402 } else {
2403 /*
2404 * Wakeup the worker thread for more processing.
2405 * We optimize for throughput in this case.
2406 */
2407 mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
2408 MAC_SRS_WORKER_WAKEUP(mac_srs);
2409 srs_rx->sr_poll_sig_worker++;
2410 }
2411 } else if ((mac_srs->srs_first == NULL) &&
2412 !(mac_srs->srs_state & SRS_WORKER)) {
2413 /*
2414 * There is nothing queued in SRS and
2415 * no worker thread running. Plus we
2416 * didn't get anything from the H/W
2417 * as well (head == NULL);
2418 */
2419 ASSERT(head == NULL);
2420 mac_srs->srs_state &=
2421 ~(SRS_PROC|SRS_GET_PKTS);
2422
2423 /*
2424 * If we have a packets in soft ring, don't allow
2425 * more packets to come into this SRS by keeping the
2426 * interrupts off but not polling the H/W. The
2427 * poll thread will get signaled as soon as
2428 * srs_poll_pkt_cnt dips below poll threshold.
2429 */
2430 if (srs_rx->sr_poll_pkt_cnt == 0) {
2431 srs_rx->sr_poll_intr_enable++;
2432 MAC_SRS_POLLING_OFF(mac_srs);
2433 } else {
2434 /*
2435 * We know nothing is queued in SRS
2436 * since we are here after checking
2437 * srs_first is NULL. The backlog
2438 * is entirely due to packets queued
2439 * in Soft ring which will wake us up
2440 * and get the interface out of polling
2441 * mode once the backlog dips below
2442 * sr_poll_thres.
2443 */
2444 srs_rx->sr_poll_no_poll++;
2445 }
2446 } else {
2447 /*
2448 * Worker thread is already running.
2449 * Nothing much to do. If the polling
2450 * was enabled, worker thread will deal
2451 * with that.
2452 */
2453 mac_srs->srs_state &= ~SRS_GET_PKTS;
2454 srs_rx->sr_poll_goto_sleep++;
2455 }
2456 }
2457 done:
2458 mac_srs->srs_state |= SRS_POLL_THR_QUIESCED;
2459 cv_signal(&mac_srs->srs_async);
2460 /*
2461 * If this is a temporary quiesce then wait for the restart signal
2462 * from the srs worker. Then clear the flags and signal the srs worker
2463 * to ensure a positive handshake and go back to start.
2464 */
2465 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_POLL_THR_RESTART)))
2466 cv_wait(async, lock);
2467 if (mac_srs->srs_state & SRS_POLL_THR_RESTART) {
2468 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
2469 mac_srs->srs_state &=
2470 ~(SRS_POLL_THR_QUIESCED | SRS_POLL_THR_RESTART);
2471 cv_signal(&mac_srs->srs_async);
2472 goto start;
2473 } else {
2474 mac_srs->srs_state |= SRS_POLL_THR_EXITED;
2475 cv_signal(&mac_srs->srs_async);
2476 CALLB_CPR_EXIT(&cprinfo);
2477 thread_exit();
2478 }
2479 }
2480
2481 /*
2482 * mac_srs_pick_chain
2483 *
2484 * In Bandwidth control case, checks how many packets can be processed
2485 * and return them in a sub chain.
2486 */
2487 static mblk_t *
2488 mac_srs_pick_chain(mac_soft_ring_set_t *mac_srs, mblk_t **chain_tail,
2489 size_t *chain_sz, int *chain_cnt)
2490 {
2491 mblk_t *head = NULL;
2492 mblk_t *tail = NULL;
2493 size_t sz;
2494 size_t tsz = 0;
2495 int cnt = 0;
2496 mblk_t *mp;
2497
2498 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2499 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2500 if (((mac_srs->srs_bw->mac_bw_used + mac_srs->srs_size) <=
2501 mac_srs->srs_bw->mac_bw_limit) ||
2502 (mac_srs->srs_bw->mac_bw_limit == 0)) {
2503 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2504 head = mac_srs->srs_first;
2505 mac_srs->srs_first = NULL;
2506 *chain_tail = mac_srs->srs_last;
2507 mac_srs->srs_last = NULL;
2508 *chain_sz = mac_srs->srs_size;
2509 *chain_cnt = mac_srs->srs_count;
2510 mac_srs->srs_count = 0;
2511 mac_srs->srs_size = 0;
2512 return (head);
2513 }
2514
2515 /*
2516 * Can't clear the entire backlog.
2517 * Need to find how many packets to pick
2518 */
2519 ASSERT(MUTEX_HELD(&mac_srs->srs_bw->mac_bw_lock));
2520 while ((mp = mac_srs->srs_first) != NULL) {
2521 sz = msgdsize(mp);
2522 if ((tsz + sz + mac_srs->srs_bw->mac_bw_used) >
2523 mac_srs->srs_bw->mac_bw_limit) {
2524 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED))
2525 mac_srs->srs_bw->mac_bw_state |=
2526 SRS_BW_ENFORCED;
2527 break;
2528 }
2529
2530 /*
2531 * The _size & cnt is decremented from the softrings
2532 * when they send up the packet for polling to work
2533 * properly.
2534 */
2535 tsz += sz;
2536 cnt++;
2537 mac_srs->srs_count--;
2538 mac_srs->srs_size -= sz;
2539 if (tail != NULL)
2540 tail->b_next = mp;
2541 else
2542 head = mp;
2543 tail = mp;
2544 mac_srs->srs_first = mac_srs->srs_first->b_next;
2545 }
2546 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2547 if (mac_srs->srs_first == NULL)
2548 mac_srs->srs_last = NULL;
2549
2550 if (tail != NULL)
2551 tail->b_next = NULL;
2552 *chain_tail = tail;
2553 *chain_cnt = cnt;
2554 *chain_sz = tsz;
2555
2556 return (head);
2557 }
2558
2559 /*
2560 * mac_rx_srs_drain
2561 *
2562 * The SRS drain routine. Gets to run to clear the queue. Any thread
2563 * (worker, interrupt, poll) can call this based on processing model.
2564 * The first thing we do is disable interrupts if possible and then
2565 * drain the queue. we also try to poll the underlying hardware if
2566 * there is a dedicated hardware Rx ring assigned to this SRS.
2567 *
2568 * There is a equivalent drain routine in bandwidth control mode
2569 * mac_rx_srs_drain_bw. There is some code duplication between the two
2570 * routines but they are highly performance sensitive and are easier
2571 * to read/debug if they stay separate. Any code changes here might
2572 * also apply to mac_rx_srs_drain_bw as well.
2573 */
2574 void
2575 mac_rx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
2576 {
2577 mblk_t *head;
2578 mblk_t *tail;
2579 timeout_id_t tid;
2580 int cnt = 0;
2581 mac_client_impl_t *mcip = mac_srs->srs_mcip;
2582 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
2583
2584 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2585 ASSERT(!(mac_srs->srs_type & SRST_BW_CONTROL));
2586
2587 /* If we are blanked i.e. can't do upcalls, then we are done */
2588 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
2589 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
2590 (mac_srs->srs_state & SRS_PAUSE));
2591 goto out;
2592 }
2593
2594 if (mac_srs->srs_first == NULL)
2595 goto out;
2596
2597 if (!(mac_srs->srs_state & SRS_LATENCY_OPT) &&
2598 (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)) {
2599 /*
2600 * In the normal case, the SRS worker thread does no
2601 * work and we wait for a backlog to build up before
2602 * we switch into polling mode. In case we are
2603 * optimizing for throughput, we use the worker thread
2604 * as well. The goal is to let worker thread process
2605 * the queue and poll thread to feed packets into
2606 * the queue. As such, we should signal the poll
2607 * thread to try and get more packets.
2608 *
2609 * We could have pulled this check in the POLL_RING
2610 * macro itself but keeping it explicit here makes
2611 * the architecture more human understandable.
2612 */
2613 MAC_SRS_POLL_RING(mac_srs);
2614 }
2615
2616 again:
2617 head = mac_srs->srs_first;
2618 mac_srs->srs_first = NULL;
2619 tail = mac_srs->srs_last;
2620 mac_srs->srs_last = NULL;
2621 cnt = mac_srs->srs_count;
2622 mac_srs->srs_count = 0;
2623
2624 ASSERT(head != NULL);
2625 ASSERT(tail != NULL);
2626
2627 if ((tid = mac_srs->srs_tid) != NULL)
2628 mac_srs->srs_tid = NULL;
2629
2630 mac_srs->srs_state |= (SRS_PROC|proc_type);
2631
2632 /*
2633 * mcip is NULL for broadcast and multicast flows. The promisc
2634 * callbacks for broadcast and multicast packets are delivered from
2635 * mac_rx() and we don't need to worry about that case in this path
2636 */
2637 if (mcip != NULL) {
2638 if (mcip->mci_promisc_list != NULL) {
2639 mutex_exit(&mac_srs->srs_lock);
2640 mac_promisc_client_dispatch(mcip, head);
2641 mutex_enter(&mac_srs->srs_lock);
2642 }
2643 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
2644 mutex_exit(&mac_srs->srs_lock);
2645 mac_protect_intercept_dynamic(mcip, head);
2646 mutex_enter(&mac_srs->srs_lock);
2647 }
2648 }
2649
2650 /*
2651 * Check if SRS itself is doing the processing. This direct
2652 * path applies only when subflows are present.
2653 */
2654 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
2655 mac_direct_rx_t proc;
2656 void *arg1;
2657 mac_resource_handle_t arg2;
2658
2659 /*
2660 * This is the case when a Rx is directly
2661 * assigned and we have a fully classified
2662 * protocol chain. We can deal with it in
2663 * one shot.
2664 */
2665 proc = srs_rx->sr_func;
2666 arg1 = srs_rx->sr_arg1;
2667 arg2 = srs_rx->sr_arg2;
2668
2669 mac_srs->srs_state |= SRS_CLIENT_PROC;
2670 mutex_exit(&mac_srs->srs_lock);
2671 if (tid != NULL) {
2672 (void) untimeout(tid);
2673 tid = NULL;
2674 }
2675
2676 proc(arg1, arg2, head, NULL);
2677 /*
2678 * Decrement the size and count here itelf
2679 * since the packet has been processed.
2680 */
2681 mutex_enter(&mac_srs->srs_lock);
2682 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
2683 if (mac_srs->srs_state & SRS_CLIENT_WAIT)
2684 cv_signal(&mac_srs->srs_client_cv);
2685 mac_srs->srs_state &= ~SRS_CLIENT_PROC;
2686 } else {
2687 /* Some kind of softrings based fanout is required */
2688 mutex_exit(&mac_srs->srs_lock);
2689 if (tid != NULL) {
2690 (void) untimeout(tid);
2691 tid = NULL;
2692 }
2693
2694 /*
2695 * Since the fanout routines can deal with chains,
2696 * shoot the entire chain up.
2697 */
2698 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
2699 mac_rx_srs_fanout(mac_srs, head);
2700 else
2701 mac_rx_srs_proto_fanout(mac_srs, head);
2702 mutex_enter(&mac_srs->srs_lock);
2703 }
2704
2705 if (!(mac_srs->srs_state & (SRS_BLANK|SRS_PAUSE)) &&
2706 (mac_srs->srs_first != NULL)) {
2707 /*
2708 * More packets arrived while we were clearing the
2709 * SRS. This can be possible because of one of
2710 * three conditions below:
2711 * 1) The driver is using multiple worker threads
2712 * to send the packets to us.
2713 * 2) The driver has a race in switching
2714 * between interrupt and polling mode or
2715 * 3) Packets are arriving in this SRS via the
2716 * S/W classification as well.
2717 *
2718 * We should switch to polling mode and see if we
2719 * need to send the poll thread down. Also, signal
2720 * the worker thread to process whats just arrived.
2721 */
2722 MAC_SRS_POLLING_ON(mac_srs);
2723 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat) {
2724 srs_rx->sr_drain_poll_sig++;
2725 MAC_SRS_POLL_RING(mac_srs);
2726 }
2727
2728 /*
2729 * If we didn't signal the poll thread, we need
2730 * to deal with the pending packets ourselves.
2731 */
2732 if (proc_type == SRS_WORKER) {
2733 srs_rx->sr_drain_again++;
2734 goto again;
2735 } else {
2736 srs_rx->sr_drain_worker_sig++;
2737 cv_signal(&mac_srs->srs_async);
2738 }
2739 }
2740
2741 out:
2742 if (mac_srs->srs_state & SRS_GET_PKTS) {
2743 /*
2744 * Poll thread is already running. Leave the
2745 * SRS_RPOC set and hand over the control to
2746 * poll thread.
2747 */
2748 mac_srs->srs_state &= ~proc_type;
2749 srs_rx->sr_drain_poll_running++;
2750 return;
2751 }
2752
2753 /*
2754 * Even if there are no packets queued in SRS, we
2755 * need to make sure that the shared counter is
2756 * clear and any associated softrings have cleared
2757 * all the backlog. Otherwise, leave the interface
2758 * in polling mode and the poll thread will get
2759 * signalled once the count goes down to zero.
2760 *
2761 * If someone is already draining the queue (SRS_PROC is
2762 * set) when the srs_poll_pkt_cnt goes down to zero,
2763 * then it means that drain is already running and we
2764 * will turn off polling at that time if there is
2765 * no backlog.
2766 *
2767 * As long as there are packets queued either
2768 * in soft ring set or its soft rings, we will leave
2769 * the interface in polling mode (even if the drain
2770 * was done being the interrupt thread). We signal
2771 * the poll thread as well if we have dipped below
2772 * low water mark.
2773 *
2774 * NOTE: We can't use the MAC_SRS_POLLING_ON macro
2775 * since that turn polling on only for worker thread.
2776 * Its not worth turning polling on for interrupt
2777 * thread (since NIC will not issue another interrupt)
2778 * unless a backlog builds up.
2779 */
2780 if ((srs_rx->sr_poll_pkt_cnt > 0) &&
2781 (mac_srs->srs_state & SRS_POLLING_CAPAB)) {
2782 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2783 srs_rx->sr_drain_keep_polling++;
2784 MAC_SRS_POLLING_ON(mac_srs);
2785 if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)
2786 MAC_SRS_POLL_RING(mac_srs);
2787 return;
2788 }
2789
2790 /* Nothing else to do. Get out of poll mode */
2791 MAC_SRS_POLLING_OFF(mac_srs);
2792 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2793 srs_rx->sr_drain_finish_intr++;
2794 }
2795
2796 /*
2797 * mac_rx_srs_drain_bw
2798 *
2799 * The SRS BW drain routine. Gets to run to clear the queue. Any thread
2800 * (worker, interrupt, poll) can call this based on processing model.
2801 * The first thing we do is disable interrupts if possible and then
2802 * drain the queue. we also try to poll the underlying hardware if
2803 * there is a dedicated hardware Rx ring assigned to this SRS.
2804 *
2805 * There is a equivalent drain routine in non bandwidth control mode
2806 * mac_rx_srs_drain. There is some code duplication between the two
2807 * routines but they are highly performance sensitive and are easier
2808 * to read/debug if they stay separate. Any code changes here might
2809 * also apply to mac_rx_srs_drain as well.
2810 */
2811 void
2812 mac_rx_srs_drain_bw(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
2813 {
2814 mblk_t *head;
2815 mblk_t *tail;
2816 timeout_id_t tid;
2817 size_t sz = 0;
2818 int cnt = 0;
2819 mac_client_impl_t *mcip = mac_srs->srs_mcip;
2820 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
2821 clock_t now;
2822
2823 ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2824 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
2825 again:
2826 /* Check if we are doing B/W control */
2827 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2828 now = ddi_get_lbolt();
2829 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
2830 mac_srs->srs_bw->mac_bw_curr_time = now;
2831 mac_srs->srs_bw->mac_bw_used = 0;
2832 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
2833 mac_srs->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED;
2834 } else if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) {
2835 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2836 goto done;
2837 } else if (mac_srs->srs_bw->mac_bw_used >
2838 mac_srs->srs_bw->mac_bw_limit) {
2839 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
2840 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2841 goto done;
2842 }
2843 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2844
2845 /* If we are blanked i.e. can't do upcalls, then we are done */
2846 if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
2847 ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
2848 (mac_srs->srs_state & SRS_PAUSE));
2849 goto done;
2850 }
2851
2852 sz = 0;
2853 cnt = 0;
2854 if ((head = mac_srs_pick_chain(mac_srs, &tail, &sz, &cnt)) == NULL) {
2855 /*
2856 * We couldn't pick up a single packet.
2857 */
2858 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2859 if ((mac_srs->srs_bw->mac_bw_used == 0) &&
2860 (mac_srs->srs_size != 0) &&
2861 !(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2862 /*
2863 * Seems like configured B/W doesn't
2864 * even allow processing of 1 packet
2865 * per tick.
2866 *
2867 * XXX: raise the limit to processing
2868 * at least 1 packet per tick.
2869 */
2870 mac_srs->srs_bw->mac_bw_limit +=
2871 mac_srs->srs_bw->mac_bw_limit;
2872 mac_srs->srs_bw->mac_bw_drop_threshold +=
2873 mac_srs->srs_bw->mac_bw_drop_threshold;
2874 cmn_err(CE_NOTE, "mac_rx_srs_drain: srs(%p) "
2875 "raised B/W limit to %d since not even a "
2876 "single packet can be processed per "
2877 "tick %d\n", (void *)mac_srs,
2878 (int)mac_srs->srs_bw->mac_bw_limit,
2879 (int)msgdsize(mac_srs->srs_first));
2880 }
2881 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2882 goto done;
2883 }
2884
2885 ASSERT(head != NULL);
2886 ASSERT(tail != NULL);
2887
2888 /* zero bandwidth: drop all and return to interrupt mode */
2889 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2890 if (mac_srs->srs_bw->mac_bw_limit == 0) {
2891 srs_rx->sr_stat.mrs_sdrops += cnt;
2892 ASSERT(mac_srs->srs_bw->mac_bw_sz >= sz);
2893 mac_srs->srs_bw->mac_bw_sz -= sz;
2894 mac_srs->srs_bw->mac_bw_drop_bytes += sz;
2895 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2896 mac_pkt_drop(NULL, NULL, head, B_FALSE);
2897 goto leave_poll;
2898 } else {
2899 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2900 }
2901
2902 if ((tid = mac_srs->srs_tid) != NULL)
2903 mac_srs->srs_tid = NULL;
2904
2905 mac_srs->srs_state |= (SRS_PROC|proc_type);
2906 MAC_SRS_WORKER_POLLING_ON(mac_srs);
2907
2908 /*
2909 * mcip is NULL for broadcast and multicast flows. The promisc
2910 * callbacks for broadcast and multicast packets are delivered from
2911 * mac_rx() and we don't need to worry about that case in this path
2912 */
2913 if (mcip != NULL) {
2914 if (mcip->mci_promisc_list != NULL) {
2915 mutex_exit(&mac_srs->srs_lock);
2916 mac_promisc_client_dispatch(mcip, head);
2917 mutex_enter(&mac_srs->srs_lock);
2918 }
2919 if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
2920 mutex_exit(&mac_srs->srs_lock);
2921 mac_protect_intercept_dynamic(mcip, head);
2922 mutex_enter(&mac_srs->srs_lock);
2923 }
2924 }
2925
2926 /*
2927 * Check if SRS itself is doing the processing
2928 * This direct path does not apply when subflows are present. In this
2929 * case, packets need to be dispatched to a soft ring according to the
2930 * flow's bandwidth and other resources contraints.
2931 */
2932 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
2933 mac_direct_rx_t proc;
2934 void *arg1;
2935 mac_resource_handle_t arg2;
2936
2937 /*
2938 * This is the case when a Rx is directly
2939 * assigned and we have a fully classified
2940 * protocol chain. We can deal with it in
2941 * one shot.
2942 */
2943 proc = srs_rx->sr_func;
2944 arg1 = srs_rx->sr_arg1;
2945 arg2 = srs_rx->sr_arg2;
2946
2947 mac_srs->srs_state |= SRS_CLIENT_PROC;
2948 mutex_exit(&mac_srs->srs_lock);
2949 if (tid != NULL) {
2950 (void) untimeout(tid);
2951 tid = NULL;
2952 }
2953
2954 proc(arg1, arg2, head, NULL);
2955 /*
2956 * Decrement the size and count here itelf
2957 * since the packet has been processed.
2958 */
2959 mutex_enter(&mac_srs->srs_lock);
2960 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
2961 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
2962
2963 if (mac_srs->srs_state & SRS_CLIENT_WAIT)
2964 cv_signal(&mac_srs->srs_client_cv);
2965 mac_srs->srs_state &= ~SRS_CLIENT_PROC;
2966 } else {
2967 /* Some kind of softrings based fanout is required */
2968 mutex_exit(&mac_srs->srs_lock);
2969 if (tid != NULL) {
2970 (void) untimeout(tid);
2971 tid = NULL;
2972 }
2973
2974 /*
2975 * Since the fanout routines can deal with chains,
2976 * shoot the entire chain up.
2977 */
2978 if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
2979 mac_rx_srs_fanout(mac_srs, head);
2980 else
2981 mac_rx_srs_proto_fanout(mac_srs, head);
2982 mutex_enter(&mac_srs->srs_lock);
2983 }
2984
2985 /*
2986 * Send the poll thread to pick up any packets arrived
2987 * so far. This also serves as the last check in case
2988 * nothing else is queued in the SRS. The poll thread
2989 * is signalled only in the case the drain was done
2990 * by the worker thread and SRS_WORKER is set. The
2991 * worker thread can run in parallel as long as the
2992 * SRS_WORKER flag is set. We we have nothing else to
2993 * process, we can exit while leaving SRS_PROC set
2994 * which gives the poll thread control to process and
2995 * cleanup once it returns from the NIC.
2996 *
2997 * If we have nothing else to process, we need to
2998 * ensure that we keep holding the srs_lock till
2999 * all the checks below are done and control is
3000 * handed to the poll thread if it was running.
3001 */
3002 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
3003 if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
3004 if (mac_srs->srs_first != NULL) {
3005 if (proc_type == SRS_WORKER) {
3006 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3007 if (srs_rx->sr_poll_pkt_cnt <=
3008 srs_rx->sr_lowat)
3009 MAC_SRS_POLL_RING(mac_srs);
3010 goto again;
3011 } else {
3012 cv_signal(&mac_srs->srs_async);
3013 }
3014 }
3015 }
3016 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3017
3018 done:
3019
3020 if (mac_srs->srs_state & SRS_GET_PKTS) {
3021 /*
3022 * Poll thread is already running. Leave the
3023 * SRS_RPOC set and hand over the control to
3024 * poll thread.
3025 */
3026 mac_srs->srs_state &= ~proc_type;
3027 return;
3028 }
3029
3030 /*
3031 * If we can't process packets because we have exceeded
3032 * B/W limit for this tick, just set the timeout
3033 * and leave.
3034 *
3035 * Even if there are no packets queued in SRS, we
3036 * need to make sure that the shared counter is
3037 * clear and any associated softrings have cleared
3038 * all the backlog. Otherwise, leave the interface
3039 * in polling mode and the poll thread will get
3040 * signalled once the count goes down to zero.
3041 *
3042 * If someone is already draining the queue (SRS_PROC is
3043 * set) when the srs_poll_pkt_cnt goes down to zero,
3044 * then it means that drain is already running and we
3045 * will turn off polling at that time if there is
3046 * no backlog. As long as there are packets queued either
3047 * is soft ring set or its soft rings, we will leave
3048 * the interface in polling mode.
3049 */
3050 mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
3051 if ((mac_srs->srs_state & SRS_POLLING_CAPAB) &&
3052 ((mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) ||
3053 (srs_rx->sr_poll_pkt_cnt > 0))) {
3054 MAC_SRS_POLLING_ON(mac_srs);
3055 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
3056 if ((mac_srs->srs_first != NULL) &&
3057 (mac_srs->srs_tid == NULL))
3058 mac_srs->srs_tid = timeout(mac_srs_fire,
3059 mac_srs, 1);
3060 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3061 return;
3062 }
3063 mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3064
3065 leave_poll:
3066
3067 /* Nothing else to do. Get out of poll mode */
3068 MAC_SRS_POLLING_OFF(mac_srs);
3069 mac_srs->srs_state &= ~(SRS_PROC|proc_type);
3070 }
3071
3072 /*
3073 * mac_srs_worker
3074 *
3075 * The SRS worker routine. Drains the queue when no one else is
3076 * processing it.
3077 */
3078 void
3079 mac_srs_worker(mac_soft_ring_set_t *mac_srs)
3080 {
3081 kmutex_t *lock = &mac_srs->srs_lock;
3082 kcondvar_t *async = &mac_srs->srs_async;
3083 callb_cpr_t cprinfo;
3084 boolean_t bw_ctl_flag;
3085
3086 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "srs_worker");
3087 mutex_enter(lock);
3088
3089 start:
3090 for (;;) {
3091 bw_ctl_flag = B_FALSE;
3092 if (mac_srs->srs_type & SRST_BW_CONTROL) {
3093 MAC_SRS_BW_LOCK(mac_srs);
3094 MAC_SRS_CHECK_BW_CONTROL(mac_srs);
3095 if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
3096 bw_ctl_flag = B_TRUE;
3097 MAC_SRS_BW_UNLOCK(mac_srs);
3098 }
3099 /*
3100 * The SRS_BW_ENFORCED flag may change since we have dropped
3101 * the mac_bw_lock. However the drain function can handle both
3102 * a drainable SRS or a bandwidth controlled SRS, and the
3103 * effect of scheduling a timeout is to wakeup the worker
3104 * thread which in turn will call the drain function. Since
3105 * we release the srs_lock atomically only in the cv_wait there
3106 * isn't a fear of waiting for ever.
3107 */
3108 while (((mac_srs->srs_state & SRS_PROC) ||
3109 (mac_srs->srs_first == NULL) || bw_ctl_flag ||
3110 (mac_srs->srs_state & SRS_TX_BLOCKED)) &&
3111 !(mac_srs->srs_state & SRS_PAUSE)) {
3112 /*
3113 * If we have packets queued and we are here
3114 * because B/W control is in place, we better
3115 * schedule the worker wakeup after 1 tick
3116 * to see if bandwidth control can be relaxed.
3117 */
3118 if (bw_ctl_flag && mac_srs->srs_tid == NULL) {
3119 /*
3120 * We need to ensure that a timer is already
3121 * scheduled or we force schedule one for
3122 * later so that we can continue processing
3123 * after this quanta is over.
3124 */
3125 mac_srs->srs_tid = timeout(mac_srs_fire,
3126 mac_srs, 1);
3127 }
3128 wait:
3129 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3130 cv_wait(async, lock);
3131 CALLB_CPR_SAFE_END(&cprinfo, lock);
3132
3133 if (mac_srs->srs_state & SRS_PAUSE)
3134 goto done;
3135 if (mac_srs->srs_state & SRS_PROC)
3136 goto wait;
3137
3138 if (mac_srs->srs_first != NULL &&
3139 mac_srs->srs_type & SRST_BW_CONTROL) {
3140 MAC_SRS_BW_LOCK(mac_srs);
3141 if (mac_srs->srs_bw->mac_bw_state &
3142 SRS_BW_ENFORCED) {
3143 MAC_SRS_CHECK_BW_CONTROL(mac_srs);
3144 }
3145 bw_ctl_flag = mac_srs->srs_bw->mac_bw_state &
3146 SRS_BW_ENFORCED;
3147 MAC_SRS_BW_UNLOCK(mac_srs);
3148 }
3149 }
3150
3151 if (mac_srs->srs_state & SRS_PAUSE)
3152 goto done;
3153 mac_srs->srs_drain_func(mac_srs, SRS_WORKER);
3154 }
3155 done:
3156 /*
3157 * The Rx SRS quiesce logic first cuts off packet supply to the SRS
3158 * from both hard and soft classifications and waits for such threads
3159 * to finish before signaling the worker. So at this point the only
3160 * thread left that could be competing with the worker is the poll
3161 * thread. In the case of Tx, there shouldn't be any thread holding
3162 * SRS_PROC at this point.
3163 */
3164 if (!(mac_srs->srs_state & SRS_PROC)) {
3165 mac_srs->srs_state |= SRS_PROC;
3166 } else {
3167 ASSERT((mac_srs->srs_type & SRST_TX) == 0);
3168 /*
3169 * Poll thread still owns the SRS and is still running
3170 */
3171 ASSERT((mac_srs->srs_poll_thr == NULL) ||
3172 ((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
3173 SRS_POLL_THR_OWNER));
3174 }
3175 mac_srs_worker_quiesce(mac_srs);
3176 /*
3177 * Wait for the SRS_RESTART or SRS_CONDEMNED signal from the initiator
3178 * of the quiesce operation
3179 */
3180 while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_RESTART)))
3181 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3182
3183 if (mac_srs->srs_state & SRS_RESTART) {
3184 ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
3185 mac_srs_worker_restart(mac_srs);
3186 mac_srs->srs_state &= ~SRS_PROC;
3187 goto start;
3188 }
3189
3190 if (!(mac_srs->srs_state & SRS_CONDEMNED_DONE))
3191 mac_srs_worker_quiesce(mac_srs);
3192
3193 mac_srs->srs_state &= ~SRS_PROC;
3194 /* The macro drops the srs_lock */
3195 CALLB_CPR_EXIT(&cprinfo);
3196 thread_exit();
3197 }
3198
3199 /*
3200 * mac_rx_srs_subflow_process
3201 *
3202 * Receive side routine called from interrupt path when there are
3203 * sub flows present on this SRS.
3204 */
3205 /* ARGSUSED */
3206 void
3207 mac_rx_srs_subflow_process(void *arg, mac_resource_handle_t srs,
3208 mblk_t *mp_chain, boolean_t loopback)
3209 {
3210 flow_entry_t *flent = NULL;
3211 flow_entry_t *prev_flent = NULL;
3212 mblk_t *mp = NULL;
3213 mblk_t *tail = NULL;
3214 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs;
3215 mac_client_impl_t *mcip;
3216
3217 mcip = mac_srs->srs_mcip;
3218 ASSERT(mcip != NULL);
3219
3220 /*
3221 * We need to determine the SRS for every packet
3222 * by walking the flow table, if we don't get any,
3223 * then we proceed using the SRS we came with.
3224 */
3225 mp = tail = mp_chain;
3226 while (mp != NULL) {
3227
3228 /*
3229 * We will increment the stats for the mactching subflow.
3230 * when we get the bytes/pkt count for the classified packets
3231 * later in mac_rx_srs_process.
3232 */
3233 (void) mac_flow_lookup(mcip->mci_subflow_tab, mp,
3234 FLOW_INBOUND, &flent);
3235
3236 if (mp == mp_chain || flent == prev_flent) {
3237 if (prev_flent != NULL)
3238 FLOW_REFRELE(prev_flent);
3239 prev_flent = flent;
3240 flent = NULL;
3241 tail = mp;
3242 mp = mp->b_next;
3243 continue;
3244 }
3245 tail->b_next = NULL;
3246 /*
3247 * A null indicates, this is for the mac_srs itself.
3248 * XXX-venu : probably assert for fe_rx_srs_cnt == 0.
3249 */
3250 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
3251 mac_rx_srs_process(arg,
3252 (mac_resource_handle_t)mac_srs, mp_chain,
3253 loopback);
3254 } else {
3255 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
3256 prev_flent->fe_cb_arg2, mp_chain, loopback);
3257 FLOW_REFRELE(prev_flent);
3258 }
3259 prev_flent = flent;
3260 flent = NULL;
3261 mp_chain = mp;
3262 tail = mp;
3263 mp = mp->b_next;
3264 }
3265 /* Last chain */
3266 ASSERT(mp_chain != NULL);
3267 if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
3268 mac_rx_srs_process(arg,
3269 (mac_resource_handle_t)mac_srs, mp_chain, loopback);
3270 } else {
3271 (prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
3272 prev_flent->fe_cb_arg2, mp_chain, loopback);
3273 FLOW_REFRELE(prev_flent);
3274 }
3275 }
3276
3277 /*
3278 * mac_rx_srs_process
3279 *
3280 * Receive side routine called from the interrupt path.
3281 *
3282 * loopback is set to force a context switch on the loopback
3283 * path between MAC clients.
3284 */
3285 /* ARGSUSED */
3286 void
3287 mac_rx_srs_process(void *arg, mac_resource_handle_t srs, mblk_t *mp_chain,
3288 boolean_t loopback)
3289 {
3290 mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)srs;
3291 mblk_t *mp, *tail, *head;
3292 int count = 0;
3293 int count1;
3294 size_t sz = 0;
3295 size_t chain_sz, sz1;
3296 mac_bw_ctl_t *mac_bw;
3297 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
3298
3299 /*
3300 * Set the tail, count and sz. We set the sz irrespective
3301 * of whether we are doing B/W control or not for the
3302 * purpose of updating the stats.
3303 */
3304 mp = tail = mp_chain;
3305 while (mp != NULL) {
3306 tail = mp;
3307 count++;
3308 sz += msgdsize(mp);
3309 mp = mp->b_next;
3310 }
3311
3312 mutex_enter(&mac_srs->srs_lock);
3313
3314 if (loopback) {
3315 SRS_RX_STAT_UPDATE(mac_srs, lclbytes, sz);
3316 SRS_RX_STAT_UPDATE(mac_srs, lclcnt, count);
3317
3318 } else {
3319 SRS_RX_STAT_UPDATE(mac_srs, intrbytes, sz);
3320 SRS_RX_STAT_UPDATE(mac_srs, intrcnt, count);
3321 }
3322
3323 /*
3324 * If the SRS in already being processed; has been blanked;
3325 * can be processed by worker thread only; or the B/W limit
3326 * has been reached, then queue the chain and check if
3327 * worker thread needs to be awakend.
3328 */
3329 if (mac_srs->srs_type & SRST_BW_CONTROL) {
3330 mac_bw = mac_srs->srs_bw;
3331 ASSERT(mac_bw != NULL);
3332 mutex_enter(&mac_bw->mac_bw_lock);
3333 mac_bw->mac_bw_intr += sz;
3334 if (mac_bw->mac_bw_limit == 0) {
3335 /* zero bandwidth: drop all */
3336 srs_rx->sr_stat.mrs_sdrops += count;
3337 mac_bw->mac_bw_drop_bytes += sz;
3338 mutex_exit(&mac_bw->mac_bw_lock);
3339 mutex_exit(&mac_srs->srs_lock);
3340 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
3341 return;
3342 } else {
3343 if ((mac_bw->mac_bw_sz + sz) <=
3344 mac_bw->mac_bw_drop_threshold) {
3345 mutex_exit(&mac_bw->mac_bw_lock);
3346 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain,
3347 tail, count, sz);
3348 } else {
3349 mp = mp_chain;
3350 chain_sz = 0;
3351 count1 = 0;
3352 tail = NULL;
3353 head = NULL;
3354 while (mp != NULL) {
3355 sz1 = msgdsize(mp);
3356 if (mac_bw->mac_bw_sz + chain_sz + sz1 >
3357 mac_bw->mac_bw_drop_threshold)
3358 break;
3359 chain_sz += sz1;
3360 count1++;
3361 tail = mp;
3362 mp = mp->b_next;
3363 }
3364 mutex_exit(&mac_bw->mac_bw_lock);
3365 if (tail != NULL) {
3366 head = tail->b_next;
3367 tail->b_next = NULL;
3368 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs,
3369 mp_chain, tail, count1, chain_sz);
3370 sz -= chain_sz;
3371 count -= count1;
3372 } else {
3373 /* Can't pick up any */
3374 head = mp_chain;
3375 }
3376 if (head != NULL) {
3377 /* Drop any packet over the threshold */
3378 srs_rx->sr_stat.mrs_sdrops += count;
3379 mutex_enter(&mac_bw->mac_bw_lock);
3380 mac_bw->mac_bw_drop_bytes += sz;
3381 mutex_exit(&mac_bw->mac_bw_lock);
3382 freemsgchain(head);
3383 }
3384 }
3385 MAC_SRS_WORKER_WAKEUP(mac_srs);
3386 mutex_exit(&mac_srs->srs_lock);
3387 return;
3388 }
3389 }
3390
3391 /*
3392 * If the total number of packets queued in the SRS and
3393 * its associated soft rings exceeds the max allowed,
3394 * then drop the chain. If we are polling capable, this
3395 * shouldn't be happening.
3396 */
3397 if (!(mac_srs->srs_type & SRST_BW_CONTROL) &&
3398 (srs_rx->sr_poll_pkt_cnt > srs_rx->sr_hiwat)) {
3399 mac_bw = mac_srs->srs_bw;
3400 srs_rx->sr_stat.mrs_sdrops += count;
3401 mutex_enter(&mac_bw->mac_bw_lock);
3402 mac_bw->mac_bw_drop_bytes += sz;
3403 mutex_exit(&mac_bw->mac_bw_lock);
3404 freemsgchain(mp_chain);
3405 mutex_exit(&mac_srs->srs_lock);
3406 return;
3407 }
3408
3409 MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain, tail, count, sz);
3410
3411 if (!(mac_srs->srs_state & SRS_PROC)) {
3412 /*
3413 * If we are coming via loopback, if we are not optimizing for
3414 * latency, or if our stack is running deep, we should signal
3415 * the worker thread.
3416 */
3417 if (loopback || !(mac_srs->srs_state & SRS_LATENCY_OPT) ||
3418 MAC_RX_SRS_TOODEEP()) {
3419 /*
3420 * For loopback, We need to let the worker take
3421 * over as we don't want to continue in the same
3422 * thread even if we can. This could lead to stack
3423 * overflows and may also end up using
3424 * resources (cpu) incorrectly.
3425 */
3426 cv_signal(&mac_srs->srs_async);
3427 } else {
3428 /*
3429 * Seems like no one is processing the SRS and
3430 * there is no backlog. We also inline process
3431 * our packet if its a single packet in non
3432 * latency optimized case (in latency optimized
3433 * case, we inline process chains of any size).
3434 */
3435 mac_srs->srs_drain_func(mac_srs, SRS_PROC_FAST);
3436 }
3437 }
3438 mutex_exit(&mac_srs->srs_lock);
3439 }
3440
3441 /* TX SIDE ROUTINES (RUNTIME) */
3442
3443 /*
3444 * mac_tx_srs_no_desc
3445 *
3446 * This routine is called by Tx single ring default mode
3447 * when Tx ring runs out of descs.
3448 */
3449 mac_tx_cookie_t
3450 mac_tx_srs_no_desc(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3451 uint16_t flag, mblk_t **ret_mp)
3452 {
3453 mac_tx_cookie_t cookie = 0;
3454 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3455 boolean_t wakeup_worker = B_TRUE;
3456 uint32_t tx_mode = srs_tx->st_mode;
3457 int cnt, sz;
3458 mblk_t *tail;
3459
3460 ASSERT(tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_BW);
3461 if (flag & MAC_DROP_ON_NO_DESC) {
3462 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3463 } else {
3464 if (mac_srs->srs_first != NULL)
3465 wakeup_worker = B_FALSE;
3466 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3467 if (flag & MAC_TX_NO_ENQUEUE) {
3468 /*
3469 * If TX_QUEUED is not set, queue the
3470 * packet and let mac_tx_srs_drain()
3471 * set the TX_BLOCKED bit for the
3472 * reasons explained above. Otherwise,
3473 * return the mblks.
3474 */
3475 if (wakeup_worker) {
3476 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3477 mp_chain, tail, cnt, sz);
3478 } else {
3479 MAC_TX_SET_NO_ENQUEUE(mac_srs,
3480 mp_chain, ret_mp, cookie);
3481 }
3482 } else {
3483 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
3484 tail, cnt, sz, cookie);
3485 }
3486 if (wakeup_worker)
3487 cv_signal(&mac_srs->srs_async);
3488 }
3489 return (cookie);
3490 }
3491
3492 /*
3493 * mac_tx_srs_enqueue
3494 *
3495 * This routine is called when Tx SRS is operating in either serializer
3496 * or bandwidth mode. In serializer mode, a packet will get enqueued
3497 * when a thread cannot enter SRS exclusively. In bandwidth mode,
3498 * packets gets queued if allowed byte-count limit for a tick is
3499 * exceeded. The action that gets taken when MAC_DROP_ON_NO_DESC and
3500 * MAC_TX_NO_ENQUEUE is set is different than when operaing in either
3501 * the default mode or fanout mode. Here packets get dropped or
3502 * returned back to the caller only after hi-watermark worth of data
3503 * is queued.
3504 */
3505 static mac_tx_cookie_t
3506 mac_tx_srs_enqueue(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3507 uint16_t flag, uintptr_t fanout_hint, mblk_t **ret_mp)
3508 {
3509 mac_tx_cookie_t cookie = 0;
3510 int cnt, sz;
3511 mblk_t *tail;
3512 boolean_t wakeup_worker = B_TRUE;
3513
3514 /*
3515 * Ignore fanout hint if we don't have multiple tx rings.
3516 */
3517 if (!MAC_TX_SOFT_RINGS(mac_srs))
3518 fanout_hint = 0;
3519
3520 if (mac_srs->srs_first != NULL)
3521 wakeup_worker = B_FALSE;
3522 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3523 if (flag & MAC_DROP_ON_NO_DESC) {
3524 if (mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) {
3525 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3526 } else {
3527 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3528 mp_chain, tail, cnt, sz);
3529 }
3530 } else if (flag & MAC_TX_NO_ENQUEUE) {
3531 if ((mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) ||
3532 (mac_srs->srs_state & SRS_TX_WAKEUP_CLIENT)) {
3533 MAC_TX_SET_NO_ENQUEUE(mac_srs, mp_chain,
3534 ret_mp, cookie);
3535 } else {
3536 mp_chain->b_prev = (mblk_t *)fanout_hint;
3537 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3538 mp_chain, tail, cnt, sz);
3539 }
3540 } else {
3541 /*
3542 * If you are BW_ENFORCED, just enqueue the
3543 * packet. srs_worker will drain it at the
3544 * prescribed rate. Before enqueueing, save
3545 * the fanout hint.
3546 */
3547 mp_chain->b_prev = (mblk_t *)fanout_hint;
3548 MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
3549 tail, cnt, sz, cookie);
3550 }
3551 if (wakeup_worker)
3552 cv_signal(&mac_srs->srs_async);
3553 return (cookie);
3554 }
3555
3556 /*
3557 * There are seven tx modes:
3558 *
3559 * 1) Default mode (SRS_TX_DEFAULT)
3560 * 2) Serialization mode (SRS_TX_SERIALIZE)
3561 * 3) Fanout mode (SRS_TX_FANOUT)
3562 * 4) Bandwdith mode (SRS_TX_BW)
3563 * 5) Fanout and Bandwidth mode (SRS_TX_BW_FANOUT)
3564 * 6) aggr Tx mode (SRS_TX_AGGR)
3565 * 7) aggr Tx bw mode (SRS_TX_BW_AGGR)
3566 *
3567 * The tx mode in which an SRS operates is decided in mac_tx_srs_setup()
3568 * based on the number of Tx rings requested for an SRS and whether
3569 * bandwidth control is requested or not.
3570 *
3571 * The default mode (i.e., no fanout/no bandwidth) is used when the
3572 * underlying NIC does not have Tx rings or just one Tx ring. In this mode,
3573 * the SRS acts as a pass-thru. Packets will go directly to mac_tx_send().
3574 * When the underlying Tx ring runs out of Tx descs, it starts queueing up
3575 * packets in SRS. When flow-control is relieved, the srs_worker drains
3576 * the queued packets and informs blocked clients to restart sending
3577 * packets.
3578 *
3579 * In the SRS_TX_SERIALIZE mode, all calls to mac_tx() are serialized. This
3580 * mode is used when the link has no Tx rings or only one Tx ring.
3581 *
3582 * In the SRS_TX_FANOUT mode, packets will be fanned out to multiple
3583 * Tx rings. Each Tx ring will have a soft ring associated with it.
3584 * These soft rings will be hung off the Tx SRS. Queueing if it happens
3585 * due to lack of Tx desc will be in individual soft ring (and not srs)
3586 * associated with Tx ring.
3587 *
3588 * In the TX_BW mode, tx srs will allow packets to go down to Tx ring
3589 * only if bw is available. Otherwise the packets will be queued in
3590 * SRS. If fanout to multiple Tx rings is configured, the packets will
3591 * be fanned out among the soft rings associated with the Tx rings.
3592 *
3593 * In SRS_TX_AGGR mode, mac_tx_aggr_mode() routine is called. This routine
3594 * invokes an aggr function, aggr_find_tx_ring(), to find a pseudo Tx ring
3595 * belonging to a port on which the packet has to be sent. Aggr will
3596 * always have a pseudo Tx ring associated with it even when it is an
3597 * aggregation over a single NIC that has no Tx rings. Even in such a
3598 * case, the single pseudo Tx ring will have a soft ring associated with
3599 * it and the soft ring will hang off the SRS.
3600 *
3601 * If a bandwidth is specified for an aggr, SRS_TX_BW_AGGR mode is used.
3602 * In this mode, the bandwidth is first applied on the outgoing packets
3603 * and later mac_tx_addr_mode() function is called to send the packet out
3604 * of one of the pseudo Tx rings.
3605 *
3606 * Four flags are used in srs_state for indicating flow control
3607 * conditions : SRS_TX_BLOCKED, SRS_TX_HIWAT, SRS_TX_WAKEUP_CLIENT.
3608 * SRS_TX_BLOCKED indicates out of Tx descs. SRS expects a wakeup from the
3609 * driver below.
3610 * SRS_TX_HIWAT indicates packet count enqueued in Tx SRS exceeded Tx hiwat
3611 * and flow-control pressure is applied back to clients. The clients expect
3612 * wakeup when flow-control is relieved.
3613 * SRS_TX_WAKEUP_CLIENT get set when (flag == MAC_TX_NO_ENQUEUE) and mblk
3614 * got returned back to client either due to lack of Tx descs or due to bw
3615 * control reasons. The clients expect a wakeup when condition is relieved.
3616 *
3617 * The fourth argument to mac_tx() is the flag. Normally it will be 0 but
3618 * some clients set the following values too: MAC_DROP_ON_NO_DESC,
3619 * MAC_TX_NO_ENQUEUE
3620 * Mac clients that do not want packets to be enqueued in the mac layer set
3621 * MAC_DROP_ON_NO_DESC value. The packets won't be queued in the Tx SRS or
3622 * Tx soft rings but instead get dropped when the NIC runs out of desc. The
3623 * behaviour of this flag is different when the Tx is running in serializer
3624 * or bandwidth mode. Under these (Serializer, bandwidth) modes, the packet
3625 * get dropped when Tx high watermark is reached.
3626 * There are some mac clients like vsw, aggr that want the mblks to be
3627 * returned back to clients instead of being queued in Tx SRS (or Tx soft
3628 * rings) under flow-control (i.e., out of desc or exceeding bw limits)
3629 * conditions. These clients call mac_tx() with MAC_TX_NO_ENQUEUE flag set.
3630 * In the default and Tx fanout mode, the un-transmitted mblks will be
3631 * returned back to the clients when the driver runs out of Tx descs.
3632 * SRS_TX_WAKEUP_CLIENT (or S_RING_WAKEUP_CLIENT) will be set in SRS (or
3633 * soft ring) so that the clients can be woken up when Tx desc become
3634 * available. When running in serializer or bandwidth mode mode,
3635 * SRS_TX_WAKEUP_CLIENT will be set when tx hi-watermark is reached.
3636 */
3637
3638 mac_tx_func_t
3639 mac_tx_get_func(uint32_t mode)
3640 {
3641 return (mac_tx_mode_list[mode].mac_tx_func);
3642 }
3643
3644 /* ARGSUSED */
3645 static mac_tx_cookie_t
3646 mac_tx_single_ring_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3647 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3648 {
3649 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3650 mac_tx_stats_t stats;
3651 mac_tx_cookie_t cookie = 0;
3652
3653 ASSERT(srs_tx->st_mode == SRS_TX_DEFAULT);
3654
3655 /* Regular case with a single Tx ring */
3656 /*
3657 * SRS_TX_BLOCKED is set when underlying NIC runs
3658 * out of Tx descs and messages start getting
3659 * queued. It won't get reset until
3660 * tx_srs_drain() completely drains out the
3661 * messages.
3662 */
3663 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
3664 /* Tx descs/resources not available */
3665 mutex_enter(&mac_srs->srs_lock);
3666 if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
3667 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain,
3668 flag, ret_mp);
3669 mutex_exit(&mac_srs->srs_lock);
3670 return (cookie);
3671 }
3672 /*
3673 * While we were computing mblk count, the
3674 * flow control condition got relieved.
3675 * Continue with the transmission.
3676 */
3677 mutex_exit(&mac_srs->srs_lock);
3678 }
3679
3680 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3681 mp_chain, &stats);
3682
3683 /*
3684 * Multiple threads could be here sending packets.
3685 * Under such conditions, it is not possible to
3686 * automically set SRS_TX_BLOCKED bit to indicate
3687 * out of tx desc condition. To atomically set
3688 * this, we queue the returned packet and do
3689 * the setting of SRS_TX_BLOCKED in
3690 * mac_tx_srs_drain().
3691 */
3692 if (mp_chain != NULL) {
3693 mutex_enter(&mac_srs->srs_lock);
3694 cookie = mac_tx_srs_no_desc(mac_srs, mp_chain, flag, ret_mp);
3695 mutex_exit(&mac_srs->srs_lock);
3696 return (cookie);
3697 }
3698 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3699
3700 return (0);
3701 }
3702
3703 /*
3704 * mac_tx_serialize_mode
3705 *
3706 * This is an experimental mode implemented as per the request of PAE.
3707 * In this mode, all callers attempting to send a packet to the NIC
3708 * will get serialized. Only one thread at any time will access the
3709 * NIC to send the packet out.
3710 */
3711 /* ARGSUSED */
3712 static mac_tx_cookie_t
3713 mac_tx_serializer_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3714 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3715 {
3716 mac_tx_stats_t stats;
3717 mac_tx_cookie_t cookie = 0;
3718 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3719
3720 /* Single ring, serialize below */
3721 ASSERT(srs_tx->st_mode == SRS_TX_SERIALIZE);
3722 mutex_enter(&mac_srs->srs_lock);
3723 if ((mac_srs->srs_first != NULL) ||
3724 (mac_srs->srs_state & SRS_PROC)) {
3725 /*
3726 * In serialization mode, queue all packets until
3727 * TX_HIWAT is set.
3728 * If drop bit is set, drop if TX_HIWAT is set.
3729 * If no_enqueue is set, still enqueue until hiwat
3730 * is set and return mblks after TX_HIWAT is set.
3731 */
3732 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain,
3733 flag, 0, ret_mp);
3734 mutex_exit(&mac_srs->srs_lock);
3735 return (cookie);
3736 }
3737 /*
3738 * No packets queued, nothing on proc and no flow
3739 * control condition. Fast-path, ok. Do inline
3740 * processing.
3741 */
3742 mac_srs->srs_state |= SRS_PROC;
3743 mutex_exit(&mac_srs->srs_lock);
3744
3745 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3746 mp_chain, &stats);
3747
3748 mutex_enter(&mac_srs->srs_lock);
3749 mac_srs->srs_state &= ~SRS_PROC;
3750 if (mp_chain != NULL) {
3751 cookie = mac_tx_srs_enqueue(mac_srs,
3752 mp_chain, flag, 0, ret_mp);
3753 }
3754 if (mac_srs->srs_first != NULL) {
3755 /*
3756 * We processed inline our packet and a new
3757 * packet/s got queued while we were
3758 * processing. Wakeup srs worker
3759 */
3760 cv_signal(&mac_srs->srs_async);
3761 }
3762 mutex_exit(&mac_srs->srs_lock);
3763
3764 if (cookie == 0)
3765 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3766
3767 return (cookie);
3768 }
3769
3770 /*
3771 * mac_tx_fanout_mode
3772 *
3773 * In this mode, the SRS will have access to multiple Tx rings to send
3774 * the packet out. The fanout hint that is passed as an argument is
3775 * used to find an appropriate ring to fanout the traffic. Each Tx
3776 * ring, in turn, will have a soft ring associated with it. If a Tx
3777 * ring runs out of Tx desc's the returned packet will be queued in
3778 * the soft ring associated with that Tx ring. The srs itself will not
3779 * queue any packets.
3780 */
3781
3782 #define MAC_TX_SOFT_RING_PROCESS(chain) { \
3783 index = COMPUTE_INDEX(hash, mac_srs->srs_tx_ring_count), \
3784 softring = mac_srs->srs_tx_soft_rings[index]; \
3785 cookie = mac_tx_soft_ring_process(softring, chain, flag, ret_mp); \
3786 DTRACE_PROBE2(tx__fanout, uint64_t, hash, uint_t, index); \
3787 }
3788
3789 static mac_tx_cookie_t
3790 mac_tx_fanout_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3791 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3792 {
3793 mac_soft_ring_t *softring;
3794 uint64_t hash;
3795 uint_t index;
3796 mac_tx_cookie_t cookie = 0;
3797
3798 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
3799 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT);
3800 if (fanout_hint != 0) {
3801 /*
3802 * The hint is specified by the caller, simply pass the
3803 * whole chain to the soft ring.
3804 */
3805 hash = HASH_HINT(fanout_hint);
3806 MAC_TX_SOFT_RING_PROCESS(mp_chain);
3807 } else {
3808 mblk_t *last_mp, *cur_mp, *sub_chain;
3809 uint64_t last_hash = 0;
3810 uint_t media = mac_srs->srs_mcip->mci_mip->mi_info.mi_media;
3811
3812 /*
3813 * Compute the hash from the contents (headers) of the
3814 * packets of the mblk chain. Split the chains into
3815 * subchains of the same conversation.
3816 *
3817 * Since there may be more than one ring used for
3818 * sub-chains of the same call, and since the caller
3819 * does not maintain per conversation state since it
3820 * passed a zero hint, unsent subchains will be
3821 * dropped.
3822 */
3823
3824 flag |= MAC_DROP_ON_NO_DESC;
3825 ret_mp = NULL;
3826
3827 ASSERT(ret_mp == NULL);
3828
3829 sub_chain = NULL;
3830 last_mp = NULL;
3831
3832 for (cur_mp = mp_chain; cur_mp != NULL;
3833 cur_mp = cur_mp->b_next) {
3834 hash = mac_pkt_hash(media, cur_mp, MAC_PKT_HASH_L4,
3835 B_TRUE);
3836 if (last_hash != 0 && hash != last_hash) {
3837 /*
3838 * Starting a different subchain, send current
3839 * chain out.
3840 */
3841 ASSERT(last_mp != NULL);
3842 last_mp->b_next = NULL;
3843 MAC_TX_SOFT_RING_PROCESS(sub_chain);
3844 sub_chain = NULL;
3845 }
3846
3847 /* add packet to subchain */
3848 if (sub_chain == NULL)
3849 sub_chain = cur_mp;
3850 last_mp = cur_mp;
3851 last_hash = hash;
3852 }
3853
3854 if (sub_chain != NULL) {
3855 /* send last subchain */
3856 ASSERT(last_mp != NULL);
3857 last_mp->b_next = NULL;
3858 MAC_TX_SOFT_RING_PROCESS(sub_chain);
3859 }
3860
3861 cookie = 0;
3862 }
3863
3864 return (cookie);
3865 }
3866
3867 /*
3868 * mac_tx_bw_mode
3869 *
3870 * In the bandwidth mode, Tx srs will allow packets to go down to Tx ring
3871 * only if bw is available. Otherwise the packets will be queued in
3872 * SRS. If the SRS has multiple Tx rings, then packets will get fanned
3873 * out to a Tx rings.
3874 */
3875 static mac_tx_cookie_t
3876 mac_tx_bw_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3877 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3878 {
3879 int cnt, sz;
3880 mblk_t *tail;
3881 mac_tx_cookie_t cookie = 0;
3882 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3883 clock_t now;
3884
3885 ASSERT(TX_BANDWIDTH_MODE(mac_srs));
3886 ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
3887 mutex_enter(&mac_srs->srs_lock);
3888 if (mac_srs->srs_bw->mac_bw_limit == 0) {
3889 /*
3890 * zero bandwidth, no traffic is sent: drop the packets,
3891 * or return the whole chain if the caller requests all
3892 * unsent packets back.
3893 */
3894 if (flag & MAC_TX_NO_ENQUEUE) {
3895 cookie = (mac_tx_cookie_t)mac_srs;
3896 *ret_mp = mp_chain;
3897 } else {
3898 MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3899 }
3900 mutex_exit(&mac_srs->srs_lock);
3901 return (cookie);
3902 } else if ((mac_srs->srs_first != NULL) ||
3903 (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
3904 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
3905 fanout_hint, ret_mp);
3906 mutex_exit(&mac_srs->srs_lock);
3907 return (cookie);
3908 }
3909 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3910 now = ddi_get_lbolt();
3911 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3912 mac_srs->srs_bw->mac_bw_curr_time = now;
3913 mac_srs->srs_bw->mac_bw_used = 0;
3914 } else if (mac_srs->srs_bw->mac_bw_used >
3915 mac_srs->srs_bw->mac_bw_limit) {
3916 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3917 MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3918 mp_chain, tail, cnt, sz);
3919 /*
3920 * Wakeup worker thread. Note that worker
3921 * thread has to be woken up so that it
3922 * can fire up the timer to be woken up
3923 * on the next tick. Also once
3924 * BW_ENFORCED is set, it can only be
3925 * reset by srs_worker thread. Until then
3926 * all packets will get queued up in SRS
3927 * and hence this this code path won't be
3928 * entered until BW_ENFORCED is reset.
3929 */
3930 cv_signal(&mac_srs->srs_async);
3931 mutex_exit(&mac_srs->srs_lock);
3932 return (cookie);
3933 }
3934
3935 mac_srs->srs_bw->mac_bw_used += sz;
3936 mutex_exit(&mac_srs->srs_lock);
3937
3938 if (srs_tx->st_mode == SRS_TX_BW_FANOUT) {
3939 mac_soft_ring_t *softring;
3940 uint_t indx, hash;
3941
3942 hash = HASH_HINT(fanout_hint);
3943 indx = COMPUTE_INDEX(hash,
3944 mac_srs->srs_tx_ring_count);
3945 softring = mac_srs->srs_tx_soft_rings[indx];
3946 return (mac_tx_soft_ring_process(softring, mp_chain, flag,
3947 ret_mp));
3948 } else if (srs_tx->st_mode == SRS_TX_BW_AGGR) {
3949 return (mac_tx_aggr_mode(mac_srs, mp_chain,
3950 fanout_hint, flag, ret_mp));
3951 } else {
3952 mac_tx_stats_t stats;
3953
3954 mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3955 mp_chain, &stats);
3956
3957 if (mp_chain != NULL) {
3958 mutex_enter(&mac_srs->srs_lock);
3959 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3960 if (mac_srs->srs_bw->mac_bw_used > sz)
3961 mac_srs->srs_bw->mac_bw_used -= sz;
3962 else
3963 mac_srs->srs_bw->mac_bw_used = 0;
3964 cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
3965 fanout_hint, ret_mp);
3966 mutex_exit(&mac_srs->srs_lock);
3967 return (cookie);
3968 }
3969 SRS_TX_STATS_UPDATE(mac_srs, &stats);
3970
3971 return (0);
3972 }
3973 }
3974
3975 /*
3976 * mac_tx_aggr_mode
3977 *
3978 * This routine invokes an aggr function, aggr_find_tx_ring(), to find
3979 * a (pseudo) Tx ring belonging to a port on which the packet has to
3980 * be sent. aggr_find_tx_ring() first finds the outgoing port based on
3981 * L2/L3/L4 policy and then uses the fanout_hint passed to it to pick
3982 * a Tx ring from the selected port.
3983 *
3984 * Note that a port can be deleted from the aggregation. In such a case,
3985 * the aggregation layer first separates the port from the rest of the
3986 * ports making sure that port (and thus any Tx rings associated with
3987 * it) won't get selected in the call to aggr_find_tx_ring() function.
3988 * Later calls are made to mac_group_rem_ring() passing pseudo Tx ring
3989 * handles one by one which in turn will quiesce the Tx SRS and remove
3990 * the soft ring associated with the pseudo Tx ring. Unlike Rx side
3991 * where a cookie is used to protect against mac_rx_ring() calls on
3992 * rings that have been removed, no such cookie is needed on the Tx
3993 * side as the pseudo Tx ring won't be available anymore to
3994 * aggr_find_tx_ring() once the port has been removed.
3995 */
3996 static mac_tx_cookie_t
3997 mac_tx_aggr_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3998 uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3999 {
4000 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
4001 mac_tx_ring_fn_t find_tx_ring_fn;
4002 mac_ring_handle_t ring = NULL;
4003 void *arg;
4004 mac_soft_ring_t *sringp;
4005
4006 find_tx_ring_fn = srs_tx->st_capab_aggr.mca_find_tx_ring_fn;
4007 arg = srs_tx->st_capab_aggr.mca_arg;
4008 if (find_tx_ring_fn(arg, mp_chain, fanout_hint, &ring) == NULL)
4009 return (0);
4010 sringp = srs_tx->st_soft_rings[((mac_ring_t *)ring)->mr_index];
4011 return (mac_tx_soft_ring_process(sringp, mp_chain, flag, ret_mp));
4012 }
4013
4014 void
4015 mac_tx_invoke_callbacks(mac_client_impl_t *mcip, mac_tx_cookie_t cookie)
4016 {
4017 mac_cb_t *mcb;
4018 mac_tx_notify_cb_t *mtnfp;
4019
4020 /* Wakeup callback registered clients */
4021 MAC_CALLBACK_WALKER_INC(&mcip->mci_tx_notify_cb_info);
4022 for (mcb = mcip->mci_tx_notify_cb_list; mcb != NULL;
4023 mcb = mcb->mcb_nextp) {
4024 mtnfp = (mac_tx_notify_cb_t *)mcb->mcb_objp;
4025 mtnfp->mtnf_fn(mtnfp->mtnf_arg, cookie);
4026 }
4027 MAC_CALLBACK_WALKER_DCR(&mcip->mci_tx_notify_cb_info,
4028 &mcip->mci_tx_notify_cb_list);
4029 }
4030
4031 /* ARGSUSED */
4032 void
4033 mac_tx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
4034 {
4035 mblk_t *head, *tail;
4036 size_t sz;
4037 uint32_t tx_mode;
4038 uint_t saved_pkt_count;
4039 mac_tx_stats_t stats;
4040 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
4041 clock_t now;
4042
4043 saved_pkt_count = 0;
4044 ASSERT(mutex_owned(&mac_srs->srs_lock));
4045 ASSERT(!(mac_srs->srs_state & SRS_PROC));
4046
4047 mac_srs->srs_state |= SRS_PROC;
4048
4049 tx_mode = srs_tx->st_mode;
4050 if (tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_SERIALIZE) {
4051 if (mac_srs->srs_first != NULL) {
4052 head = mac_srs->srs_first;
4053 tail = mac_srs->srs_last;
4054 saved_pkt_count = mac_srs->srs_count;
4055 mac_srs->srs_first = NULL;
4056 mac_srs->srs_last = NULL;
4057 mac_srs->srs_count = 0;
4058 mutex_exit(&mac_srs->srs_lock);
4059
4060 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
4061 head, &stats);
4062
4063 mutex_enter(&mac_srs->srs_lock);
4064 if (head != NULL) {
4065 /* Device out of tx desc, set block */
4066 if (head->b_next == NULL)
4067 VERIFY(head == tail);
4068 tail->b_next = mac_srs->srs_first;
4069 mac_srs->srs_first = head;
4070 mac_srs->srs_count +=
4071 (saved_pkt_count - stats.mts_opackets);
4072 if (mac_srs->srs_last == NULL)
4073 mac_srs->srs_last = tail;
4074 MAC_TX_SRS_BLOCK(mac_srs, head);
4075 } else {
4076 srs_tx->st_woken_up = B_FALSE;
4077 SRS_TX_STATS_UPDATE(mac_srs, &stats);
4078 }
4079 }
4080 } else if (tx_mode == SRS_TX_BW) {
4081 /*
4082 * We are here because the timer fired and we have some data
4083 * to tranmit. Also mac_tx_srs_worker should have reset
4084 * SRS_BW_ENFORCED flag
4085 */
4086 ASSERT(!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED));
4087 head = tail = mac_srs->srs_first;
4088 while (mac_srs->srs_first != NULL) {
4089 tail = mac_srs->srs_first;
4090 tail->b_prev = NULL;
4091 mac_srs->srs_first = tail->b_next;
4092 if (mac_srs->srs_first == NULL)
4093 mac_srs->srs_last = NULL;
4094 mac_srs->srs_count--;
4095 sz = msgdsize(tail);
4096 mac_srs->srs_size -= sz;
4097 saved_pkt_count++;
4098 MAC_TX_UPDATE_BW_INFO(mac_srs, sz);
4099
4100 if (mac_srs->srs_bw->mac_bw_used <
4101 mac_srs->srs_bw->mac_bw_limit)
4102 continue;
4103
4104 now = ddi_get_lbolt();
4105 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
4106 mac_srs->srs_bw->mac_bw_curr_time = now;
4107 mac_srs->srs_bw->mac_bw_used = sz;
4108 continue;
4109 }
4110 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
4111 break;
4112 }
4113
4114 ASSERT((head == NULL && tail == NULL) ||
4115 (head != NULL && tail != NULL));
4116 if (tail != NULL) {
4117 tail->b_next = NULL;
4118 mutex_exit(&mac_srs->srs_lock);
4119
4120 head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
4121 head, &stats);
4122
4123 mutex_enter(&mac_srs->srs_lock);
4124 if (head != NULL) {
4125 uint_t size_sent;
4126
4127 /* Device out of tx desc, set block */
4128 if (head->b_next == NULL)
4129 VERIFY(head == tail);
4130 tail->b_next = mac_srs->srs_first;
4131 mac_srs->srs_first = head;
4132 mac_srs->srs_count +=
4133 (saved_pkt_count - stats.mts_opackets);
4134 if (mac_srs->srs_last == NULL)
4135 mac_srs->srs_last = tail;
4136 size_sent = sz - stats.mts_obytes;
4137 mac_srs->srs_size += size_sent;
4138 mac_srs->srs_bw->mac_bw_sz += size_sent;
4139 if (mac_srs->srs_bw->mac_bw_used > size_sent) {
4140 mac_srs->srs_bw->mac_bw_used -=
4141 size_sent;
4142 } else {
4143 mac_srs->srs_bw->mac_bw_used = 0;
4144 }
4145 MAC_TX_SRS_BLOCK(mac_srs, head);
4146 } else {
4147 srs_tx->st_woken_up = B_FALSE;
4148 SRS_TX_STATS_UPDATE(mac_srs, &stats);
4149 }
4150 }
4151 } else if (tx_mode == SRS_TX_BW_FANOUT || tx_mode == SRS_TX_BW_AGGR) {
4152 mblk_t *prev;
4153 uint64_t hint;
4154
4155 /*
4156 * We are here because the timer fired and we
4157 * have some quota to tranmit.
4158 */
4159 prev = NULL;
4160 head = tail = mac_srs->srs_first;
4161 while (mac_srs->srs_first != NULL) {
4162 tail = mac_srs->srs_first;
4163 mac_srs->srs_first = tail->b_next;
4164 if (mac_srs->srs_first == NULL)
4165 mac_srs->srs_last = NULL;
4166 mac_srs->srs_count--;
4167 sz = msgdsize(tail);
4168 mac_srs->srs_size -= sz;
4169 mac_srs->srs_bw->mac_bw_used += sz;
4170 if (prev == NULL)
4171 hint = (ulong_t)tail->b_prev;
4172 if (hint != (ulong_t)tail->b_prev) {
4173 prev->b_next = NULL;
4174 mutex_exit(&mac_srs->srs_lock);
4175 TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
4176 head = tail;
4177 hint = (ulong_t)tail->b_prev;
4178 mutex_enter(&mac_srs->srs_lock);
4179 }
4180
4181 prev = tail;
4182 tail->b_prev = NULL;
4183 if (mac_srs->srs_bw->mac_bw_used <
4184 mac_srs->srs_bw->mac_bw_limit)
4185 continue;
4186
4187 now = ddi_get_lbolt();
4188 if (mac_srs->srs_bw->mac_bw_curr_time != now) {
4189 mac_srs->srs_bw->mac_bw_curr_time = now;
4190 mac_srs->srs_bw->mac_bw_used = 0;
4191 continue;
4192 }
4193 mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
4194 break;
4195 }
4196 ASSERT((head == NULL && tail == NULL) ||
4197 (head != NULL && tail != NULL));
4198 if (tail != NULL) {
4199 tail->b_next = NULL;
4200 mutex_exit(&mac_srs->srs_lock);
4201 TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
4202 mutex_enter(&mac_srs->srs_lock);
4203 }
4204 }
4205 /*
4206 * SRS_TX_FANOUT case not considered here because packets
4207 * won't be queued in the SRS for this case. Packets will
4208 * be sent directly to soft rings underneath and if there
4209 * is any queueing at all, it would be in Tx side soft
4210 * rings.
4211 */
4212
4213 /*
4214 * When srs_count becomes 0, reset SRS_TX_HIWAT and
4215 * SRS_TX_WAKEUP_CLIENT and wakeup registered clients.
4216 */
4217 if (mac_srs->srs_count == 0 && (mac_srs->srs_state &
4218 (SRS_TX_HIWAT | SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED))) {
4219 mac_client_impl_t *mcip = mac_srs->srs_mcip;
4220 boolean_t wakeup_required = B_FALSE;
4221
4222 if (mac_srs->srs_state &
4223 (SRS_TX_HIWAT|SRS_TX_WAKEUP_CLIENT)) {
4224 wakeup_required = B_TRUE;
4225 }
4226 mac_srs->srs_state &= ~(SRS_TX_HIWAT |
4227 SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED);
4228 mutex_exit(&mac_srs->srs_lock);
4229 if (wakeup_required) {
4230 mac_tx_invoke_callbacks(mcip, (mac_tx_cookie_t)mac_srs);
4231 /*
4232 * If the client is not the primary MAC client, then we
4233 * need to send the notification to the clients upper
4234 * MAC, i.e. mci_upper_mip.
4235 */
4236 mac_tx_notify(mcip->mci_upper_mip != NULL ?
4237 mcip->mci_upper_mip : mcip->mci_mip);
4238 }
4239 mutex_enter(&mac_srs->srs_lock);
4240 }
4241 mac_srs->srs_state &= ~SRS_PROC;
4242 }
4243
4244 /*
4245 * Given a packet, get the flow_entry that identifies the flow
4246 * to which that packet belongs. The flow_entry will contain
4247 * the transmit function to be used to send the packet. If the
4248 * function returns NULL, the packet should be sent using the
4249 * underlying NIC.
4250 */
4251 static flow_entry_t *
4252 mac_tx_classify(mac_impl_t *mip, mblk_t *mp)
4253 {
4254 flow_entry_t *flent = NULL;
4255 mac_client_impl_t *mcip;
4256 int err;
4257
4258 /*
4259 * Do classification on the packet.
4260 */
4261 err = mac_flow_lookup(mip->mi_flow_tab, mp, FLOW_OUTBOUND, &flent);
4262 if (err != 0)
4263 return (NULL);
4264
4265 /*
4266 * This flent might just be an additional one on the MAC client,
4267 * i.e. for classification purposes (different fdesc), however
4268 * the resources, SRS et. al., are in the mci_flent, so if
4269 * this isn't the mci_flent, we need to get it.
4270 */
4271 if ((mcip = flent->fe_mcip) != NULL && mcip->mci_flent != flent) {
4272 FLOW_REFRELE(flent);
4273 flent = mcip->mci_flent;
4274 FLOW_TRY_REFHOLD(flent, err);
4275 if (err != 0)
4276 return (NULL);
4277 }
4278
4279 return (flent);
4280 }
4281
4282 /*
4283 * This macro is only meant to be used by mac_tx_send().
4284 */
4285 #define CHECK_VID_AND_ADD_TAG(mp) { \
4286 if (vid_check) { \
4287 int err = 0; \
4288 \
4289 MAC_VID_CHECK(src_mcip, (mp), err); \
4290 if (err != 0) { \
4291 freemsg((mp)); \
4292 (mp) = next; \
4293 oerrors++; \
4294 continue; \
4295 } \
4296 } \
4297 if (add_tag) { \
4298 (mp) = mac_add_vlan_tag((mp), 0, vid); \
4299 if ((mp) == NULL) { \
4300 (mp) = next; \
4301 oerrors++; \
4302 continue; \
4303 } \
4304 } \
4305 }
4306
4307 mblk_t *
4308 mac_tx_send(mac_client_handle_t mch, mac_ring_handle_t ring, mblk_t *mp_chain,
4309 mac_tx_stats_t *stats)
4310 {
4311 mac_client_impl_t *src_mcip = (mac_client_impl_t *)mch;
4312 mac_impl_t *mip = src_mcip->mci_mip;
4313 uint_t obytes = 0, opackets = 0, oerrors = 0;
4314 mblk_t *mp = NULL, *next;
4315 boolean_t vid_check, add_tag;
4316 uint16_t vid = 0;
4317
4318 if (mip->mi_nclients > 1) {
4319 vid_check = MAC_VID_CHECK_NEEDED(src_mcip);
4320 add_tag = MAC_TAG_NEEDED(src_mcip);
4321 if (add_tag)
4322 vid = mac_client_vid(mch);
4323 } else {
4324 ASSERT(mip->mi_nclients == 1);
4325 vid_check = add_tag = B_FALSE;
4326 }
4327
4328 /*
4329 * Fastpath: if there's only one client, we simply send
4330 * the packet down to the underlying NIC.
4331 */
4332 if (mip->mi_nactiveclients == 1) {
4333 DTRACE_PROBE2(fastpath,
4334 mac_client_impl_t *, src_mcip, mblk_t *, mp_chain);
4335
4336 mp = mp_chain;
4337 while (mp != NULL) {
4338 next = mp->b_next;
4339 mp->b_next = NULL;
4340 opackets++;
4341 obytes += (mp->b_cont == NULL ? MBLKL(mp) :
4342 msgdsize(mp));
4343
4344 CHECK_VID_AND_ADD_TAG(mp);
4345 MAC_TX(mip, ring, mp, src_mcip);
4346
4347 /*
4348 * If the driver is out of descriptors and does a
4349 * partial send it will return a chain of unsent
4350 * mblks. Adjust the accounting stats.
4351 */
4352 if (mp != NULL) {
4353 opackets--;
4354 obytes -= msgdsize(mp);
4355 mp->b_next = next;
4356 break;
4357 }
4358 mp = next;
4359 }
4360 goto done;
4361 }
4362
4363 /*
4364 * No fastpath, we either have more than one MAC client
4365 * defined on top of the same MAC, or one or more MAC
4366 * client promiscuous callbacks.
4367 */
4368 DTRACE_PROBE3(slowpath, mac_client_impl_t *,
4369 src_mcip, int, mip->mi_nclients, mblk_t *, mp_chain);
4370
4371 mp = mp_chain;
4372 while (mp != NULL) {
4373 flow_entry_t *dst_flow_ent;
4374 void *flow_cookie;
4375 size_t pkt_size;
4376 mblk_t *mp1;
4377
4378 next = mp->b_next;
4379 mp->b_next = NULL;
4380 opackets++;
4381 pkt_size = (mp->b_cont == NULL ? MBLKL(mp) : msgdsize(mp));
4382 obytes += pkt_size;
4383 CHECK_VID_AND_ADD_TAG(mp);
4384
4385 /*
4386 * Find the destination.
4387 */
4388 dst_flow_ent = mac_tx_classify(mip, mp);
4389
4390 if (dst_flow_ent != NULL) {
4391 size_t hdrsize;
4392 int err = 0;
4393
4394 if (mip->mi_info.mi_nativemedia == DL_ETHER) {
4395 struct ether_vlan_header *evhp =
4396 (struct ether_vlan_header *)mp->b_rptr;
4397
4398 if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN)
4399 hdrsize = sizeof (*evhp);
4400 else
4401 hdrsize = sizeof (struct ether_header);
4402 } else {
4403 mac_header_info_t mhi;
4404
4405 err = mac_header_info((mac_handle_t)mip,
4406 mp, &mhi);
4407 if (err == 0)
4408 hdrsize = mhi.mhi_hdrsize;
4409 }
4410
4411 /*
4412 * Got a matching flow. It's either another
4413 * MAC client, or a broadcast/multicast flow.
4414 * Make sure the packet size is within the
4415 * allowed size. If not drop the packet and
4416 * move to next packet.
4417 */
4418 if (err != 0 ||
4419 (pkt_size - hdrsize) > mip->mi_sdu_max) {
4420 oerrors++;
4421 DTRACE_PROBE2(loopback__drop, size_t, pkt_size,
4422 mblk_t *, mp);
4423 freemsg(mp);
4424 mp = next;
4425 FLOW_REFRELE(dst_flow_ent);
4426 continue;
4427 }
4428 flow_cookie = mac_flow_get_client_cookie(dst_flow_ent);
4429 if (flow_cookie != NULL) {
4430 /*
4431 * The vnic_bcast_send function expects
4432 * to receive the sender MAC client
4433 * as value for arg2.
4434 */
4435 mac_bcast_send(flow_cookie, src_mcip, mp,
4436 B_TRUE);
4437 } else {
4438 /*
4439 * loopback the packet to a local MAC
4440 * client. We force a context switch
4441 * if both source and destination MAC
4442 * clients are used by IP, i.e.
4443 * bypass is set.
4444 */
4445 boolean_t do_switch;
4446 mac_client_impl_t *dst_mcip =
4447 dst_flow_ent->fe_mcip;
4448
4449 /*
4450 * Check if there are promiscuous mode
4451 * callbacks defined. This check is
4452 * done here in the 'else' case and
4453 * not in other cases because this
4454 * path is for local loopback
4455 * communication which does not go
4456 * through MAC_TX(). For paths that go
4457 * through MAC_TX(), the promisc_list
4458 * check is done inside the MAC_TX()
4459 * macro.
4460 */
4461 if (mip->mi_promisc_list != NULL)
4462 mac_promisc_dispatch(mip, mp, src_mcip);
4463
4464 do_switch = ((src_mcip->mci_state_flags &
4465 dst_mcip->mci_state_flags &
4466 MCIS_CLIENT_POLL_CAPABLE) != 0);
4467
4468 if ((mp1 = mac_fix_cksum(mp)) != NULL) {
4469 (dst_flow_ent->fe_cb_fn)(
4470 dst_flow_ent->fe_cb_arg1,
4471 dst_flow_ent->fe_cb_arg2,
4472 mp1, do_switch);
4473 }
4474 }
4475 FLOW_REFRELE(dst_flow_ent);
4476 } else {
4477 /*
4478 * Unknown destination, send via the underlying
4479 * NIC.
4480 */
4481 MAC_TX(mip, ring, mp, src_mcip);
4482 if (mp != NULL) {
4483 /*
4484 * Adjust for the last packet that
4485 * could not be transmitted
4486 */
4487 opackets--;
4488 obytes -= pkt_size;
4489 mp->b_next = next;
4490 break;
4491 }
4492 }
4493 mp = next;
4494 }
4495
4496 done:
4497 stats->mts_obytes = obytes;
4498 stats->mts_opackets = opackets;
4499 stats->mts_oerrors = oerrors;
4500 return (mp);
4501 }
4502
4503 /*
4504 * mac_tx_srs_ring_present
4505 *
4506 * Returns whether the specified ring is part of the specified SRS.
4507 */
4508 boolean_t
4509 mac_tx_srs_ring_present(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
4510 {
4511 int i;
4512 mac_soft_ring_t *soft_ring;
4513
4514 if (srs->srs_tx.st_arg2 == tx_ring)
4515 return (B_TRUE);
4516
4517 for (i = 0; i < srs->srs_tx_ring_count; i++) {
4518 soft_ring = srs->srs_tx_soft_rings[i];
4519 if (soft_ring->s_ring_tx_arg2 == tx_ring)
4520 return (B_TRUE);
4521 }
4522
4523 return (B_FALSE);
4524 }
4525
4526 /*
4527 * mac_tx_srs_get_soft_ring
4528 *
4529 * Returns the TX soft ring associated with the given ring, if present.
4530 */
4531 mac_soft_ring_t *
4532 mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
4533 {
4534 int i;
4535 mac_soft_ring_t *soft_ring;
4536
4537 if (srs->srs_tx.st_arg2 == tx_ring)
4538 return (NULL);
4539
4540 for (i = 0; i < srs->srs_tx_ring_count; i++) {
4541 soft_ring = srs->srs_tx_soft_rings[i];
4542 if (soft_ring->s_ring_tx_arg2 == tx_ring)
4543 return (soft_ring);
4544 }
4545
4546 return (NULL);
4547 }
4548
4549 /*
4550 * mac_tx_srs_wakeup
4551 *
4552 * Called when Tx desc become available. Wakeup the appropriate worker
4553 * thread after resetting the SRS_TX_BLOCKED/S_RING_BLOCK bit in the
4554 * state field.
4555 */
4556 void
4557 mac_tx_srs_wakeup(mac_soft_ring_set_t *mac_srs, mac_ring_handle_t ring)
4558 {
4559 int i;
4560 mac_soft_ring_t *sringp;
4561 mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
4562
4563 mutex_enter(&mac_srs->srs_lock);
4564 /*
4565 * srs_tx_ring_count == 0 is the single ring mode case. In
4566 * this mode, there will not be Tx soft rings associated
4567 * with the SRS.
4568 */
4569 if (!MAC_TX_SOFT_RINGS(mac_srs)) {
4570 if (srs_tx->st_arg2 == ring &&
4571 mac_srs->srs_state & SRS_TX_BLOCKED) {
4572 mac_srs->srs_state &= ~SRS_TX_BLOCKED;
4573 srs_tx->st_stat.mts_unblockcnt++;
4574 cv_signal(&mac_srs->srs_async);
4575 }
4576 /*
4577 * A wakeup can come before tx_srs_drain() could
4578 * grab srs lock and set SRS_TX_BLOCKED. So
4579 * always set woken_up flag when we come here.
4580 */
4581 srs_tx->st_woken_up = B_TRUE;
4582 mutex_exit(&mac_srs->srs_lock);
4583 return;
4584 }
4585
4586 /*
4587 * If you are here, it is for FANOUT, BW_FANOUT,
4588 * AGGR_MODE or AGGR_BW_MODE case
4589 */
4590 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
4591 sringp = mac_srs->srs_tx_soft_rings[i];
4592 mutex_enter(&sringp->s_ring_lock);
4593 if (sringp->s_ring_tx_arg2 == ring) {
4594 if (sringp->s_ring_state & S_RING_BLOCK) {
4595 sringp->s_ring_state &= ~S_RING_BLOCK;
4596 sringp->s_st_stat.mts_unblockcnt++;
4597 cv_signal(&sringp->s_ring_async);
4598 }
4599 sringp->s_ring_tx_woken_up = B_TRUE;
4600 }
4601 mutex_exit(&sringp->s_ring_lock);
4602 }
4603 mutex_exit(&mac_srs->srs_lock);
4604 }
4605
4606 /*
4607 * Once the driver is done draining, send a MAC_NOTE_TX notification to unleash
4608 * the blocked clients again.
4609 */
4610 void
4611 mac_tx_notify(mac_impl_t *mip)
4612 {
4613 i_mac_notify(mip, MAC_NOTE_TX);
4614 }
4615
4616 /*
4617 * RX SOFTRING RELATED FUNCTIONS
4618 *
4619 * These functions really belong in mac_soft_ring.c and here for
4620 * a short period.
4621 */
4622
4623 #define SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \
4624 /* \
4625 * Enqueue our mblk chain. \
4626 */ \
4627 ASSERT(MUTEX_HELD(&(ringp)->s_ring_lock)); \
4628 \
4629 if ((ringp)->s_ring_last != NULL) \
4630 (ringp)->s_ring_last->b_next = (mp); \
4631 else \
4632 (ringp)->s_ring_first = (mp); \
4633 (ringp)->s_ring_last = (tail); \
4634 (ringp)->s_ring_count += (cnt); \
4635 ASSERT((ringp)->s_ring_count > 0); \
4636 if ((ringp)->s_ring_type & ST_RING_BW_CTL) { \
4637 (ringp)->s_ring_size += sz; \
4638 } \
4639 }
4640
4641 /*
4642 * Default entry point to deliver a packet chain to a MAC client.
4643 * If the MAC client has flows, do the classification with these
4644 * flows as well.
4645 */
4646 /* ARGSUSED */
4647 void
4648 mac_rx_deliver(void *arg1, mac_resource_handle_t mrh, mblk_t *mp_chain,
4649 mac_header_info_t *arg3)
4650 {
4651 mac_client_impl_t *mcip = arg1;
4652
4653 if (mcip->mci_nvids == 1 &&
4654 !(mcip->mci_state_flags & MCIS_STRIP_DISABLE)) {
4655 /*
4656 * If the client has exactly one VID associated with it
4657 * and striping of VLAN header is not disabled,
4658 * remove the VLAN tag from the packet before
4659 * passing it on to the client's receive callback.
4660 * Note that this needs to be done after we dispatch
4661 * the packet to the promiscuous listeners of the
4662 * client, since they expect to see the whole
4663 * frame including the VLAN headers.
4664 *
4665 * The MCIS_STRIP_DISABLE is only issued when sun4v
4666 * vsw is in play.
4667 */
4668 mp_chain = mac_strip_vlan_tag_chain(mp_chain);
4669 }
4670
4671 mcip->mci_rx_fn(mcip->mci_rx_arg, mrh, mp_chain, B_FALSE);
4672 }
4673
4674 /*
4675 * Process a chain for a given soft ring. If the number of packets
4676 * queued in the SRS and its associated soft rings (including this
4677 * one) is very small (tracked by srs_poll_pkt_cnt) then allow the
4678 * entering thread (interrupt or poll thread) to process the chain
4679 * inline. This is meant to reduce latency under low load.
4680 *
4681 * The proc and arg for each mblk is already stored in the mblk in
4682 * appropriate places.
4683 */
4684 /* ARGSUSED */
4685 void
4686 mac_rx_soft_ring_process(mac_client_impl_t *mcip, mac_soft_ring_t *ringp,
4687 mblk_t *mp_chain, mblk_t *tail, int cnt, size_t sz)
4688 {
4689 mac_direct_rx_t proc;
4690 void *arg1;
4691 mac_resource_handle_t arg2;
4692 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
4693
4694 ASSERT(ringp != NULL);
4695 ASSERT(mp_chain != NULL);
4696 ASSERT(tail != NULL);
4697 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4698
4699 mutex_enter(&ringp->s_ring_lock);
4700 ringp->s_ring_total_inpkt += cnt;
4701 ringp->s_ring_total_rbytes += sz;
4702 if ((mac_srs->srs_rx.sr_poll_pkt_cnt <= 1) &&
4703 !(ringp->s_ring_type & ST_RING_WORKER_ONLY)) {
4704 /* If on processor or blanking on, then enqueue and return */
4705 if (ringp->s_ring_state & S_RING_BLANK ||
4706 ringp->s_ring_state & S_RING_PROC) {
4707 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4708 mutex_exit(&ringp->s_ring_lock);
4709 return;
4710 }
4711 proc = ringp->s_ring_rx_func;
4712 arg1 = ringp->s_ring_rx_arg1;
4713 arg2 = ringp->s_ring_rx_arg2;
4714 /*
4715 * See if anything is already queued. If we are the
4716 * first packet, do inline processing else queue the
4717 * packet and do the drain.
4718 */
4719 if (ringp->s_ring_first == NULL) {
4720 /*
4721 * Fast-path, ok to process and nothing queued.
4722 */
4723 ringp->s_ring_run = curthread;
4724 ringp->s_ring_state |= (S_RING_PROC);
4725
4726 mutex_exit(&ringp->s_ring_lock);
4727
4728 /*
4729 * We are the chain of 1 packet so
4730 * go through this fast path.
4731 */
4732 ASSERT(mp_chain->b_next == NULL);
4733
4734 (*proc)(arg1, arg2, mp_chain, NULL);
4735
4736 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4737 /*
4738 * If we have an SRS performing bandwidth
4739 * control then we need to decrement the size
4740 * and count so the SRS has an accurate count
4741 * of the data queued between the SRS and its
4742 * soft rings. We decrement the counters only
4743 * when the packet is processed by both the
4744 * SRS and the soft ring.
4745 */
4746 mutex_enter(&mac_srs->srs_lock);
4747 MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
4748 MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
4749 mutex_exit(&mac_srs->srs_lock);
4750
4751 mutex_enter(&ringp->s_ring_lock);
4752 ringp->s_ring_run = NULL;
4753 ringp->s_ring_state &= ~S_RING_PROC;
4754 if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
4755 cv_signal(&ringp->s_ring_client_cv);
4756
4757 if ((ringp->s_ring_first == NULL) ||
4758 (ringp->s_ring_state & S_RING_BLANK)) {
4759 /*
4760 * We processed a single packet inline
4761 * and nothing new has arrived or our
4762 * receiver doesn't want to receive
4763 * any packets. We are done.
4764 */
4765 mutex_exit(&ringp->s_ring_lock);
4766 return;
4767 }
4768 } else {
4769 SOFT_RING_ENQUEUE_CHAIN(ringp,
4770 mp_chain, tail, cnt, sz);
4771 }
4772
4773 /*
4774 * We are here because either we couldn't do inline
4775 * processing (because something was already
4776 * queued), or we had a chain of more than one
4777 * packet, or something else arrived after we were
4778 * done with inline processing.
4779 */
4780 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
4781 ASSERT(ringp->s_ring_first != NULL);
4782
4783 ringp->s_ring_drain_func(ringp);
4784 mutex_exit(&ringp->s_ring_lock);
4785 return;
4786 } else {
4787 /* ST_RING_WORKER_ONLY case */
4788 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4789 mac_soft_ring_worker_wakeup(ringp);
4790 mutex_exit(&ringp->s_ring_lock);
4791 }
4792 }
4793
4794 /*
4795 * TX SOFTRING RELATED FUNCTIONS
4796 *
4797 * These functions really belong in mac_soft_ring.c and here for
4798 * a short period.
4799 */
4800
4801 #define TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) { \
4802 ASSERT(MUTEX_HELD(&ringp->s_ring_lock)); \
4803 ringp->s_ring_state |= S_RING_ENQUEUED; \
4804 SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz); \
4805 }
4806
4807 /*
4808 * mac_tx_sring_queued
4809 *
4810 * When we are out of transmit descriptors and we already have a
4811 * queue that exceeds hiwat (or the client called us with
4812 * MAC_TX_NO_ENQUEUE or MAC_DROP_ON_NO_DESC flag), return the
4813 * soft ring pointer as the opaque cookie for the client enable
4814 * flow control.
4815 */
4816 static mac_tx_cookie_t
4817 mac_tx_sring_enqueue(mac_soft_ring_t *ringp, mblk_t *mp_chain, uint16_t flag,
4818 mblk_t **ret_mp)
4819 {
4820 int cnt;
4821 size_t sz;
4822 mblk_t *tail;
4823 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
4824 mac_tx_cookie_t cookie = 0;
4825 boolean_t wakeup_worker = B_TRUE;
4826
4827 ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
4828 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
4829 if (flag & MAC_DROP_ON_NO_DESC) {
4830 mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
4831 /* increment freed stats */
4832 ringp->s_ring_drops += cnt;
4833 cookie = (mac_tx_cookie_t)ringp;
4834 } else {
4835 if (ringp->s_ring_first != NULL)
4836 wakeup_worker = B_FALSE;
4837
4838 if (flag & MAC_TX_NO_ENQUEUE) {
4839 /*
4840 * If QUEUED is not set, queue the packet
4841 * and let mac_tx_soft_ring_drain() set
4842 * the TX_BLOCKED bit for the reasons
4843 * explained above. Otherwise, return the
4844 * mblks.
4845 */
4846 if (wakeup_worker) {
4847 TX_SOFT_RING_ENQUEUE_CHAIN(ringp,
4848 mp_chain, tail, cnt, sz);
4849 } else {
4850 ringp->s_ring_state |= S_RING_WAKEUP_CLIENT;
4851 cookie = (mac_tx_cookie_t)ringp;
4852 *ret_mp = mp_chain;
4853 }
4854 } else {
4855 boolean_t enqueue = B_TRUE;
4856
4857 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
4858 /*
4859 * flow-controlled. Store ringp in cookie
4860 * so that it can be returned as
4861 * mac_tx_cookie_t to client
4862 */
4863 ringp->s_ring_state |= S_RING_TX_HIWAT;
4864 cookie = (mac_tx_cookie_t)ringp;
4865 ringp->s_ring_hiwat_cnt++;
4866 if (ringp->s_ring_count >
4867 ringp->s_ring_tx_max_q_cnt) {
4868 /* increment freed stats */
4869 ringp->s_ring_drops += cnt;
4870 /*
4871 * b_prev may be set to the fanout hint
4872 * hence can't use freemsg directly
4873 */
4874 mac_pkt_drop(NULL, NULL,
4875 mp_chain, B_FALSE);
4876 DTRACE_PROBE1(tx_queued_hiwat,
4877 mac_soft_ring_t *, ringp);
4878 enqueue = B_FALSE;
4879 }
4880 }
4881 if (enqueue) {
4882 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain,
4883 tail, cnt, sz);
4884 }
4885 }
4886 if (wakeup_worker)
4887 cv_signal(&ringp->s_ring_async);
4888 }
4889 return (cookie);
4890 }
4891
4892
4893 /*
4894 * mac_tx_soft_ring_process
4895 *
4896 * This routine is called when fanning out outgoing traffic among
4897 * multipe Tx rings.
4898 * Note that a soft ring is associated with a h/w Tx ring.
4899 */
4900 mac_tx_cookie_t
4901 mac_tx_soft_ring_process(mac_soft_ring_t *ringp, mblk_t *mp_chain,
4902 uint16_t flag, mblk_t **ret_mp)
4903 {
4904 mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
4905 int cnt;
4906 size_t sz;
4907 mblk_t *tail;
4908 mac_tx_cookie_t cookie = 0;
4909
4910 ASSERT(ringp != NULL);
4911 ASSERT(mp_chain != NULL);
4912 ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4913 /*
4914 * The following modes can come here: SRS_TX_BW_FANOUT,
4915 * SRS_TX_FANOUT, SRS_TX_AGGR, SRS_TX_BW_AGGR.
4916 */
4917 ASSERT(MAC_TX_SOFT_RINGS(mac_srs));
4918 ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
4919 mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT ||
4920 mac_srs->srs_tx.st_mode == SRS_TX_AGGR ||
4921 mac_srs->srs_tx.st_mode == SRS_TX_BW_AGGR);
4922
4923 if (ringp->s_ring_type & ST_RING_WORKER_ONLY) {
4924 /* Serialization mode */
4925
4926 mutex_enter(&ringp->s_ring_lock);
4927 if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
4928 cookie = mac_tx_sring_enqueue(ringp, mp_chain,
4929 flag, ret_mp);
4930 mutex_exit(&ringp->s_ring_lock);
4931 return (cookie);
4932 }
4933 MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
4934 TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4935 if (ringp->s_ring_state & (S_RING_BLOCK | S_RING_PROC)) {
4936 /*
4937 * If ring is blocked due to lack of Tx
4938 * descs, just return. Worker thread
4939 * will get scheduled when Tx desc's
4940 * become available.
4941 */
4942 mutex_exit(&ringp->s_ring_lock);
4943 return (cookie);
4944 }
4945 mac_soft_ring_worker_wakeup(ringp);
4946 mutex_exit(&ringp->s_ring_lock);
4947 return (cookie);
4948 } else {
4949 /* Default fanout mode */
4950 /*
4951 * S_RING_BLOCKED is set when underlying NIC runs
4952 * out of Tx descs and messages start getting
4953 * queued. It won't get reset until
4954 * tx_srs_drain() completely drains out the
4955 * messages.
4956 */
4957 mac_tx_stats_t stats;
4958
4959 if (ringp->s_ring_state & S_RING_ENQUEUED) {
4960 /* Tx descs/resources not available */
4961 mutex_enter(&ringp->s_ring_lock);
4962 if (ringp->s_ring_state & S_RING_ENQUEUED) {
4963 cookie = mac_tx_sring_enqueue(ringp, mp_chain,
4964 flag, ret_mp);
4965 mutex_exit(&ringp->s_ring_lock);
4966 return (cookie);
4967 }
4968 /*
4969 * While we were computing mblk count, the
4970 * flow control condition got relieved.
4971 * Continue with the transmission.
4972 */
4973 mutex_exit(&ringp->s_ring_lock);
4974 }
4975
4976 mp_chain = mac_tx_send(ringp->s_ring_tx_arg1,
4977 ringp->s_ring_tx_arg2, mp_chain, &stats);
4978
4979 /*
4980 * Multiple threads could be here sending packets.
4981 * Under such conditions, it is not possible to
4982 * automically set S_RING_BLOCKED bit to indicate
4983 * out of tx desc condition. To atomically set
4984 * this, we queue the returned packet and do
4985 * the setting of S_RING_BLOCKED in
4986 * mac_tx_soft_ring_drain().
4987 */
4988 if (mp_chain != NULL) {
4989 mutex_enter(&ringp->s_ring_lock);
4990 cookie =
4991 mac_tx_sring_enqueue(ringp, mp_chain, flag, ret_mp);
4992 mutex_exit(&ringp->s_ring_lock);
4993 return (cookie);
4994 }
4995 SRS_TX_STATS_UPDATE(mac_srs, &stats);
4996 SOFTRING_TX_STATS_UPDATE(ringp, &stats);
4997
4998 return (0);
4999 }
5000 }