Print this page
11493 aggr needs support for multiple pseudo rx groups
Portions contributed by: Dan McDonald <danmcd@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>


  54 
  55 static kmem_cache_t *aggr_port_cache;
  56 static id_space_t *aggr_portids;
  57 
  58 static void aggr_port_notify_cb(void *, mac_notify_type_t);
  59 
  60 /*ARGSUSED*/
  61 static int
  62 aggr_port_constructor(void *buf, void *arg, int kmflag)
  63 {
  64         bzero(buf, sizeof (aggr_port_t));
  65         return (0);
  66 }
  67 
  68 /*ARGSUSED*/
  69 static void
  70 aggr_port_destructor(void *buf, void *arg)
  71 {
  72         aggr_port_t *port = buf;
  73 
  74         ASSERT(port->lp_mnh == NULL);
  75         ASSERT(port->lp_mphp == NULL);
  76         ASSERT(!port->lp_rx_grp_added && !port->lp_tx_grp_added);
  77         ASSERT(port->lp_hwgh == NULL);
  78 }
  79 
  80 void
  81 aggr_port_init(void)
  82 {
  83         aggr_port_cache = kmem_cache_create("aggr_port_cache",
  84             sizeof (aggr_port_t), 0, aggr_port_constructor,
  85             aggr_port_destructor, NULL, NULL, NULL, 0);
  86 
  87         /*
  88          * Allocate a id space to manage port identification. The range of
  89          * the arena will be from 1 to UINT16_MAX, because the LACP protocol
  90          * specifies 16-bit unique identification.
  91          */
  92         aggr_portids = id_space_create("aggr_portids", 1, UINT16_MAX);
  93         ASSERT(aggr_portids != NULL);
  94 }
  95 
  96 void
  97 aggr_port_fini(void)


 111 {
 112         /* add the port's receive callback */
 113         port->lp_mnh = mac_notify_add(port->lp_mh, aggr_port_notify_cb, port);
 114         /*
 115          * Hold a reference of the grp and the port and this reference will
 116          * be released when the thread exits.
 117          *
 118          * The reference on the port is used for aggr_port_delete() to
 119          * continue without waiting for the thread to exit; the reference
 120          * on the grp is used for aggr_grp_delete() to wait for the thread
 121          * to exit before calling mac_unregister().
 122          *
 123          * Note that these references will be released either in
 124          * aggr_port_delete() when mac_notify_remove() succeeds, or in
 125          * the aggr_port_notify_cb() callback when the port is deleted
 126          * (lp_closing is set).
 127          */
 128         aggr_grp_port_hold(port);
 129 }
 130 
 131 /* ARGSUSED */
 132 int
 133 aggr_port_create(aggr_grp_t *grp, const datalink_id_t linkid, boolean_t force,
 134     aggr_port_t **pp)
 135 {
 136         int err;
 137         mac_handle_t mh;
 138         mac_client_handle_t mch = NULL;
 139         aggr_port_t *port;
 140         uint16_t portid;
 141         uint_t i;
 142         boolean_t no_link_update = B_FALSE;
 143         const mac_info_t *mip;
 144         uint32_t note;
 145         uint32_t margin;
 146         char client_name[MAXNAMELEN];
 147         char aggr_name[MAXNAMELEN];
 148         char port_name[MAXNAMELEN];
 149         mac_diag_t diag;
 150         mac_unicast_handle_t mah;
 151 


 180 
 181         if (((err = dls_mgmt_get_linkinfo(grp->lg_linkid,
 182             aggr_name, NULL, NULL, NULL)) != 0) ||
 183             ((err = dls_mgmt_get_linkinfo(linkid, port_name,
 184             NULL, NULL, NULL)) != 0)) {
 185                 goto fail;
 186         }
 187 
 188         (void) snprintf(client_name, MAXNAMELEN, "%s-%s", aggr_name, port_name);
 189         if ((err = mac_client_open(mh, &mch, client_name,
 190             MAC_OPEN_FLAGS_IS_AGGR_PORT | MAC_OPEN_FLAGS_EXCLUSIVE)) != 0) {
 191                 goto fail;
 192         }
 193 
 194         if ((portid = (uint16_t)id_alloc(aggr_portids)) == 0) {
 195                 err = ENOMEM;
 196                 goto fail;
 197         }
 198 
 199         /*
 200          * As the underlying mac's current margin size is used to determine
 201          * the margin size of the aggregation itself, request the underlying
 202          * mac not to change to a smaller size.
 203          */
 204         if ((err = mac_margin_add(mh, &margin, B_TRUE)) != 0) {
 205                 id_free(aggr_portids, portid);
 206                 goto fail;
 207         }
 208 
 209         if ((err = mac_unicast_add(mch, NULL, MAC_UNICAST_PRIMARY |
 210             MAC_UNICAST_DISABLE_TX_VID_CHECK, &mah, 0, &diag)) != 0) {
 211                 VERIFY(mac_margin_remove(mh, margin) == 0);
 212                 id_free(aggr_portids, portid);
 213                 goto fail;
 214         }
 215 
 216         port = kmem_cache_alloc(aggr_port_cache, KM_SLEEP);
 217 
 218         port->lp_refs = 1;
 219         port->lp_next = NULL;
 220         port->lp_mh = mh;
 221         port->lp_mch = mch;
 222         port->lp_mip = mip;
 223         port->lp_linkid = linkid;
 224         port->lp_closing = B_FALSE;
 225         port->lp_mah = mah;
 226 
 227         /* get the port's original MAC address */
 228         mac_unicast_primary_get(port->lp_mh, port->lp_addr);
 229 
 230         /* initialize state */
 231         port->lp_state = AGGR_PORT_STATE_STANDBY;


 246          * the constituent ports.
 247          */
 248         for (i = 0; i < MAC_NSTAT; i++) {
 249                 port->lp_stat[i] =
 250                     aggr_port_stat(port, i + MAC_STAT_MIN);
 251         }
 252         for (i = 0; i < ETHER_NSTAT; i++) {
 253                 port->lp_ether_stat[i] =
 254                     aggr_port_stat(port, i + MACTYPE_STAT_MIN);
 255         }
 256 
 257         /* LACP related state */
 258         port->lp_collector_enabled = B_FALSE;
 259 
 260         *pp = port;
 261         return (0);
 262 
 263 fail:
 264         if (mch != NULL)
 265                 mac_client_close(mch, MAC_CLOSE_FLAGS_EXCLUSIVE);

 266         mac_close(mh);
 267         return (err);
 268 }
 269 
 270 void
 271 aggr_port_delete(aggr_port_t *port)
 272 {
 273         aggr_lacp_port_t *pl = &port->lp_lacp;
 274 
 275         ASSERT(port->lp_mphp == NULL);
 276         ASSERT(!port->lp_promisc_on);
 277 
 278         port->lp_closing = B_TRUE;


 279 
 280         VERIFY(mac_margin_remove(port->lp_mh, port->lp_margin) == 0);
 281         mac_rx_clear(port->lp_mch);
 282         /*
 283          * If the notification callback is already in process and waiting for
 284          * the aggr grp's mac perimeter, don't wait (otherwise there would be
 285          * deadlock). Otherwise, if mac_notify_remove() succeeds, we can
 286          * release the reference held when mac_notify_add() is called.
 287          */
 288         if ((port->lp_mnh != NULL) &&
 289             (mac_notify_remove(port->lp_mnh, B_FALSE) == 0)) {
 290                 aggr_grp_port_rele(port);
 291         }
 292         port->lp_mnh = NULL;
 293 
 294         /*
 295          * Inform the the port lacp timer thread to exit. Note that waiting
 296          * for the thread to exit may cause deadlock since that thread may
 297          * need to enter into the mac perimeter which we are currently in.
 298          * It is fine to continue without waiting though since that thread
 299          * is holding a reference of the port.
 300          */
 301         mutex_enter(&pl->lacp_timer_lock);
 302         pl->lacp_timer_bits |= LACP_THREAD_EXIT;
 303         cv_broadcast(&pl->lacp_timer_cv);
 304         mutex_exit(&pl->lacp_timer_lock);
 305 
 306         /*
 307          * Restore the port MAC address. Note it is called after the
 308          * port's notification callback being removed. This prevent
 309          * port's MAC_NOTE_UNICST notify callback function being called.
 310          */
 311         (void) mac_unicast_primary_set(port->lp_mh, port->lp_addr);

 312         if (port->lp_mah != NULL)
 313                 (void) mac_unicast_remove(port->lp_mch, port->lp_mah);

 314         mac_client_close(port->lp_mch, MAC_CLOSE_FLAGS_EXCLUSIVE);
 315         mac_close(port->lp_mh);
 316         AGGR_PORT_REFRELE(port);
 317 }
 318 
 319 void
 320 aggr_port_free(aggr_port_t *port)
 321 {
 322         ASSERT(port->lp_refs == 0);
 323         if (port->lp_grp != NULL)
 324                 AGGR_GRP_REFRELE(port->lp_grp);
 325         port->lp_grp = NULL;
 326         id_free(aggr_portids, port->lp_portid);
 327         port->lp_portid = 0;
 328         mutex_destroy(&port->lp_lacp.lacp_timer_lock);
 329         cv_destroy(&port->lp_lacp.lacp_timer_cv);
 330         kmem_cache_free(aggr_port_cache, port);
 331 }
 332 
 333 /*


 504 
 505         port->lp_started = B_TRUE;
 506         aggr_grp_multicst_port(port, B_TRUE);
 507         return (0);
 508 }
 509 
 510 void
 511 aggr_port_stop(aggr_port_t *port)
 512 {
 513         ASSERT(MAC_PERIM_HELD(port->lp_mh));
 514 
 515         if (!port->lp_started)
 516                 return;
 517 
 518         aggr_grp_multicst_port(port, B_FALSE);
 519 
 520         /* update the port state */
 521         port->lp_started = B_FALSE;
 522 }
 523 




 524 int
 525 aggr_port_promisc(aggr_port_t *port, boolean_t on)
 526 {
 527         int rc;
 528 
 529         ASSERT(MAC_PERIM_HELD(port->lp_mh));
 530 
 531         if (on == port->lp_promisc_on)
 532                 /* already in desired promiscous mode */
 533                 return (0);
 534 
 535         if (on) {
 536                 mac_rx_clear(port->lp_mch);
 537 
 538                 /*
 539                  * We use the promisc callback because without hardware
 540                  * rings, we deliver through flows that will cause duplicate
 541                  * delivery of packets when we've flipped into this mode
 542                  * to compensate for the lack of hardware MAC matching
 543                  */
 544                 rc = mac_promisc_add(port->lp_mch, MAC_CLIENT_PROMISC_ALL,
 545                     aggr_recv_promisc_cb, port, &port->lp_mphp,
 546                     MAC_PROMISC_FLAGS_NO_TX_LOOP);
 547                 if (rc != 0) {
 548                         mac_rx_set(port->lp_mch, aggr_recv_cb, port);
 549                         return (rc);
 550                 }
 551         } else {
 552                 mac_promisc_remove(port->lp_mphp);
 553                 port->lp_mphp = NULL;
 554                 mac_rx_set(port->lp_mch, aggr_recv_cb, port);
 555         }
 556 
 557         port->lp_promisc_on = on;
 558 
 559         return (0);
 560 }
 561 
 562 /*
 563  * Set the MAC address of a port.
 564  */
 565 int
 566 aggr_port_unicst(aggr_port_t *port)
 567 {
 568         aggr_grp_t              *grp = port->lp_grp;
 569 
 570         ASSERT(MAC_PERIM_HELD(grp->lg_mh));
 571         ASSERT(MAC_PERIM_HELD(port->lp_mh));
 572 
 573         return (mac_unicast_primary_set(port->lp_mh, grp->lg_addr));
 574 }
 575 
 576 /*
 577  * Add or remove a multicast address to/from a port.
 578  */
 579 int
 580 aggr_port_multicst(void *arg, boolean_t add, const uint8_t *addrp)
 581 {
 582         aggr_port_t *port = arg;
 583 
 584         if (add) {
 585                 return (mac_multicast_add(port->lp_mch, addrp));
 586         } else {
 587                 mac_multicast_remove(port->lp_mch, addrp);
 588                 return (0);
 589         }
 590 }
 591 
 592 uint64_t
 593 aggr_port_stat(aggr_port_t *port, uint_t stat)
 594 {
 595         return (mac_stat_get(port->lp_mh, stat));
 596 }
 597 
 598 /*
 599  * Add a non-primary unicast address to the underlying port. If the port
 600  * supports HW Rx group, try to add the address into the HW Rx group of
 601  * the port first. If that fails, or if the port does not support HW Rx
 602  * group, enable the port's promiscous mode.
 603  */
 604 int
 605 aggr_port_addmac(aggr_port_t *port, const uint8_t *mac_addr)
 606 {
 607         aggr_unicst_addr_t      *addr, **pprev;
 608         mac_perim_handle_t      pmph;
 609         int                     err;
 610 
 611         ASSERT(MAC_PERIM_HELD(port->lp_grp->lg_mh));

 612         mac_perim_enter_by_mh(port->lp_mh, &pmph);
 613 
 614         /*
 615          * If the underlying port support HW Rx group, add the mac to its
 616          * RX group directly.


 617          */
 618         if ((port->lp_hwgh != NULL) &&
 619             ((mac_hwgroup_addmac(port->lp_hwgh, mac_addr)) == 0)) {







 620                 mac_perim_exit(pmph);
 621                 return (0);
 622         }
 623 
 624         /*
 625          * If that fails, or if the port does not support HW Rx group, enable
 626          * the port's promiscous mode. (Note that we turn on the promiscous
 627          * mode only if the port is already started.
 628          */
 629         if (port->lp_started &&
 630             ((err = aggr_port_promisc(port, B_TRUE)) != 0)) {
 631                 mac_perim_exit(pmph);
 632                 return (err);
 633         }
 634 
 635         /*
 636          * Walk through the unicast addresses that requires promiscous mode
 637          * enabled on this port, and add this address to the end of the list.
 638          */
 639         pprev = &port->lp_prom_addr;
 640         while ((addr = *pprev) != NULL) {
 641                 ASSERT(bcmp(mac_addr, addr->aua_addr, ETHERADDRL) != 0);
 642                 pprev = &addr->aua_next;
 643         }
 644         addr = kmem_alloc(sizeof (aggr_unicst_addr_t), KM_SLEEP);
 645         bcopy(mac_addr, addr->aua_addr, ETHERADDRL);
 646         addr->aua_next = NULL;
 647         *pprev = addr;
 648         mac_perim_exit(pmph);
 649         return (0);
 650 }
 651 
 652 /*
 653  * Remove a non-primary unicast address from the underlying port. This address
 654  * must has been added by aggr_port_addmac(). As a result, we probably need to
 655  * remove the address from the port's HW Rx group, or to disable the port's
 656  * promiscous mode.
 657  */
 658 void
 659 aggr_port_remmac(aggr_port_t *port, const uint8_t *mac_addr)
 660 {
 661         aggr_grp_t              *grp = port->lp_grp;
 662         aggr_unicst_addr_t      *addr, **pprev;
 663         mac_perim_handle_t      pmph;
 664 
 665         ASSERT(MAC_PERIM_HELD(grp->lg_mh));

 666         mac_perim_enter_by_mh(port->lp_mh, &pmph);
 667 
 668         /*
 669          * See whether this address is in the list of addresses that requires
 670          * the port being promiscous mode.
 671          */
 672         pprev = &port->lp_prom_addr;
 673         while ((addr = *pprev) != NULL) {
 674                 if (bcmp(mac_addr, addr->aua_addr, ETHERADDRL) == 0)
 675                         break;
 676                 pprev = &addr->aua_next;
 677         }

 678         if (addr != NULL) {
 679                 /*
 680                  * This unicast address put the port into the promiscous mode,
 681                  * delete this address from the lp_prom_addr list. If this is
 682                  * the last address in that list, disable the promiscous mode
 683                  * if the aggregation is not in promiscous mode.
 684                  */
 685                 *pprev = addr->aua_next;
 686                 kmem_free(addr, sizeof (aggr_unicst_addr_t));
 687                 if (port->lp_prom_addr == NULL && !grp->lg_promisc)
 688                         (void) aggr_port_promisc(port, B_FALSE);
 689         } else {
 690                 ASSERT(port->lp_hwgh != NULL);
 691                 (void) mac_hwgroup_remmac(port->lp_hwgh, mac_addr);




 692         }

 693         mac_perim_exit(pmph);
 694 }
 695 
 696 int
 697 aggr_port_addvlan(aggr_port_t *port, uint16_t vid)
 698 {
 699         mac_perim_handle_t      pmph;
 700         int                     err;
 701 
 702         ASSERT(MAC_PERIM_HELD(port->lp_grp->lg_mh));

 703         mac_perim_enter_by_mh(port->lp_mh, &pmph);
 704 




 705         /*
 706          * Add the VLAN filter to the HW group if the port has a HW
 707          * group. If the port doesn't have a HW group, then it will
 708          * implicitly allow tagged traffic to pass and there is
 709          * nothing to do.
 710          */
 711         if (port->lp_hwgh == NULL)
 712                 return (0);


 713 
 714         err = mac_hwgroup_addvlan(port->lp_hwgh, vid);
 715         mac_perim_exit(pmph);
 716         return (err);
 717 }
 718 
 719 int
 720 aggr_port_remvlan(aggr_port_t *port, uint16_t vid)
 721 {
 722         mac_perim_handle_t      pmph;
 723         int                     err;
 724 
 725         ASSERT(MAC_PERIM_HELD(port->lp_grp->lg_mh));

 726         mac_perim_enter_by_mh(port->lp_mh, &pmph);
 727 
 728         if (port->lp_hwgh == NULL)
 729                 return (0);

 730 
 731         err = mac_hwgroup_remvlan(port->lp_hwgh, vid);




 732         mac_perim_exit(pmph);
 733         return (err);
 734 }


  54 
  55 static kmem_cache_t *aggr_port_cache;
  56 static id_space_t *aggr_portids;
  57 
  58 static void aggr_port_notify_cb(void *, mac_notify_type_t);
  59 
  60 /*ARGSUSED*/
  61 static int
  62 aggr_port_constructor(void *buf, void *arg, int kmflag)
  63 {
  64         bzero(buf, sizeof (aggr_port_t));
  65         return (0);
  66 }
  67 
  68 /*ARGSUSED*/
  69 static void
  70 aggr_port_destructor(void *buf, void *arg)
  71 {
  72         aggr_port_t *port = buf;
  73 
  74         ASSERT3P(port->lp_mnh, ==, NULL);
  75         ASSERT(!port->lp_tx_grp_added);
  76         for (uint_t i = 0; i < MAX_GROUPS_PER_PORT; i++)
  77                 ASSERT3P(port->lp_hwghs[i], ==, NULL);
  78 }
  79 
  80 void
  81 aggr_port_init(void)
  82 {
  83         aggr_port_cache = kmem_cache_create("aggr_port_cache",
  84             sizeof (aggr_port_t), 0, aggr_port_constructor,
  85             aggr_port_destructor, NULL, NULL, NULL, 0);
  86 
  87         /*
  88          * Allocate a id space to manage port identification. The range of
  89          * the arena will be from 1 to UINT16_MAX, because the LACP protocol
  90          * specifies 16-bit unique identification.
  91          */
  92         aggr_portids = id_space_create("aggr_portids", 1, UINT16_MAX);
  93         ASSERT(aggr_portids != NULL);
  94 }
  95 
  96 void
  97 aggr_port_fini(void)


 111 {
 112         /* add the port's receive callback */
 113         port->lp_mnh = mac_notify_add(port->lp_mh, aggr_port_notify_cb, port);
 114         /*
 115          * Hold a reference of the grp and the port and this reference will
 116          * be released when the thread exits.
 117          *
 118          * The reference on the port is used for aggr_port_delete() to
 119          * continue without waiting for the thread to exit; the reference
 120          * on the grp is used for aggr_grp_delete() to wait for the thread
 121          * to exit before calling mac_unregister().
 122          *
 123          * Note that these references will be released either in
 124          * aggr_port_delete() when mac_notify_remove() succeeds, or in
 125          * the aggr_port_notify_cb() callback when the port is deleted
 126          * (lp_closing is set).
 127          */
 128         aggr_grp_port_hold(port);
 129 }
 130 

 131 int
 132 aggr_port_create(aggr_grp_t *grp, const datalink_id_t linkid, boolean_t force,
 133     aggr_port_t **pp)
 134 {
 135         int err;
 136         mac_handle_t mh;
 137         mac_client_handle_t mch = NULL;
 138         aggr_port_t *port;
 139         uint16_t portid;
 140         uint_t i;
 141         boolean_t no_link_update = B_FALSE;
 142         const mac_info_t *mip;
 143         uint32_t note;
 144         uint32_t margin;
 145         char client_name[MAXNAMELEN];
 146         char aggr_name[MAXNAMELEN];
 147         char port_name[MAXNAMELEN];
 148         mac_diag_t diag;
 149         mac_unicast_handle_t mah;
 150 


 179 
 180         if (((err = dls_mgmt_get_linkinfo(grp->lg_linkid,
 181             aggr_name, NULL, NULL, NULL)) != 0) ||
 182             ((err = dls_mgmt_get_linkinfo(linkid, port_name,
 183             NULL, NULL, NULL)) != 0)) {
 184                 goto fail;
 185         }
 186 
 187         (void) snprintf(client_name, MAXNAMELEN, "%s-%s", aggr_name, port_name);
 188         if ((err = mac_client_open(mh, &mch, client_name,
 189             MAC_OPEN_FLAGS_IS_AGGR_PORT | MAC_OPEN_FLAGS_EXCLUSIVE)) != 0) {
 190                 goto fail;
 191         }
 192 
 193         if ((portid = (uint16_t)id_alloc(aggr_portids)) == 0) {
 194                 err = ENOMEM;
 195                 goto fail;
 196         }
 197 
 198         /*
 199          * As the underlying MAC's current margin size is used to determine
 200          * the margin size of the aggregation itself, request the underlying
 201          * MAC not to change to a smaller size.
 202          */
 203         if ((err = mac_margin_add(mh, &margin, B_TRUE)) != 0) {
 204                 id_free(aggr_portids, portid);
 205                 goto fail;
 206         }
 207 
 208         if ((err = mac_unicast_add(mch, NULL, MAC_UNICAST_PRIMARY |
 209             MAC_UNICAST_DISABLE_TX_VID_CHECK, &mah, 0, &diag)) != 0) {
 210                 VERIFY3S(mac_margin_remove(mh, margin), ==, 0);
 211                 id_free(aggr_portids, portid);
 212                 goto fail;
 213         }
 214 
 215         port = kmem_cache_alloc(aggr_port_cache, KM_SLEEP);
 216 
 217         port->lp_refs = 1;
 218         port->lp_next = NULL;
 219         port->lp_mh = mh;
 220         port->lp_mch = mch;
 221         port->lp_mip = mip;
 222         port->lp_linkid = linkid;
 223         port->lp_closing = B_FALSE;
 224         port->lp_mah = mah;
 225 
 226         /* get the port's original MAC address */
 227         mac_unicast_primary_get(port->lp_mh, port->lp_addr);
 228 
 229         /* initialize state */
 230         port->lp_state = AGGR_PORT_STATE_STANDBY;


 245          * the constituent ports.
 246          */
 247         for (i = 0; i < MAC_NSTAT; i++) {
 248                 port->lp_stat[i] =
 249                     aggr_port_stat(port, i + MAC_STAT_MIN);
 250         }
 251         for (i = 0; i < ETHER_NSTAT; i++) {
 252                 port->lp_ether_stat[i] =
 253                     aggr_port_stat(port, i + MACTYPE_STAT_MIN);
 254         }
 255 
 256         /* LACP related state */
 257         port->lp_collector_enabled = B_FALSE;
 258 
 259         *pp = port;
 260         return (0);
 261 
 262 fail:
 263         if (mch != NULL)
 264                 mac_client_close(mch, MAC_CLOSE_FLAGS_EXCLUSIVE);
 265 
 266         mac_close(mh);
 267         return (err);
 268 }
 269 
 270 void
 271 aggr_port_delete(aggr_port_t *port)
 272 {
 273         aggr_lacp_port_t *pl = &port->lp_lacp;
 274 

 275         ASSERT(!port->lp_promisc_on);

 276         port->lp_closing = B_TRUE;
 277         VERIFY0(mac_margin_remove(port->lp_mh, port->lp_margin));
 278         mac_client_clear_flow_cb(port->lp_mch);
 279 


 280         /*
 281          * If the notification callback is already in process and waiting for
 282          * the aggr grp's mac perimeter, don't wait (otherwise there would be
 283          * deadlock). Otherwise, if mac_notify_remove() succeeds, we can
 284          * release the reference held when mac_notify_add() is called.
 285          */
 286         if ((port->lp_mnh != NULL) &&
 287             (mac_notify_remove(port->lp_mnh, B_FALSE) == 0)) {
 288                 aggr_grp_port_rele(port);
 289         }
 290         port->lp_mnh = NULL;
 291 
 292         /*
 293          * Inform the the port lacp timer thread to exit. Note that waiting
 294          * for the thread to exit may cause deadlock since that thread may
 295          * need to enter into the mac perimeter which we are currently in.
 296          * It is fine to continue without waiting though since that thread
 297          * is holding a reference of the port.
 298          */
 299         mutex_enter(&pl->lacp_timer_lock);
 300         pl->lacp_timer_bits |= LACP_THREAD_EXIT;
 301         cv_broadcast(&pl->lacp_timer_cv);
 302         mutex_exit(&pl->lacp_timer_lock);
 303 
 304         /*
 305          * Restore the port MAC address. Note it is called after the
 306          * port's notification callback being removed. This prevent
 307          * port's MAC_NOTE_UNICST notify callback function being called.
 308          */
 309         (void) mac_unicast_primary_set(port->lp_mh, port->lp_addr);
 310 
 311         if (port->lp_mah != NULL)
 312                 (void) mac_unicast_remove(port->lp_mch, port->lp_mah);
 313 
 314         mac_client_close(port->lp_mch, MAC_CLOSE_FLAGS_EXCLUSIVE);
 315         mac_close(port->lp_mh);
 316         AGGR_PORT_REFRELE(port);
 317 }
 318 
 319 void
 320 aggr_port_free(aggr_port_t *port)
 321 {
 322         ASSERT(port->lp_refs == 0);
 323         if (port->lp_grp != NULL)
 324                 AGGR_GRP_REFRELE(port->lp_grp);
 325         port->lp_grp = NULL;
 326         id_free(aggr_portids, port->lp_portid);
 327         port->lp_portid = 0;
 328         mutex_destroy(&port->lp_lacp.lacp_timer_lock);
 329         cv_destroy(&port->lp_lacp.lacp_timer_cv);
 330         kmem_cache_free(aggr_port_cache, port);
 331 }
 332 
 333 /*


 504 
 505         port->lp_started = B_TRUE;
 506         aggr_grp_multicst_port(port, B_TRUE);
 507         return (0);
 508 }
 509 
 510 void
 511 aggr_port_stop(aggr_port_t *port)
 512 {
 513         ASSERT(MAC_PERIM_HELD(port->lp_mh));
 514 
 515         if (!port->lp_started)
 516                 return;
 517 
 518         aggr_grp_multicst_port(port, B_FALSE);
 519 
 520         /* update the port state */
 521         port->lp_started = B_FALSE;
 522 }
 523 
 524 /*
 525  * Set the promisc mode of the port. If the port is already in the
 526  * requested mode then do nothing.
 527  */
 528 int
 529 aggr_port_promisc(aggr_port_t *port, boolean_t on)
 530 {
 531         int rc;
 532 
 533         ASSERT(MAC_PERIM_HELD(port->lp_mh));
 534 
 535         if (on == port->lp_promisc_on)

 536                 return (0);
 537 
 538         rc = mac_set_promisc(port->lp_mh, on);

 539 
 540         if (rc == 0)


















 541                 port->lp_promisc_on = on;
 542 
 543         return (rc);
 544 }
 545 
 546 /*
 547  * Set the MAC address of a port.
 548  */
 549 int
 550 aggr_port_unicst(aggr_port_t *port)
 551 {
 552         aggr_grp_t              *grp = port->lp_grp;
 553 
 554         ASSERT(MAC_PERIM_HELD(grp->lg_mh));
 555         ASSERT(MAC_PERIM_HELD(port->lp_mh));
 556 
 557         return (mac_unicast_primary_set(port->lp_mh, grp->lg_addr));
 558 }
 559 
 560 /*
 561  * Add or remove a multicast address to/from a port.
 562  */
 563 int
 564 aggr_port_multicst(void *arg, boolean_t add, const uint8_t *addrp)
 565 {
 566         aggr_port_t *port = arg;
 567 
 568         if (add) {
 569                 return (mac_multicast_add(port->lp_mch, addrp));
 570         } else {
 571                 mac_multicast_remove(port->lp_mch, addrp);
 572                 return (0);
 573         }
 574 }
 575 
 576 uint64_t
 577 aggr_port_stat(aggr_port_t *port, uint_t stat)
 578 {
 579         return (mac_stat_get(port->lp_mh, stat));
 580 }
 581 
 582 /*
 583  * Add a non-primary unicast address to the underlying port. If the
 584  * port supports HW Rx groups, then try to add the address filter to
 585  * the HW group first. If that fails, or if the port does not support
 586  * RINGS capab, then enable the port's promiscous mode.
 587  */
 588 int
 589 aggr_port_addmac(aggr_port_t *port, uint_t idx, const uint8_t *mac_addr)
 590 {
 591         aggr_unicst_addr_t      *addr, **pprev;
 592         mac_perim_handle_t      pmph;
 593         int                     err;
 594 
 595         ASSERT(MAC_PERIM_HELD(port->lp_grp->lg_mh));
 596         ASSERT3U(idx, <, MAX_GROUPS_PER_PORT);
 597         mac_perim_enter_by_mh(port->lp_mh, &pmph);
 598 
 599         /*
 600          * If the port doesn't have a HW group to back the aggr's
 601          * pseudo group, then try using the port's default group and
 602          * let the aggr SW classify its traffic. This scenario happens
 603          * when mixing ports with a different number of HW groups.
 604          */
 605         if (port->lp_hwghs[idx] == NULL)
 606                 idx = 0;
 607 
 608         /*
 609          * If there is an underlying HW Rx group, then try adding this
 610          * unicast address to it.
 611          */
 612         if ((port->lp_hwghs[idx] != NULL) &&
 613             ((mac_hwgroup_addmac(port->lp_hwghs[idx], mac_addr)) == 0)) {
 614                 mac_perim_exit(pmph);
 615                 return (0);
 616         }
 617 
 618         /*
 619          * If the port doesn't have HW groups, or we failed to add the
 620          * HW filter, then enable the port's promiscuous mode. We
 621          * enable promiscuous mode only if the port is already started.
 622          */
 623         if (port->lp_started &&
 624             ((err = aggr_port_promisc(port, B_TRUE)) != 0)) {
 625                 mac_perim_exit(pmph);
 626                 return (err);
 627         }
 628 
 629         /*
 630          * Walk through the unicast addresses that requires promiscous mode
 631          * enabled on this port, and add this address to the end of the list.
 632          */
 633         pprev = &port->lp_prom_addr;
 634         while ((addr = *pprev) != NULL) {
 635                 ASSERT(bcmp(mac_addr, addr->aua_addr, ETHERADDRL) != 0);
 636                 pprev = &addr->aua_next;
 637         }
 638         addr = kmem_alloc(sizeof (aggr_unicst_addr_t), KM_SLEEP);
 639         bcopy(mac_addr, addr->aua_addr, ETHERADDRL);
 640         addr->aua_next = NULL;
 641         *pprev = addr;
 642         mac_perim_exit(pmph);
 643         return (0);
 644 }
 645 
 646 /*
 647  * Remove a non-primary unicast address from the underlying port. This address
 648  * must has been added by aggr_port_addmac(). As a result, we probably need to
 649  * remove the address from the port's HW Rx group, or to disable the port's
 650  * promiscous mode.
 651  */
 652 void
 653 aggr_port_remmac(aggr_port_t *port, uint_t idx, const uint8_t *mac_addr)
 654 {
 655         aggr_grp_t              *grp = port->lp_grp;
 656         aggr_unicst_addr_t      *addr, **pprev;
 657         mac_perim_handle_t      pmph;
 658 
 659         ASSERT(MAC_PERIM_HELD(grp->lg_mh));
 660         ASSERT3U(idx, <, MAX_GROUPS_PER_PORT);
 661         mac_perim_enter_by_mh(port->lp_mh, &pmph);
 662 
 663         /*
 664          * See whether this address is in the list of addresses that requires
 665          * the port being promiscous mode.
 666          */
 667         pprev = &port->lp_prom_addr;
 668         while ((addr = *pprev) != NULL) {
 669                 if (bcmp(mac_addr, addr->aua_addr, ETHERADDRL) == 0)
 670                         break;
 671                 pprev = &addr->aua_next;
 672         }
 673 
 674         if (addr != NULL) {
 675                 /*
 676                  * This unicast address put the port into the promiscous mode,
 677                  * delete this address from the lp_prom_addr list. If this is
 678                  * the last address in that list, disable the promiscous mode
 679                  * if the aggregation is not in promiscous mode.
 680                  */
 681                 *pprev = addr->aua_next;
 682                 kmem_free(addr, sizeof (aggr_unicst_addr_t));
 683                 if (port->lp_prom_addr == NULL && !grp->lg_promisc)
 684                         (void) aggr_port_promisc(port, B_FALSE);
 685         } else {
 686                 /* See comment in aggr_port_addmac(). */
 687                 if (port->lp_hwghs[idx] == NULL)
 688                         idx = 0;
 689 
 690                 ASSERT3P(port->lp_hwghs[idx], !=, NULL);
 691                 (void) mac_hwgroup_remmac(port->lp_hwghs[idx], mac_addr);
 692         }
 693 
 694         mac_perim_exit(pmph);
 695 }
 696 
 697 int
 698 aggr_port_addvlan(aggr_port_t *port, uint_t idx, uint16_t vid)
 699 {
 700         mac_perim_handle_t      pmph;
 701         int                     err;
 702 
 703         ASSERT(MAC_PERIM_HELD(port->lp_grp->lg_mh));
 704         ASSERT3U(idx, <, MAX_GROUPS_PER_PORT);
 705         mac_perim_enter_by_mh(port->lp_mh, &pmph);
 706 
 707         /* See comment in aggr_port_addmac(). */
 708         if (port->lp_hwghs[idx] == NULL)
 709                 idx = 0;
 710 
 711         /*
 712          * Add the VLAN filter to the HW group if the port has a HW
 713          * group. If the port doesn't have a HW group, then it will
 714          * implicitly allow tagged traffic to pass and there is
 715          * nothing to do.
 716          */
 717         if (port->lp_hwghs[idx] == NULL)
 718                 err = 0;
 719         else
 720                 err = mac_hwgroup_addvlan(port->lp_hwghs[idx], vid);
 721 

 722         mac_perim_exit(pmph);
 723         return (err);
 724 }
 725 
 726 int
 727 aggr_port_remvlan(aggr_port_t *port, uint_t idx, uint16_t vid)
 728 {
 729         mac_perim_handle_t      pmph;
 730         int                     err;
 731 
 732         ASSERT(MAC_PERIM_HELD(port->lp_grp->lg_mh));
 733         ASSERT3U(idx, <, MAX_GROUPS_PER_PORT);
 734         mac_perim_enter_by_mh(port->lp_mh, &pmph);
 735 
 736         /* See comment in aggr_port_addmac(). */
 737         if (port->lp_hwghs[idx] == NULL)
 738                 idx = 0;
 739 
 740         if (port->lp_hwghs[idx] == NULL)
 741                 err = 0;
 742         else
 743                 err = mac_hwgroup_remvlan(port->lp_hwghs[idx], vid);
 744 
 745         mac_perim_exit(pmph);
 746         return (err);
 747 }