[RESEND,net-next,v9,5/6] net: ngbe: add sriov function support

Commit Message

[e-mail] March 24, 2025, 2 a.m. UTC

Add sriov_configure for driver ops.
Add mailbox handler wx_msg_task for ngbe in
the interrupt handler.
Add the notification flow when the vfs exist.

Signed-off-by: Mengyuan Lou <[e-mail]>
---
 drivers/net/ethernet/wangxun/libwx/wx_sriov.c | 31 +++++++
 drivers/net/ethernet/wangxun/libwx/wx_sriov.h |  2 +
 drivers/net/ethernet/wangxun/libwx/wx_type.h  |  2 +
 drivers/net/ethernet/wangxun/ngbe/ngbe_main.c | 93 +++++++++++++++++--
 drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c |  5 +
 drivers/net/ethernet/wangxun/ngbe/ngbe_type.h |  3 +
 6 files changed, 127 insertions(+), 9 deletions(-)

Comments

Simon Horman March 24, 2025, 7:21 p.m. UTC | #1

On Mon, Mar 24, 2025 at 10:00:32AM +0800, Mengyuan Lou wrote:
> Add sriov_configure for driver ops.
> Add mailbox handler wx_msg_task for ngbe in
> the interrupt handler.
> Add the notification flow when the vfs exist.
> 
> Signed-off-by: Mengyuan Lou <[e-mail]>

...

> diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c

...

> @@ -200,12 +206,10 @@ static irqreturn_t ngbe_intr(int __always_unused irq, void *data)
>  	return IRQ_HANDLED;
>  }
>  
> -static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
> +static irqreturn_t ngbe_msix_common(struct wx *wx, u32 eicr)
>  {
> -	struct wx *wx = data;
> -	u32 eicr;
> -
> -	eicr = wx_misc_isb(wx, WX_ISB_MISC);
> +	if (eicr & NGBE_PX_MISC_IC_VF_MBOX)
> +		wx_msg_task(wx);
>  
>  	if (unlikely(eicr & NGBE_PX_MISC_IC_TIMESYNC))
>  		wx_ptp_check_pps_event(wx);
> @@ -217,6 +221,35 @@ static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
>  	return IRQ_HANDLED;
>  }
>  
> +static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
> +{
> +	struct wx *wx = data;
> +	u32 eicr;
> +
> +	eicr = wx_misc_isb(wx, WX_ISB_MISC);
> +
> +	return ngbe_msix_common(wx, eicr);
> +}
> +
> +static irqreturn_t ngbe_msic_and_queue(int __always_unused irq, void *data)
> +{
> +	struct wx_q_vector *q_vector;
> +	struct wx *wx = data;
> +	u32 eicr;
> +
> +	eicr = wx_misc_isb(wx, WX_ISB_MISC);
> +	if (!eicr) {
> +		/* queue */
> +		q_vector = wx->q_vector[0];
> +		napi_schedule_irqoff(&q_vector->napi);
> +		if (netif_running(wx->netdev))
> +			ngbe_irq_enable(wx, true);
> +		return IRQ_HANDLED;
> +	}
> +
> +	return ngbe_msix_common(wx, eicr);
> +}
> +
>  /**
>   * ngbe_request_msix_irqs - Initialize MSI-X interrupts
>   * @wx: board private structure
> @@ -249,8 +282,16 @@ static int ngbe_request_msix_irqs(struct wx *wx)
>  		}
>  	}
>  
> -	err = request_irq(wx->msix_entry->vector,
> -			  ngbe_msix_other, 0, netdev->name, wx);
> +	/* Due to hardware design, when num_vfs < 7, pf can use 0 for misc and 1
> +	 * for queue. But when num_vfs == 7, vector[1] is assigned to vf6.
> +	 * Misc and queue should reuse interrupt vector[0].
> +	 */
> +	if (wx->num_vfs == 7)
> +		err = request_irq(wx->msix_entry->vector,
> +				  ngbe_msic_and_queue, 0, netdev->name, wx);
> +	else
> +		err = request_irq(wx->msix_entry->vector,
> +				  ngbe_msix_other, 0, netdev->name, wx);

Sorry for the late review. It has been a busy time.

I have been thinking about the IRQ handler registration above in the
context of the feedback from Jakub on v7:

	"Do you have proper synchronization in place to make sure IRQs
         don't get mis-routed when SR-IOV is enabled?
         The goal should be to make sure the right handler is register
         for the IRQ, or at least do the muxing earlier in a safe fashion.
         Not decide that it was a packet IRQ half way thru a function called
         ngbe_msix_other"

	Link: https://lore.kernel.org/all/[e-mail]/

My understanding is that is that:

* In the case where num_vfs < 7, vector 1 is used by the pf for
  "queue". But when num_vfs == 7 (the maximum value), vector 1 is used
  by the VF6.

* Correspondingly, when num_vfs < 7 vector 0 is only used for
  "misc". While when num_vfs == 7 is used for both "misc" and "queue".

* The code registration above is about vector 0 (while other vectors are
  registered in the code just above this hunk).

* ngbe_msix_other only handles "misc" interrupts, while

* ngbe_msic_and_queue demuxes "misc" and "queue" interrupts
  (without evaluating num_vfs), handling "queue" interrupts inline
  and using a helper function, which is also used by ngbe_msix_other,
  to handle "misc" interrupts.

If so, I believe this addresses Jakub's concerns.

And given that we are at v9 and the last feedback of substance was the
above comment from Jakub, I think this looks good.

Reviewed-by: Simon Horman <[e-mail]>

But I would like to say that there could be some follow-up to align
the comment and the names of the handlers:

* "other" seems to be used as a synonym for "misc".
  Perhaps ngbe_msix_misc() ?
* "common" seems to only process "misc" interrupts.
  Perhaps __ngbe_msix_misc() ?
* msic seems to be a misspelling of misc.

>  
>  	if (err) {
>  		wx_err(wx, "request_irq for msix_other failed: %d\n", err);

...

[e-mail] March 25, 2025, 2:36 a.m. UTC | #2

> 2025年3月25日 03:21，Simon Horman <[e-mail]> 写道：
> 
> On Mon, Mar 24, 2025 at 10:00:32AM +0800, Mengyuan Lou wrote:
>> Add sriov_configure for driver ops.
>> Add mailbox handler wx_msg_task for ngbe in
>> the interrupt handler.
>> Add the notification flow when the vfs exist.
>> 
>> Signed-off-by: Mengyuan Lou <[e-mail]>
> 
> ...
> 
>> diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
> 
> ...
> 
>> @@ -200,12 +206,10 @@ static irqreturn_t ngbe_intr(int __always_unused irq, void *data)
>> return IRQ_HANDLED;
>> }
>> 
>> -static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
>> +static irqreturn_t ngbe_msix_common(struct wx *wx, u32 eicr)
>> {
>> - struct wx *wx = data;
>> - u32 eicr;
>> -
>> - eicr = wx_misc_isb(wx, WX_ISB_MISC);
>> + if (eicr & NGBE_PX_MISC_IC_VF_MBOX)
>> + wx_msg_task(wx);
>> 
>> if (unlikely(eicr & NGBE_PX_MISC_IC_TIMESYNC))
>> wx_ptp_check_pps_event(wx);
>> @@ -217,6 +221,35 @@ static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
>> return IRQ_HANDLED;
>> }
>> 
>> +static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
>> +{
>> + struct wx *wx = data;
>> + u32 eicr;
>> +
>> + eicr = wx_misc_isb(wx, WX_ISB_MISC);
>> +
>> + return ngbe_msix_common(wx, eicr);
>> +}
>> +
>> +static irqreturn_t ngbe_msic_and_queue(int __always_unused irq, void *data)
>> +{
>> + struct wx_q_vector *q_vector;
>> + struct wx *wx = data;
>> + u32 eicr;
>> +
>> + eicr = wx_misc_isb(wx, WX_ISB_MISC);
>> + if (!eicr) {
>> + /* queue */
>> + q_vector = wx->q_vector[0];
>> + napi_schedule_irqoff(&q_vector->napi);
>> + if (netif_running(wx->netdev))
>> + ngbe_irq_enable(wx, true);
>> + return IRQ_HANDLED;
>> + }
>> +
>> + return ngbe_msix_common(wx, eicr);
>> +}
>> +
>> /**
>>  * ngbe_request_msix_irqs - Initialize MSI-X interrupts
>>  * @wx: board private structure
>> @@ -249,8 +282,16 @@ static int ngbe_request_msix_irqs(struct wx *wx)
>> }
>> }
>> 
>> - err = request_irq(wx->msix_entry->vector,
>> -  ngbe_msix_other, 0, netdev->name, wx);
>> + /* Due to hardware design, when num_vfs < 7, pf can use 0 for misc and 1
>> + * for queue. But when num_vfs == 7, vector[1] is assigned to vf6.
>> + * Misc and queue should reuse interrupt vector[0].
>> + */
>> + if (wx->num_vfs == 7)
>> + err = request_irq(wx->msix_entry->vector,
>> +  ngbe_msic_and_queue, 0, netdev->name, wx);
>> + else
>> + err = request_irq(wx->msix_entry->vector,
>> +  ngbe_msix_other, 0, netdev->name, wx);
> 
> Sorry for the late review. It has been a busy time.
> 
> I have been thinking about the IRQ handler registration above in the
> context of the feedback from Jakub on v7:
> 
> "Do you have proper synchronization in place to make sure IRQs
>         don't get mis-routed when SR-IOV is enabled?
>         The goal should be to make sure the right handler is register
>         for the IRQ, or at least do the muxing earlier in a safe fashion.
>         Not decide that it was a packet IRQ half way thru a function called
>         ngbe_msix_other"
> 
> Link: https://lore.kernel.org/all/[e-mail]/
> 
> My understanding is that is that:
> 
> * In the case where num_vfs < 7, vector 1 is used by the pf for
>  "queue". But when num_vfs == 7 (the maximum value), vector 1 is used
>  by the VF6.
> 
> * Correspondingly, when num_vfs < 7 vector 0 is only used for
>  "misc". While when num_vfs == 7 is used for both "misc" and "queue".
> 
> * The code registration above is about vector 0 (while other vectors are
>  registered in the code just above this hunk).
> 
> * ngbe_msix_other only handles "misc" interrupts, while
> 
> * ngbe_msic_and_queue demuxes "misc" and "queue" interrupts
>  (without evaluating num_vfs), handling "queue" interrupts inline
>  and using a helper function, which is also used by ngbe_msix_other,
>  to handle "misc" interrupts.
> 
That’s right.
> If so, I believe this addresses Jakub's concerns.
> 
> And given that we are at v9 and the last feedback of substance was the
> above comment from Jakub, I think this looks good.
> 
> Reviewed-by: Simon Horman <[e-mail]>
> 
> But I would like to say that there could be some follow-up to align
> the comment and the names of the handlers:
> 
> * "other" seems to be used as a synonym for "misc".
>  Perhaps ngbe_msix_misc() ?
> * "common" seems to only process "misc" interrupts.
>  Perhaps __ngbe_msix_misc() ?
> * msic seems to be a misspelling of misc.
> 


>> +static irqreturn_t ngbe_msix_misc(int __always_unused irq, void *data)
>> +{
>>  ...
>> + return __ngbe_msix_misc(wx, eicr);
>> +}
>> +
>> +static irqreturn_t ngbe_misc_and_queue(int __always_unused irq, void *data)
>> +{
>>  ...
>> + return __ngbe_msix_misc(wx, eicr);


if (wx->num_vfs == 7)
	err = request_irq(wx->msix_entry->vector,
			  ngbe_misc_and_queue, 0, netdev->name, wx);
else
	err = request_irq(wx->msix_entry->vector,
			  ngbe_msix_misc, 0, netdev->name, wx);

It’s more appropriate.

Thanks！
>> 
>> if (err) {
>> wx_err(wx, "request_irq for msix_other failed: %d\n", err);
> 
> ...
> 
>

Simon Horman March 25, 2025, 3:20 p.m. UTC | #3

On Tue, Mar 25, 2025 at 10:36:00AM +0800, [e-mail] wrote:
> > 2025年3月25日 03:21，Simon Horman <[e-mail]> 写道：
> > On Mon, Mar 24, 2025 at 10:00:32AM +0800, Mengyuan Lou wrote:

...

> That’s right.
> > If so, I believe this addresses Jakub's concerns.
> > 
> > And given that we are at v9 and the last feedback of substance was the
> > above comment from Jakub, I think this looks good.
> > 
> > Reviewed-by: Simon Horman <[e-mail]>
> > 
> > But I would like to say that there could be some follow-up to align
> > the comment and the names of the handlers:
> > 
> > * "other" seems to be used as a synonym for "misc".
> >  Perhaps ngbe_msix_misc() ?
> > * "common" seems to only process "misc" interrupts.
> >  Perhaps __ngbe_msix_misc() ?
> > * msic seems to be a misspelling of misc.
> > 
> 
> 
> >> +static irqreturn_t ngbe_msix_misc(int __always_unused irq, void *data)
> >> +{
> >>  ...
> >> + return __ngbe_msix_misc(wx, eicr);
> >> +}
> >> +
> >> +static irqreturn_t ngbe_misc_and_queue(int __always_unused irq, void *data)
> >> +{
> >>  ...
> >> + return __ngbe_msix_misc(wx, eicr);
> 
> 
> if (wx->num_vfs == 7)
> 	err = request_irq(wx->msix_entry->vector,
> 			  ngbe_misc_and_queue, 0, netdev->name, wx);
> else
> 	err = request_irq(wx->msix_entry->vector,
> 			  ngbe_msix_misc, 0, netdev->name, wx);
> 
> It’s more appropriate.
> 
> Thanks！

Yes, thanks.

And I also think it would also be nice to rename ngbe_msix_common()
as __ngbe_msix_misc().

But please hold off with any updates or follow-up as net-next is now closed
for the merge window.

Link: https://lore.kernel.org/netdev/[e-mail]/

...

Patch

diff --git a/drivers/net/ethernet/wangxun/libwx/wx_sriov.c b/drivers/net/ethernet/wangxun/libwx/wx_sriov.c
index 1a7cadbf7234..a31b574a343e 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_sriov.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_sriov.c
@@ -834,3 +834,34 @@  void wx_msg_task(struct wx *wx)
 	}
 }
 EXPORT_SYMBOL(wx_msg_task);
+
+void wx_disable_vf_rx_tx(struct wx *wx)
+{
+	wr32(wx, WX_TDM_VFTE_CLR(0), U32_MAX);
+	wr32(wx, WX_RDM_VFRE_CLR(0), U32_MAX);
+	if (wx->mac.type != wx_mac_em) {
+		wr32(wx, WX_TDM_VFTE_CLR(1), U32_MAX);
+		wr32(wx, WX_RDM_VFRE_CLR(1), U32_MAX);
+	}
+}
+EXPORT_SYMBOL(wx_disable_vf_rx_tx);
+
+void wx_ping_all_vfs_with_link_status(struct wx *wx, bool link_up)
+{
+	u32 msgbuf[2] = {0, 0};
+	u16 i;
+
+	if (!wx->num_vfs)
+		return;
+	msgbuf[0] = WX_PF_NOFITY_VF_LINK_STATUS | WX_PF_CONTROL_MSG;
+	if (link_up)
+		msgbuf[1] = FIELD_PREP(GENMASK(31, 1), wx->speed) | link_up;
+	if (wx->notify_down)
+		msgbuf[1] |= WX_PF_NOFITY_VF_NET_NOT_RUNNING;
+	for (i = 0; i < wx->num_vfs; i++) {
+		if (wx->vfinfo[i].clear_to_send)
+			msgbuf[0] |= WX_VT_MSGTYPE_CTS;
+		wx_write_mbx_pf(wx, msgbuf, 2, i);
+	}
+}
+EXPORT_SYMBOL(wx_ping_all_vfs_with_link_status);
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_sriov.h b/drivers/net/ethernet/wangxun/libwx/wx_sriov.h
index d3f29617c7d3..376d8e0e49f3 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_sriov.h
+++ b/drivers/net/ethernet/wangxun/libwx/wx_sriov.h
@@ -11,5 +11,7 @@ 
 void wx_disable_sriov(struct wx *wx);
 int wx_pci_sriov_configure(struct pci_dev *pdev, int num_vfs);
 void wx_msg_task(struct wx *wx);
+void wx_disable_vf_rx_tx(struct wx *wx);
+void wx_ping_all_vfs_with_link_status(struct wx *wx, bool link_up);
 
 #endif /* _WX_SRIOV_H_ */
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h
index a5ca2ca0aba7..9b9345290594 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_type.h
+++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h
@@ -92,6 +92,7 @@ 
 /************************* Port Registers ************************************/
 /* port cfg Registers */
 #define WX_CFG_PORT_CTL              0x14400
+#define WX_CFG_PORT_CTL_PFRSTD       BIT(14)
 #define WX_CFG_PORT_CTL_DRV_LOAD     BIT(3)
 #define WX_CFG_PORT_CTL_QINQ         BIT(2)
 #define WX_CFG_PORT_CTL_D_VLAN       BIT(0) /* double vlan*/
@@ -1231,6 +1232,7 @@  struct wx {
 	u8 swfw_index;
 
 	/* PHY stuff */
+	bool notify_down;
 	unsigned int link;
 	int speed;
 	int duplex;
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
index a6159214ec0a..d321c7f23b0b 100644
--- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
@@ -15,6 +15,8 @@ 
 #include "../libwx/wx_hw.h"
 #include "../libwx/wx_lib.h"
 #include "../libwx/wx_ptp.h"
+#include "../libwx/wx_mbx.h"
+#include "../libwx/wx_sriov.h"
 #include "ngbe_type.h"
 #include "ngbe_mdio.h"
 #include "ngbe_hw.h"
@@ -129,6 +131,10 @@  static int ngbe_sw_init(struct wx *wx)
 	wx->tx_work_limit = NGBE_DEFAULT_TX_WORK;
 	wx->rx_work_limit = NGBE_DEFAULT_RX_WORK;
 
+	wx->mbx.size = WX_VXMAILBOX_SIZE;
+	wx->setup_tc = ngbe_setup_tc;
+	set_bit(0, &wx->fwd_bitmask);
+
 	return 0;
 }
 
@@ -200,12 +206,10 @@  static irqreturn_t ngbe_intr(int __always_unused irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
+static irqreturn_t ngbe_msix_common(struct wx *wx, u32 eicr)
 {
-	struct wx *wx = data;
-	u32 eicr;
-
-	eicr = wx_misc_isb(wx, WX_ISB_MISC);
+	if (eicr & NGBE_PX_MISC_IC_VF_MBOX)
+		wx_msg_task(wx);
 
 	if (unlikely(eicr & NGBE_PX_MISC_IC_TIMESYNC))
 		wx_ptp_check_pps_event(wx);
@@ -217,6 +221,35 @@  static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t ngbe_msix_other(int __always_unused irq, void *data)
+{
+	struct wx *wx = data;
+	u32 eicr;
+
+	eicr = wx_misc_isb(wx, WX_ISB_MISC);
+
+	return ngbe_msix_common(wx, eicr);
+}
+
+static irqreturn_t ngbe_msic_and_queue(int __always_unused irq, void *data)
+{
+	struct wx_q_vector *q_vector;
+	struct wx *wx = data;
+	u32 eicr;
+
+	eicr = wx_misc_isb(wx, WX_ISB_MISC);
+	if (!eicr) {
+		/* queue */
+		q_vector = wx->q_vector[0];
+		napi_schedule_irqoff(&q_vector->napi);
+		if (netif_running(wx->netdev))
+			ngbe_irq_enable(wx, true);
+		return IRQ_HANDLED;
+	}
+
+	return ngbe_msix_common(wx, eicr);
+}
+
 /**
  * ngbe_request_msix_irqs - Initialize MSI-X interrupts
  * @wx: board private structure
@@ -249,8 +282,16 @@  static int ngbe_request_msix_irqs(struct wx *wx)
 		}
 	}
 
-	err = request_irq(wx->msix_entry->vector,
-			  ngbe_msix_other, 0, netdev->name, wx);
+	/* Due to hardware design, when num_vfs < 7, pf can use 0 for misc and 1
+	 * for queue. But when num_vfs == 7, vector[1] is assigned to vf6.
+	 * Misc and queue should reuse interrupt vector[0].
+	 */
+	if (wx->num_vfs == 7)
+		err = request_irq(wx->msix_entry->vector,
+				  ngbe_msic_and_queue, 0, netdev->name, wx);
+	else
+		err = request_irq(wx->msix_entry->vector,
+				  ngbe_msix_other, 0, netdev->name, wx);
 
 	if (err) {
 		wx_err(wx, "request_irq for msix_other failed: %d\n", err);
@@ -302,6 +343,22 @@  static void ngbe_disable_device(struct wx *wx)
 	struct net_device *netdev = wx->netdev;
 	u32 i;
 
+	if (wx->num_vfs) {
+		/* Clear EITR Select mapping */
+		wr32(wx, WX_PX_ITRSEL, 0);
+
+		/* Mark all the VFs as inactive */
+		for (i = 0; i < wx->num_vfs; i++)
+			wx->vfinfo[i].clear_to_send = 0;
+		wx->notify_down = true;
+		/* ping all the active vfs to let them know we are going down */
+		wx_ping_all_vfs_with_link_status(wx, false);
+		wx->notify_down = false;
+
+		/* Disable all VFTE/VFRE TX/RX */
+		wx_disable_vf_rx_tx(wx);
+	}
+
 	/* disable all enabled rx queues */
 	for (i = 0; i < wx->num_rx_queues; i++)
 		/* this call also flushes the previous write */
@@ -324,12 +381,19 @@  static void ngbe_disable_device(struct wx *wx)
 	wx_update_stats(wx);
 }
 
+static void ngbe_reset(struct wx *wx)
+{
+	wx_flush_sw_mac_table(wx);
+	wx_mac_set_default_filter(wx, wx->mac.addr);
+	if (test_bit(WX_STATE_PTP_RUNNING, wx->state))
+		wx_ptp_reset(wx);
+}
+
 void ngbe_down(struct wx *wx)
 {
 	phylink_stop(wx->phylink);
 	ngbe_disable_device(wx);
-	if (test_bit(WX_STATE_PTP_RUNNING, wx->state))
-		wx_ptp_reset(wx);
+	ngbe_reset(wx);
 	wx_clean_all_tx_rings(wx);
 	wx_clean_all_rx_rings(wx);
 }
@@ -352,6 +416,11 @@  void ngbe_up(struct wx *wx)
 		ngbe_sfp_modules_txrx_powerctl(wx, true);
 
 	phylink_start(wx->phylink);
+	/* Set PF Reset Done bit so PF/VF Mail Ops can work */
+	wr32m(wx, WX_CFG_PORT_CTL,
+	      WX_CFG_PORT_CTL_PFRSTD, WX_CFG_PORT_CTL_PFRSTD);
+	if (wx->num_vfs)
+		wx_ping_all_vfs_with_link_status(wx, false);
 }
 
 /**
@@ -596,6 +665,10 @@  static int ngbe_probe(struct pci_dev *pdev,
 		goto err_pci_release_regions;
 	}
 
+	/* The emerald supports up to 8 VFs per pf, but physical
+	 * function also need one pool for basic networking.
+	 */
+	pci_sriov_set_totalvfs(pdev, NGBE_MAX_VFS_DRV_LIMIT);
 	wx->driver_name = ngbe_driver_name;
 	ngbe_set_ethtool_ops(netdev);
 	netdev->netdev_ops = &ngbe_netdev_ops;
@@ -743,6 +816,7 @@  static void ngbe_remove(struct pci_dev *pdev)
 	struct net_device *netdev;
 
 	netdev = wx->netdev;
+	wx_disable_sriov(wx);
 	unregister_netdev(netdev);
 	phylink_destroy(wx->phylink);
 	pci_release_selected_regions(pdev,
@@ -802,6 +876,7 @@  static struct pci_driver ngbe_driver = {
 	.suspend  = ngbe_suspend,
 	.resume   = ngbe_resume,
 	.shutdown = ngbe_shutdown,
+	.sriov_configure = wx_pci_sriov_configure,
 };
 
 module_pci_driver(ngbe_driver);
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c
index ea1d7e9a91f3..c63bb6e6f405 100644
--- a/drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_mdio.c
@@ -9,6 +9,7 @@ 
 #include "../libwx/wx_type.h"
 #include "../libwx/wx_ptp.h"
 #include "../libwx/wx_hw.h"
+#include "../libwx/wx_sriov.h"
 #include "ngbe_type.h"
 #include "ngbe_mdio.h"
 
@@ -70,6 +71,8 @@  static void ngbe_mac_link_down(struct phylink_config *config,
 	wx->speed = SPEED_UNKNOWN;
 	if (test_bit(WX_STATE_PTP_RUNNING, wx->state))
 		wx_ptp_reset_cyclecounter(wx);
+	/* ping all the active vfs to let them know we are going down */
+	wx_ping_all_vfs_with_link_status(wx, false);
 }
 
 static void ngbe_mac_link_up(struct phylink_config *config,
@@ -114,6 +117,8 @@  static void ngbe_mac_link_up(struct phylink_config *config,
 	wx->last_rx_ptp_check = jiffies;
 	if (test_bit(WX_STATE_PTP_RUNNING, wx->state))
 		wx_ptp_reset_cyclecounter(wx);
+	/* ping all the active vfs to let them know we are going up */
+	wx_ping_all_vfs_with_link_status(wx, true);
 }
 
 static const struct phylink_mac_ops ngbe_mac_ops = {
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h
index 992adbb98c7d..bb74263f0498 100644
--- a/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h
@@ -73,12 +73,14 @@ 
 #define NGBE_PX_MISC_IEN_TIMESYNC		BIT(11)
 #define NGBE_PX_MISC_IEN_ETH_LK			BIT(18)
 #define NGBE_PX_MISC_IEN_INT_ERR		BIT(20)
+#define NGBE_PX_MISC_IC_VF_MBOX			BIT(23)
 #define NGBE_PX_MISC_IEN_GPIO			BIT(26)
 #define NGBE_PX_MISC_IEN_MASK ( \
 				NGBE_PX_MISC_IEN_DEV_RST | \
 				NGBE_PX_MISC_IEN_TIMESYNC | \
 				NGBE_PX_MISC_IEN_ETH_LK | \
 				NGBE_PX_MISC_IEN_INT_ERR | \
+				NGBE_PX_MISC_IC_VF_MBOX | \
 				NGBE_PX_MISC_IEN_GPIO)
 
 /* Extended Interrupt Cause Read */
@@ -134,6 +136,7 @@ 
 #define NGBE_MAX_RXD				8192
 #define NGBE_MIN_RXD				128
 
+#define NGBE_MAX_VFS_DRV_LIMIT			7
 extern char ngbe_driver_name[];
 
 void ngbe_down(struct wx *wx);