[v4,4/4] mac80211: Use Airtime-based Queue Limits (AQL) on packet dequeue
diff mbox series

Message ID 157148503865.2989444.7118792679603045723.stgit@toke.dk
State New
Headers show
Series
  • Add Airtime Queue Limits (AQL) to mac80211
Related show

Commit Message

Toke Høiland-Jørgensen Oct. 19, 2019, 11:37 a.m. UTC
From: Toke Høiland-Jørgensen <toke@redhat.com>

The previous commit added the ability to throttle stations when they queue
too much airtime in the hardware. This commit enables the functionality by
calculating the expected airtime usage of each packet that is dequeued from
the TXQs in mac80211, and accounting that as pending airtime.

The estimated airtime for each skb is stored in the tx_info, so we can
subtract the same amount from the running total when the skb is freed or
recycled. The throttling mechanism relies on this accounting to be
accurate (i.e., that we are not freeing skbs without subtracting any
airtime they were accounted for), so we put the subtraction into
ieee80211_report_used_skb(). As an optimisation, we also subtract the
airtime on regular TX completion, zeroing out the value stored in the
packet afterwards, to avoid having to do an expensive lookup of the
station from the packet data on every packet.

This patch does *not* include any mechanism to wake a throttled TXQ again,
on the assumption that this will happen anyway as a side effect of whatever
freed the skb (most commonly a TX completion).

Signed-off-by: Toke Høiland-Jørgensen <toke@redhat.com>
---
 net/mac80211/status.c |   38 ++++++++++++++++++++++++++++++++++++++
 net/mac80211/tx.c     |   21 +++++++++++++++++++++
 2 files changed, 59 insertions(+)

Comments

Kan Yan Oct. 22, 2019, 6:20 a.m. UTC | #1
> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
> +                       qc = ieee80211_get_qos_ctl(hdr);
> +                       tid = qc[0] & 0xf;
> +                       ac = ieee80211_ac_from_tid(tid);
> +               } else {
> +                       ac = IEEE80211_AC_BE;
> +               }

The tid/ac is incorrect either here or in __ieee80211_tx_status() when
tested with ath10k. The ac is set to AC_BE with test done using BK
class traffic,  hence the pending airtime get updated for the wrong
txq.

The rest of the patch seems to work as expected, after I did a quick
hack to release the pending airtime from ath10k_txrx_tx_unref()
instead, where the ac/tid can be directly retrieved from struck struct
ieee80211_tx.


On Sat, Oct 19, 2019 at 4:37 AM Toke Høiland-Jørgensen <toke@redhat.com> wrote:
>
> From: Toke Høiland-Jørgensen <toke@redhat.com>
>
> The previous commit added the ability to throttle stations when they queue
> too much airtime in the hardware. This commit enables the functionality by
> calculating the expected airtime usage of each packet that is dequeued from
> the TXQs in mac80211, and accounting that as pending airtime.
>
> The estimated airtime for each skb is stored in the tx_info, so we can
> subtract the same amount from the running total when the skb is freed or
> recycled. The throttling mechanism relies on this accounting to be
> accurate (i.e., that we are not freeing skbs without subtracting any
> airtime they were accounted for), so we put the subtraction into
> ieee80211_report_used_skb(). As an optimisation, we also subtract the
> airtime on regular TX completion, zeroing out the value stored in the
> packet afterwards, to avoid having to do an expensive lookup of the
> station from the packet data on every packet.
>
> This patch does *not* include any mechanism to wake a throttled TXQ again,
> on the assumption that this will happen anyway as a side effect of whatever
> freed the skb (most commonly a TX completion).
>
> Signed-off-by: Toke Høiland-Jørgensen <toke@redhat.com>
> ---
>  net/mac80211/status.c |   38 ++++++++++++++++++++++++++++++++++++++
>  net/mac80211/tx.c     |   21 +++++++++++++++++++++
>  2 files changed, 59 insertions(+)
>
> diff --git a/net/mac80211/status.c b/net/mac80211/status.c
> index ab8ba5835ca0..905b4afd457d 100644
> --- a/net/mac80211/status.c
> +++ b/net/mac80211/status.c
> @@ -676,6 +676,33 @@ static void ieee80211_report_used_skb(struct ieee80211_local *local,
>         if (dropped)
>                 acked = false;
>
> +       if (info->tx_time_est) {
> +               struct ieee80211_sub_if_data *sdata;
> +               struct sta_info *sta = NULL;
> +               u8 *qc, ac;
> +               int tid;
> +
> +               rcu_read_lock();
> +
> +               sdata = ieee80211_sdata_from_skb(local, skb);
> +               if (sdata)
> +                       sta = sta_info_get_bss(sdata, skb_mac_header(skb));
> +
> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
> +                       qc = ieee80211_get_qos_ctl(hdr);
> +                       tid = qc[0] & 0xf;
> +                       ac = ieee80211_ac_from_tid(tid);
> +               } else {
> +                       ac = IEEE80211_AC_BE;
> +               }
> +
> +               ieee80211_sta_update_pending_airtime(local, sta, ac,
> +                                                    info->tx_time_est << 2,
> +                                                    true);
> +               rcu_read_unlock();
> +
> +       }
> +
>         if (info->flags & IEEE80211_TX_INTFL_MLME_CONN_TX) {
>                 struct ieee80211_sub_if_data *sdata;
>
> @@ -930,6 +957,17 @@ static void __ieee80211_tx_status(struct ieee80211_hw *hw,
>                         tid = qc[0] & 0xf;
>                 }
>
> +               if (info->tx_time_est) {
> +                       /* Do this here to avoid the expensive lookup of the sta
> +                        * in ieee80211_report_used_skb().
> +                        */
> +                       ieee80211_sta_update_pending_airtime(local, sta,
> +                                                            ieee80211_ac_from_tid(tid),
> +                                                            info->tx_time_est << 2,
> +                                                            true);
> +                       info->tx_time_est = 0;
> +               }
> +
>                 if (!acked && ieee80211_is_back_req(fc)) {
>                         u16 control;
>
> diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
> index 12653d873b8c..b8ff56d1d661 100644
> --- a/net/mac80211/tx.c
> +++ b/net/mac80211/tx.c
> @@ -3542,6 +3542,9 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
>
>         WARN_ON_ONCE(softirq_count() == 0);
>
> +       if (!ieee80211_txq_airtime_check(hw, txq))
> +               return NULL;
> +
>  begin:
>         spin_lock_bh(&fq->lock);
>
> @@ -3652,6 +3655,24 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
>         }
>
>         IEEE80211_SKB_CB(skb)->control.vif = vif;
> +
> +       if (local->airtime_flags & AIRTIME_USE_AQL) {
> +               u32 airtime;
> +
> +               airtime = ieee80211_calc_expected_tx_airtime(hw, vif, txq->sta,
> +                                                            skb->len);
> +               if (airtime) {
> +                       /* We only have 10 bits in tx_time_est, so store airtime
> +                        * in increments of 4 us and clamp the maximum to 2**12-1
> +                        */
> +                       airtime = min_t(u32, airtime, 4095) & ~3U;
> +                       info->tx_time_est = airtime >> 2;
> +                       ieee80211_sta_update_pending_airtime(local, tx.sta,
> +                                                            txq->ac, airtime,
> +                                                            false);
> +               }
> +       }
> +
>         return skb;
>
>  out:
>
Toke Høiland-Jørgensen Oct. 22, 2019, 7:14 a.m. UTC | #2
Kan Yan <kyan@google.com> writes:

>> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
>> +                       qc = ieee80211_get_qos_ctl(hdr);
>> +                       tid = qc[0] & 0xf;
>> +                       ac = ieee80211_ac_from_tid(tid);
>> +               } else {
>> +                       ac = IEEE80211_AC_BE;
>> +               }
>
> The tid/ac is incorrect either here or in __ieee80211_tx_status() when
> tested with ath10k. The ac is set to AC_BE with test done using BK
> class traffic, hence the pending airtime get updated for the wrong
> txq.

Huh, well that won't do, obviously :)

Any idea why it might be wrong?

> The rest of the patch seems to work as expected, after I did a quick
> hack to release the pending airtime from ath10k_txrx_tx_unref()
> instead, where the ac/tid can be directly retrieved from struck struct
> ieee80211_tx.

Awesome! Thanks for testing!

-Toke
Toke Høiland-Jørgensen Oct. 22, 2019, 11:35 a.m. UTC | #3
Kan Yan <kyan@google.com> writes:

>> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
>> +                       qc = ieee80211_get_qos_ctl(hdr);
>> +                       tid = qc[0] & 0xf;
>> +                       ac = ieee80211_ac_from_tid(tid);
>> +               } else {
>> +                       ac = IEEE80211_AC_BE;
>> +               }
>
> The tid/ac is incorrect either here or in __ieee80211_tx_status() when
> tested with ath10k. The ac is set to AC_BE with test done using BK
> class traffic,  hence the pending airtime get updated for the wrong
> txq.

Hmm, I guess we could just get the ac using skb_get_queue_mapping().
I'll send an update with this fixed for you to try :)

-Toke
Kan Yan Oct. 23, 2019, 6:41 a.m. UTC | #4
> >> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
> >> +                       qc = ieee80211_get_qos_ctl(hdr);
> >> +                       tid = qc[0] & 0xf;
> >> +                       ac = ieee80211_ac_from_tid(tid);
> >> +               } else {
> >> +                       ac = IEEE80211_AC_BE;
> >> +               }
> >
> > The tid/ac is incorrect either here or in __ieee80211_tx_status() when
> > tested with ath10k. The ac is set to AC_BE with test done using BK
> > class traffic, hence the pending airtime get updated for the wrong
> > txq.
>
> Huh, well that won't do, obviously :)
>
> Any idea why it might be wrong?

somehow  ieee80211_is_data_qos() returns false. Besides,  qos_control
field doesn't seems to be set in ieee80211_build_hdr().

> Hmm, I guess we could just get the ac using skb_get_queue_mapping().
> I'll send an update with this fixed for you to try :)
Thanks for the quick update. It is getting much better, but
unfortunately the pending airtime accounting sometimes is still not
correct and cause txq stuck occasionally.





On Tue, Oct 22, 2019 at 4:35 AM Toke Høiland-Jørgensen <toke@redhat.com> wrote:
>
> Kan Yan <kyan@google.com> writes:
>
> >> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
> >> +                       qc = ieee80211_get_qos_ctl(hdr);
> >> +                       tid = qc[0] & 0xf;
> >> +                       ac = ieee80211_ac_from_tid(tid);
> >> +               } else {
> >> +                       ac = IEEE80211_AC_BE;
> >> +               }
> >
> > The tid/ac is incorrect either here or in __ieee80211_tx_status() when
> > tested with ath10k. The ac is set to AC_BE with test done using BK
> > class traffic,  hence the pending airtime get updated for the wrong
> > txq.
>
> Hmm, I guess we could just get the ac using skb_get_queue_mapping().
> I'll send an update with this fixed for you to try :)
>
> -Toke
>
Toke Høiland-Jørgensen Oct. 23, 2019, 8:18 a.m. UTC | #5
Kan Yan <kyan@google.com> writes:

>> >> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
>> >> +                       qc = ieee80211_get_qos_ctl(hdr);
>> >> +                       tid = qc[0] & 0xf;
>> >> +                       ac = ieee80211_ac_from_tid(tid);
>> >> +               } else {
>> >> +                       ac = IEEE80211_AC_BE;
>> >> +               }
>> >
>> > The tid/ac is incorrect either here or in __ieee80211_tx_status() when
>> > tested with ath10k. The ac is set to AC_BE with test done using BK
>> > class traffic, hence the pending airtime get updated for the wrong
>> > txq.
>>
>> Huh, well that won't do, obviously :)
>>
>> Any idea why it might be wrong?
>
> somehow  ieee80211_is_data_qos() returns false. Besides,  qos_control
> field doesn't seems to be set in ieee80211_build_hdr().
>
>> Hmm, I guess we could just get the ac using skb_get_queue_mapping().
>> I'll send an update with this fixed for you to try :)
> Thanks for the quick update. It is getting much better, but
> unfortunately the pending airtime accounting sometimes is still not
> correct and cause txq stuck occasionally.

OK, so that has to mean that there are packets getting dropped somewhere
without going through ieee80211_report_used_skb(). Assuming you're not
hitting the underflow warnings, just seeing the counter not get back
down to zero?

Could you see if you can find out if ath10k does that anywhere? I'll go
hunting in mac80211.

Looking for calls to kfree_skb() or kfree_skb_list() should hopefully
turn up something...

-Toke
Toke Høiland-Jørgensen Oct. 23, 2019, 9:52 a.m. UTC | #6
Toke Høiland-Jørgensen <toke@redhat.com> writes:

> Kan Yan <kyan@google.com> writes:
>
>>> >> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
>>> >> +                       qc = ieee80211_get_qos_ctl(hdr);
>>> >> +                       tid = qc[0] & 0xf;
>>> >> +                       ac = ieee80211_ac_from_tid(tid);
>>> >> +               } else {
>>> >> +                       ac = IEEE80211_AC_BE;
>>> >> +               }
>>> >
>>> > The tid/ac is incorrect either here or in __ieee80211_tx_status() when
>>> > tested with ath10k. The ac is set to AC_BE with test done using BK
>>> > class traffic, hence the pending airtime get updated for the wrong
>>> > txq.
>>>
>>> Huh, well that won't do, obviously :)
>>>
>>> Any idea why it might be wrong?
>>
>> somehow  ieee80211_is_data_qos() returns false. Besides,  qos_control
>> field doesn't seems to be set in ieee80211_build_hdr().
>>
>>> Hmm, I guess we could just get the ac using skb_get_queue_mapping().
>>> I'll send an update with this fixed for you to try :)
>> Thanks for the quick update. It is getting much better, but
>> unfortunately the pending airtime accounting sometimes is still not
>> correct and cause txq stuck occasionally.
>
> OK, so that has to mean that there are packets getting dropped somewhere
> without going through ieee80211_report_used_skb(). Assuming you're not
> hitting the underflow warnings, just seeing the counter not get back
> down to zero?
>
> Could you see if you can find out if ath10k does that anywhere? I'll go
> hunting in mac80211.
>
> Looking for calls to kfree_skb() or kfree_skb_list() should hopefully
> turn up something...

Aha! Turns out I was doing the sta lookup completely wrong in
ieee80211_report_used_skb(); so anything frames that were dropped and
went through there would not get its airtime subtracted correctly. Will
send a v6 with a fix :)

-Toke
Kan Yan Oct. 23, 2019, 3:43 p.m. UTC | #7
> Aha! Turns out I was doing the sta lookup completely wrong in
> ieee80211_report_used_skb(); so anything frames that were dropped and
> went through there would not get its airtime subtracted correctly. Will
> send a v6 with a fix :)

Awesome, thanks! That looks very promising.  The symptom I see with
previous patch is the interface's pending airtime count looks fine,
but the station's airtime get wrong, either due to airtime is credited
to the wrong station or wrong AC.

On Wed, Oct 23, 2019 at 2:52 AM Toke Høiland-Jørgensen <toke@redhat.com> wrote:
>
> Toke Høiland-Jørgensen <toke@redhat.com> writes:
>
> > Kan Yan <kyan@google.com> writes:
> >
> >>> >> +               if (ieee80211_is_data_qos(hdr->frame_control)) {
> >>> >> +                       qc = ieee80211_get_qos_ctl(hdr);
> >>> >> +                       tid = qc[0] & 0xf;
> >>> >> +                       ac = ieee80211_ac_from_tid(tid);
> >>> >> +               } else {
> >>> >> +                       ac = IEEE80211_AC_BE;
> >>> >> +               }
> >>> >
> >>> > The tid/ac is incorrect either here or in __ieee80211_tx_status() when
> >>> > tested with ath10k. The ac is set to AC_BE with test done using BK
> >>> > class traffic, hence the pending airtime get updated for the wrong
> >>> > txq.
> >>>
> >>> Huh, well that won't do, obviously :)
> >>>
> >>> Any idea why it might be wrong?
> >>
> >> somehow  ieee80211_is_data_qos() returns false. Besides,  qos_control
> >> field doesn't seems to be set in ieee80211_build_hdr().
> >>
> >>> Hmm, I guess we could just get the ac using skb_get_queue_mapping().
> >>> I'll send an update with this fixed for you to try :)
> >> Thanks for the quick update. It is getting much better, but
> >> unfortunately the pending airtime accounting sometimes is still not
> >> correct and cause txq stuck occasionally.
> >
> > OK, so that has to mean that there are packets getting dropped somewhere
> > without going through ieee80211_report_used_skb(). Assuming you're not
> > hitting the underflow warnings, just seeing the counter not get back
> > down to zero?
> >
> > Could you see if you can find out if ath10k does that anywhere? I'll go
> > hunting in mac80211.
> >
> > Looking for calls to kfree_skb() or kfree_skb_list() should hopefully
> > turn up something...
>
> Aha! Turns out I was doing the sta lookup completely wrong in
> ieee80211_report_used_skb(); so anything frames that were dropped and
> went through there would not get its airtime subtracted correctly. Will
> send a v6 with a fix :)
>
> -Toke
>
Toke Høiland-Jørgensen Oct. 23, 2019, 9:09 p.m. UTC | #8
Kan Yan <kyan@google.com> writes:

>>
>> Aha! Turns out I was doing the sta lookup completely wrong in
>> ieee80211_report_used_skb(); so anything frames that were dropped and
>> went through there would not get its airtime subtracted correctly. Will
>> send a v6 with a fix :)
>
> Awesome, thanks! That looks very promising.  The symptom I see with
> previous patch is the interface's pending airtime count looks fine, but the
> station's airtime get wrong, either due to airtime is credited to the wrong
> station or wrong AC.

Right, if the interface count is fine that means it's not a missing call
to the subtraction. So hopefully it will be fixed with v6 :)

-Toke

Patch
diff mbox series

diff --git a/net/mac80211/status.c b/net/mac80211/status.c
index ab8ba5835ca0..905b4afd457d 100644
--- a/net/mac80211/status.c
+++ b/net/mac80211/status.c
@@ -676,6 +676,33 @@  static void ieee80211_report_used_skb(struct ieee80211_local *local,
 	if (dropped)
 		acked = false;
 
+	if (info->tx_time_est) {
+		struct ieee80211_sub_if_data *sdata;
+		struct sta_info *sta = NULL;
+		u8 *qc, ac;
+		int tid;
+
+		rcu_read_lock();
+
+		sdata = ieee80211_sdata_from_skb(local, skb);
+		if (sdata)
+			sta = sta_info_get_bss(sdata, skb_mac_header(skb));
+
+		if (ieee80211_is_data_qos(hdr->frame_control)) {
+			qc = ieee80211_get_qos_ctl(hdr);
+			tid = qc[0] & 0xf;
+			ac = ieee80211_ac_from_tid(tid);
+		} else {
+			ac = IEEE80211_AC_BE;
+		}
+
+		ieee80211_sta_update_pending_airtime(local, sta, ac,
+						     info->tx_time_est << 2,
+						     true);
+		rcu_read_unlock();
+
+	}
+
 	if (info->flags & IEEE80211_TX_INTFL_MLME_CONN_TX) {
 		struct ieee80211_sub_if_data *sdata;
 
@@ -930,6 +957,17 @@  static void __ieee80211_tx_status(struct ieee80211_hw *hw,
 			tid = qc[0] & 0xf;
 		}
 
+		if (info->tx_time_est) {
+			/* Do this here to avoid the expensive lookup of the sta
+			 * in ieee80211_report_used_skb().
+			 */
+			ieee80211_sta_update_pending_airtime(local, sta,
+							     ieee80211_ac_from_tid(tid),
+							     info->tx_time_est << 2,
+							     true);
+			info->tx_time_est = 0;
+		}
+
 		if (!acked && ieee80211_is_back_req(fc)) {
 			u16 control;
 
diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
index 12653d873b8c..b8ff56d1d661 100644
--- a/net/mac80211/tx.c
+++ b/net/mac80211/tx.c
@@ -3542,6 +3542,9 @@  struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
 
 	WARN_ON_ONCE(softirq_count() == 0);
 
+	if (!ieee80211_txq_airtime_check(hw, txq))
+		return NULL;
+
 begin:
 	spin_lock_bh(&fq->lock);
 
@@ -3652,6 +3655,24 @@  struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
 	}
 
 	IEEE80211_SKB_CB(skb)->control.vif = vif;
+
+	if (local->airtime_flags & AIRTIME_USE_AQL) {
+		u32 airtime;
+
+		airtime = ieee80211_calc_expected_tx_airtime(hw, vif, txq->sta,
+							     skb->len);
+		if (airtime) {
+			/* We only have 10 bits in tx_time_est, so store airtime
+			 * in increments of 4 us and clamp the maximum to 2**12-1
+			 */
+			airtime = min_t(u32, airtime, 4095) & ~3U;
+			info->tx_time_est = airtime >> 2;
+			ieee80211_sta_update_pending_airtime(local, tx.sta,
+							     txq->ac, airtime,
+							     false);
+		}
+	}
+
 	return skb;
 
 out: