|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: Issue: Networking performance in Xen VM on Arm64
On Fri, Oct 28, 2022 at 05:54:05PM +0800, Leo Yan wrote:
[...]
> If map to the code, I think the function xennet_start_xmit() in Xen
> frontend driver is critical for the sending interval in domU. I can
> see several things cost time when sending a packet:
>
> - Xen frontend driver needs to setup grant table for every skb, it
> invokes the function xennet_tx_setup_grant() and
> gnttab_grant_foreign_access_ref() to prepare grant table;
>
> - Xen frontend driver sends notification by calling
> notify_remote_via_irq(). It will trap to Xen hypervisor to send the
> interrupt, this takes several macro seonds for this step.
>
> - Xen frontend driver calls xennet_tx_buf_gc(), the interval for this
> function is vary, it will take ~30us in the case for reclaiming grant
> table.
>
> Any thoughts for this?
Supplement info with Ftrace function graph. You could see below
log which shows the time is spent in xennet_start_xmit():
1) | xennet_start_xmit() {
1) | /* xennet_start_xmit: TSC: 50702652609 */
1) 0.240 us | _raw_spin_lock_irqsave();
1) | netif_skb_features() {
1) 0.280 us | skb_network_protocol();
1) 0.920 us | }
1) | gnttab_foreach_grant_in_range() {
1) | xennet_tx_setup_grant() {
1) 0.280 us | gnttab_claim_grant_reference();
1) | gnttab_grant_foreign_access_ref() {
1) 0.280 us | gnttab_update_entry_v1();
1) 0.800 us | }
1) | /* id=103 ref=871 offset=2050 len=1514 TSC:
50702652709 */
1) 2.200 us | }
1) 2.800 us | }
1) | /* xennet_notify_tx_irq: TSC: 50702652741 */
1) | notify_remote_via_irq() {
1) | irq_get_irq_data() {
1) 0.240 us | irq_to_desc();
1) 0.760 us | }
1) 3.880 us | }
1) 0.280 us | xennet_tx_buf_gc();
1) 0.240 us | _raw_spin_unlock_irqrestore();
1) + 11.120 us | }
1) | xennet_start_xmit() {
1) | /* xennet_start_xmit: TSC: 50702652974 */
1) 0.280 us | _raw_spin_lock_irqsave();
1) | netif_skb_features() {
1) 0.240 us | skb_network_protocol();
1) 0.760 us | }
1) | gnttab_foreach_grant_in_range() {
1) | xennet_tx_setup_grant() {
1) 0.280 us | gnttab_claim_grant_reference();
1) | gnttab_grant_foreign_access_ref() {
1) 0.360 us | gnttab_update_entry_v1();
1) 0.840 us | }
1) | /* id=101 ref=869 offset=1026 len=574 TSC:
50702653093 */
1) 2.800 us | }
1) 3.320 us | }
1) | /* xennet_notify_tx_irq: TSC: 50702653124 */
1) | notify_remote_via_irq() {
1) | irq_get_irq_data() {
1) 0.240 us | irq_to_desc();
1) 0.760 us | }
1) 3.840 us | }
1) 0.280 us | xennet_tx_buf_gc();
1) 0.280 us | _raw_spin_unlock_irqrestore();
1) + 11.800 us | }
1) | /* finish_transmit_data */
1) | /* transmit_data */
1) | /* finish_transmit_data */
1) | /* transmit_data */
1) | /* finish_transmit_data */
1) | /* transmit_data */
1) | /* finish_transmit_data */
1) | /* Before_throttle */
1) | xennet_start_xmit() {
1) | /* xennet_start_xmit: TSC: 50702654697 */
1) 0.280 us | _raw_spin_lock_irqsave();
1) | netif_skb_features() {
1) 0.240 us | skb_network_protocol();
1) 0.760 us | }
1) | gnttab_foreach_grant_in_range() {
1) | xennet_tx_setup_grant() {
1) 0.280 us | gnttab_claim_grant_reference();
1) | gnttab_grant_foreign_access_ref() {
1) 0.320 us | gnttab_update_entry_v1();
1) 0.920 us | }
1) | /* id=3 ref=771 offset=2050 len=1514 TSC:
50702654801 */
1) 2.400 us | }
1) 2.960 us | }
1) | /* xennet_notify_tx_irq: TSC: 50702654832 */
1) | notify_remote_via_irq() {
1) | irq_get_irq_data() {
1) 0.280 us | irq_to_desc();
1) 0.760 us | }
1) 4.160 us | }
1) | xennet_tx_buf_gc() {
1) | gnttab_end_foreign_access_ref() {
1) 0.480 us | gnttab_end_foreign_access_ref_v1();
1) 0.960 us | }
1) 0.280 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.240 us | __raise_softirq_irqoff();
1) 1.240 us | }
1) | gnttab_end_foreign_access_ref() {
1) 0.480 us | gnttab_end_foreign_access_ref_v1();
1) 0.960 us | }
1) 0.240 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.240 us | __raise_softirq_irqoff();
1) 0.760 us | }
1) | gnttab_end_foreign_access_ref() {
1) 0.400 us | gnttab_end_foreign_access_ref_v1();
1) 0.840 us | }
1) 0.280 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.280 us | __raise_softirq_irqoff();
1) 0.880 us | }
1) | gnttab_end_foreign_access_ref() {
1) 0.360 us | gnttab_end_foreign_access_ref_v1();
1) 0.840 us | }
1) 0.240 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.240 us | __raise_softirq_irqoff();
1) 0.760 us | }
1) | gnttab_end_foreign_access_ref() {
1) 0.240 us | gnttab_end_foreign_access_ref_v1();
1) 0.760 us | }
1) 0.240 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.240 us | __raise_softirq_irqoff();
1) 0.760 us | }
1) | gnttab_end_foreign_access_ref() {
1) 0.360 us | gnttab_end_foreign_access_ref_v1();
1) 0.960 us | }
1) 0.240 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.240 us | __raise_softirq_irqoff();
1) 0.760 us | }
1) + 17.720 us | }
1) 0.360 us | _raw_spin_unlock_irqrestore();
1) + 29.320 us | }
1) | xennet_start_xmit() {
1) | /* xennet_start_xmit: TSC: 50702655824 */
1) 0.360 us | _raw_spin_lock_irqsave();
1) | netif_skb_features() {
1) 0.280 us | skb_network_protocol();
1) 0.760 us | }
1) | gnttab_foreach_grant_in_range() {
1) | xennet_tx_setup_grant() {
1) 0.280 us | gnttab_claim_grant_reference();
1) | gnttab_grant_foreign_access_ref() {
1) 0.280 us | gnttab_update_entry_v1();
1) 0.880 us | }
1) | /* id=28 ref=796 offset=2 len=1514 TSC: 50702655925
*/
1) 2.280 us | }
1) 2.920 us | }
1) | /* xennet_notify_tx_irq: TSC: 50702655958 */
1) | notify_remote_via_irq() {
1) | irq_get_irq_data() {
1) 0.280 us | irq_to_desc();
1) 0.760 us | }
1) 4.160 us | }
1) | xennet_tx_buf_gc() {
1) | gnttab_end_foreign_access_ref() {
1) 0.360 us | gnttab_end_foreign_access_ref_v1();
1) 0.880 us | }
1) 0.280 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.280 us | __raise_softirq_irqoff();
1) 1.280 us | }
1) | gnttab_end_foreign_access_ref() {
1) 0.360 us | gnttab_end_foreign_access_ref_v1();
1) 0.880 us | }
1) 0.760 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.280 us | __raise_softirq_irqoff();
1) 0.760 us | }
1) | gnttab_end_foreign_access_ref() {
1) 0.400 us | gnttab_end_foreign_access_ref_v1();
1) 0.880 us | }
1) 0.240 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.400 us | __raise_softirq_irqoff();
1) 0.880 us | }
1) | gnttab_end_foreign_access_ref() {
1) 0.400 us | gnttab_end_foreign_access_ref_v1();
1) 0.880 us | }
1) 0.280 us | gnttab_release_grant_reference();
1) | __dev_kfree_skb_irq() {
1) 0.240 us | __raise_softirq_irqoff();
1) 0.760 us | }
1) + 12.440 us | }
1) 0.800 us | _raw_spin_unlock_irqrestore();
1) + 24.080 us | }
1) | xennet_start_xmit() {
1) | /* xennet_start_xmit: TSC: 50702656728 */
1) 0.240 us | _raw_spin_lock_irqsave();
1) | netif_skb_features() {
1) 0.240 us | skb_network_protocol();
1) 0.760 us | }
1) | gnttab_foreach_grant_in_range() {
1) | xennet_tx_setup_grant() {
1) 0.280 us | gnttab_claim_grant_reference();
1) | gnttab_grant_foreign_access_ref() {
1) 0.240 us | gnttab_update_entry_v1();
1) 0.720 us | }
1) | /* id=103 ref=871 offset=2050 len=1422 TSC:
50702656823 */
1) 2.160 us | }
1) 2.800 us | }
1) | /* xennet_notify_tx_irq: TSC: 50702656850 */
1) | notify_remote_via_irq() {
1) | irq_get_irq_data() {
1) 0.240 us | irq_to_desc();
1) 0.720 us | }
1) 3.920 us | }
1) 0.320 us | xennet_tx_buf_gc();
1) 0.280 us | _raw_spin_unlock_irqrestore();
1) + 11.000 us | }
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |