diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index d046a922a1a4e..5515b3b2ab8a6 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -1296,8 +1296,6 @@ struct macb_queue { struct macb_tx_skb *tx_skb; dma_addr_t tx_ring_dma; struct work_struct tx_error_task; - bool txubr_pending; - bool tx_pending; struct napi_struct napi_tx; dma_addr_t rx_ring_dma; diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 4a5c3bd74a0a0..613f88785ec3a 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -829,7 +829,7 @@ static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, /* Use juggling algorithm to left rotate tx ring and tx skb array */ static void gem_shuffle_tx_one_ring(struct macb_queue *queue) { - unsigned int head, tail, count, ring_size, desc_size; + unsigned int head, tail, count, ring_size, desc_size, start, done; struct macb_tx_skb tx_skb, *skb_curr, *skb_next; struct macb_dma_desc *desc_curr, *desc_next; unsigned int i, cycles, shift, curr, next; @@ -848,16 +848,27 @@ static void gem_shuffle_tx_one_ring(struct macb_queue *queue) ring_size = bp->tx_ring_size; count = CIRC_CNT(head, tail, ring_size); - if (!(tail % ring_size)) - goto unlock; + /* Frames completed while link-down had TCOMP masked are still + * unreaped and their used bit would stop the controller at the ring + * base, so rotate the first frame still to be sent there instead. + */ + rmb(); + for (start = tail; start != head && + (macb_tx_desc(queue, start)->ctrl & MACB_BIT(TX_USED)); start++) + while (start + 1 != head && !macb_tx_skb(queue, start)->skb) + start++; + done = start - tail; + + if (!(start % ring_size)) + goto reset_hw_ptr; if (!count) { queue->tx_head = 0; queue->tx_tail = 0; - goto unlock; + goto reset_hw_ptr; } - shift = tail % ring_size; + shift = start % ring_size; cycles = gcd(ring_size, shift); for (i = 0; i < cycles; i++) { @@ -897,17 +908,23 @@ static void gem_shuffle_tx_one_ring(struct macb_queue *queue) sizeof(struct macb_tx_skb)); } - queue->tx_head = count; - queue->tx_tail = 0; + queue->tx_tail = done ? ring_size - done : 0; + queue->tx_head = queue->tx_tail + count; /* Make descriptor updates visible to hardware */ wmb(); -unlock: +reset_hw_ptr: + /* The first frame to be sent is now at the ring base, so TBQP must + * be reprogrammed to match; otherwise it keeps pointing at a stale + * descriptor. Safe to write directly here as TX is still disabled + * (called from macb_mac_link_up() before TE is set). + */ + queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); } -/* Rotate the queue so that the tail is at index 0 */ +/* Rotate the queue so that the first frame to be sent is at index 0 */ static void gem_shuffle_tx_rings(struct macb *bp) { struct macb_queue *queue; @@ -980,6 +997,15 @@ static void macb_mac_link_up(struct phylink_config *config, macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { + /* Reap the frames completed while link-down had TCOMP masked */ + local_bh_disable(); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + if (READ_ONCE(queue->tx_tail) != READ_ONCE(queue->tx_head)) + napi_schedule(&queue->napi_tx); + local_bh_enable(); + } + netif_tx_wake_all_queues(ndev); } @@ -1313,7 +1339,8 @@ static void macb_tx_error_task(struct work_struct *work) * network engine about the macb/gem being halted. */ napi_disable(&queue->napi_tx); - spin_lock_irqsave(&bp->lock, flags); + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + spin_lock(&bp->lock); /* Make sure nobody is trying to queue up new packets */ netif_tx_stop_all_queues(bp->dev); @@ -1357,10 +1384,8 @@ static void macb_tx_error_task(struct work_struct *work) skb->data); bp->dev->stats.tx_packets++; queue->stats.tx_packets++; - packets++; bp->dev->stats.tx_bytes += skb->len; queue->stats.tx_bytes += skb->len; - bytes += skb->len; } } else { /* "Buffers exhausted mid-frame" errors may only happen @@ -1374,6 +1399,12 @@ static void macb_tx_error_task(struct work_struct *work) desc->ctrl = ctrl | MACB_BIT(TX_USED); } + /* BQL has to see every frame leaving the ring, sent or not */ + if (skb) { + packets++; + bytes += skb->len; + } + macb_tx_unmap(bp, tx_skb, 0); } @@ -1405,7 +1436,9 @@ static void macb_tx_error_task(struct work_struct *work) netif_tx_start_all_queues(bp->dev); macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); - spin_unlock_irqrestore(&bp->lock, flags); + spin_unlock(&bp->lock); + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); + napi_enable(&queue->napi_tx); } @@ -1917,7 +1950,11 @@ static int macb_rx_poll(struct napi_struct *napi, int budget) * potentially race with the interrupt handler doing the same * actions if an interrupt is raised just after enabling them, * but this should be harmless. + * + * The IER write is posted. Flush it before checking the ring, + * a completion in between would raise no interrupt. */ + queue_readl(queue, IMR); if (macb_rx_pending(queue)) { queue_writel(queue, IDR, bp->rx_intr_mask); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) @@ -1943,9 +1980,9 @@ static void macb_tx_restart(struct macb_queue *queue) if (queue->tx_head == queue->tx_tail) goto out_tx_ptr_unlock; - tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp); - tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp)); - head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head)); + tbqp = queue_readl(queue, TBQP) - lower_32_bits(queue->tx_ring_dma); + tbqp /= macb_dma_desc_get_size(bp); + head_idx = macb_tx_ring_wrap(bp, queue->tx_head); if (tbqp == head_idx) goto out_tx_ptr_unlock; @@ -1990,12 +2027,10 @@ static int macb_tx_poll(struct napi_struct *napi, int budget) work_done = macb_tx_complete(queue, budget); - rmb(); // ensure txubr_pending is up to date - if (queue->txubr_pending) { - queue->txubr_pending = false; - netdev_vdbg(bp->dev, "poll: tx restart\n"); - macb_tx_restart(queue); - } + /* The DMA can park in front of a queued frame without raising + * TXUBR, so check the ring position on every poll. + */ + macb_tx_restart(queue); netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n", (unsigned int)(queue - bp->queues), work_done, budget); @@ -2012,7 +2047,11 @@ static int macb_tx_poll(struct napi_struct *napi, int budget) * potentially race with the interrupt handler doing the same * actions if an interrupt is raised just after enabling them, * but this should be harmless. + * + * The IER write is posted. Flush it before checking the ring, + * a completion in between would raise no interrupt. */ + queue_readl(queue, IMR); if (macb_tx_complete_pending(queue)) { queue_writel(queue, IDR, MACB_BIT(TCOMP)); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) @@ -2170,12 +2209,6 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) queue_writel(queue, ISR, MACB_BIT(TCOMP) | MACB_BIT(TXUBR)); - if (status & MACB_BIT(TXUBR) || queue->tx_pending) { - queue->txubr_pending = true; - queue->tx_pending = 0; - wmb(); // ensure softirq can see update - } - napi_schedule(&queue->napi_tx); } @@ -2629,14 +2662,9 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) spin_lock(&bp->lock); macb_tx_lpi_wake(bp); - /* TSTART write might get dropped, so make the IRQ retrigger a buffer read */ - if (macb_readl(bp, TSR) & MACB_BIT(TGO)) - queue->tx_pending = 1; - macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); - /* Flush PCIe posted-write queue; see comment in macb_tx_restart(). - * Also flushes the preceding macb_tx_lpi_wake() NCR write and the - * TSR-read tx_pending breadcrumb above. + /* Flush the PCIe posted-write queue, see macb_tx_restart(). Also + * flushes the preceding macb_tx_lpi_wake() NCR write. */ if (bp->caps & MACB_CAPS_PCIE_POSTED_WRITES) (void)macb_readl(bp, NCR); @@ -4709,8 +4737,12 @@ static int macb_setup_tc(struct net_device *dev, enum tc_setup_type type, static void macb_tx_timeout(struct net_device *dev, unsigned int q) { struct macb *bp = netdev_priv(dev); + struct macb_queue *queue = &bp->queues[q]; - macb_tx_restart(&bp->queues[q]); + queue_writel(queue, IDR, MACB_BIT(TCOMP)); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(TCOMP)); + napi_schedule(&queue->napi_tx); } static const struct net_device_ops macb_netdev_ops = {