net: stmmac: dwmac1000: Fix invalid Tx descriptor chain after TS

It's possible to have the DMA-descriptors linked with each other on the
DW GMAC IP-cores. It's called the chain mode, when a  DMA-descriptor refers
to the next one by a pointer. But in case of the IEEE 1588-2002 Timestamp
support the chain will be broken by the DMA-engine writing the outbound
timestamp to the last descriptor of the transferred frame. So the
descriptors chain must be restored afterwards in that case.

The restoration has been implemented by means of the
stmmac_mode_ops::clean_desc3() function called in the framework of the
stmmac_tx_clean() method responsible for the Tx DMA-descriptors
preparation for the next transfers. But the procedure has been broken in
e3ad57c96715 ("stmmac: review RX/TX ring management"). The clean_desc3()
method relies on the stmmac_tx_queue::dirty_rx field state to
re-initialize the respective descriptor and find the next one. But the
dirty Tx-pointer now is advanced only after all the Tx-descriptors are
cleaned up and moved right to the current Tx-pointer. So the intermediate
descriptors will be left with the chain broken, meanwhile the first dirty
one will point to the first non-dirty. This is definitely wrong.

Let's fix that problem. But instead of either getting back the Tx
DMA-descriptors cleanup loop semantics or fixing the clean_desc3() method
semantics it's better to optimize the chain mode initialization and
restoration procedures. It can be done by moving the chain-mode
initialization to the stmmac_desc_ops::release_tx_desc() method. So after
doing that the Tx DMA-descriptors chaining will be localized in a single
method and the problem described above will be fixed since the next
descriptor pointer will be passed noew to the release_tx_desc() function.
Note the chain-mode flag has been initialized in that function even before
this change.

Fixes: e3ad57c96715 ("stmmac: review RX/TX ring management")
Signed-off-by: Serge Semin <fancer.lancer@gmail.com>

9 files changed, 51 insertions, 73 deletions

diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
index d7ff4114a7ed..d6118745d681 100644
--- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
@@ -148,27 +148,9 @@ static void refill_desc3(struct stmmac_rx_queue *rx_q, struct dma_desc *p)
 				      sizeof(struct dma_desc)));
 }
 
-static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p)
-{
-	struct stmmac_priv *priv = tx_q->priv_data;
-	unsigned int entry = tx_q->dirty_tx;
-
-	if (tx_q->tx_skbuff_dma[entry].last_segment && !priv->extend_desc &&
-	    priv->hwts_tx_en)
-		/* NOTE: Device will overwrite des3 with timestamp value if
-		 * 1588-2002 time stamping is enabled, hence reinitialize it
-		 * to keep explicit chaining in the descriptor.
-		 */
-		p->des3 = cpu_to_le32((unsigned int)((tx_q->dma_tx_phy +
-				      ((tx_q->dirty_tx + 1) %
-				       priv->dma_conf.dma_tx_size))
-				      * sizeof(struct dma_desc)));
-}
-
 const struct stmmac_mode_ops chain_mode_ops = {
 	.init = init_dma_chain,
 	.is_jumbo_frm = is_jumbo_frm,
 	.jumbo_frm = jumbo_frm,
 	.refill_desc3 = refill_desc3,
-	.clean_desc3 = clean_desc3,
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/descs_com.h b/drivers/net/ethernet/stmicro/stmmac/descs_com.h
index d5c647184046..0019f389181f 100644
--- a/drivers/net/ethernet/stmicro/stmmac/descs_com.h
+++ b/drivers/net/ethernet/stmicro/stmmac/descs_com.h
@@ -101,9 +101,13 @@ static inline void ehn_desc_rx_set_on_chain(struct dma_desc *p)
 	p->des1 |= cpu_to_le32(ERDES1_SECOND_ADDRESS_CHAINED);
 }
 
-static inline void enh_desc_end_tx_desc_on_chain(struct dma_desc *p)
+static inline void enh_desc_end_tx_desc_on_chain(struct dma_desc *p, dma_addr_t np,
+						 bool hwts_tx_en)
 {
 	p->des0 |= cpu_to_le32(ETDES0_SECOND_ADDRESS_CHAINED);
+	/* des3 is overwritten with timestamp in case of IEEE 1588-2002 */
+	if (hwts_tx_en)
+		p->des3 = cpu_to_le32(lower_32_bits(np));
 }
 
 static inline void enh_set_tx_desc_len_on_chain(struct dma_desc *p, int len)
@@ -117,9 +121,13 @@ static inline void ndesc_rx_set_on_chain(struct dma_desc *p, int end)
 	p->des1 |= cpu_to_le32(RDES1_SECOND_ADDRESS_CHAINED);
 }
 
-static inline void ndesc_tx_set_on_chain(struct dma_desc *p)
+static inline void ndesc_end_tx_desc_on_chain(struct dma_desc *p, dma_addr_t np,
+					      bool hwts_tx_en)
 {
 	p->des1 |= cpu_to_le32(TDES1_SECOND_ADDRESS_CHAINED);
+	/* des3 is overwritten with timestamp in case of IEEE 1588-2002 */
+	if (hwts_tx_en)
+		p->des3 = cpu_to_le32(lower_32_bits(np));
 }
 
 static inline void norm_set_tx_desc_len_on_chain(struct dma_desc *p, int len)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
index f51da6c80717..7918759369bb 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
@@ -417,7 +417,8 @@ static void dwmac4_rd_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
 	p->des3 = cpu_to_le32(tdes3);
 }
 
-static void dwmac4_release_tx_desc(struct dma_desc *p, int mode)
+static void dwmac4_release_tx_desc(struct dma_desc *p, int mode,
+				   dma_addr_t np, bool hwts_tx)
 {
 	p->des0 = 0;
 	p->des1 = 0;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
index deb0a787cb04..98135173a321 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
@@ -323,7 +323,8 @@ static void dwxgmac2_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
 	p->des3 = cpu_to_le32(tdes3);
 }
 
-static void dwxgmac2_release_tx_desc(struct dma_desc *p, int mode)
+static void dwxgmac2_release_tx_desc(struct dma_desc *p, int mode,
+				     dma_addr_t np, bool hwts_tx)
 {
 	p->des0 = 0;
 	p->des1 = 0;
diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
index 5b4bae8c5e89..ed91ffb02272 100644
--- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
@@ -309,7 +309,7 @@ static void enh_desc_init_tx_desc(struct dma_desc *p, int mode, int end)
 {
 	memset(p, 0, offsetof(struct dma_desc, des2));
 	if (mode == STMMAC_CHAIN_MODE)
-		enh_desc_end_tx_desc_on_chain(p);
+		enh_desc_end_tx_desc_on_chain(p, 0, false);
 	else
 		enh_desc_end_tx_desc_on_ring(p, end);
 }
@@ -334,13 +334,14 @@ static int enh_desc_get_tx_ls(struct dma_desc *p)
 	return (le32_to_cpu(p->des0) & ETDES0_LAST_SEGMENT) >> 29;
 }
 
-static void enh_desc_release_tx_desc(struct dma_desc *p, int mode)
+static void enh_desc_release_tx_desc(struct dma_desc *p, int mode,
+				     dma_addr_t np, bool hwts_tx)
 {
 	int ter = (le32_to_cpu(p->des0) & ETDES0_END_RING) >> 21;
 
 	memset(p, 0, offsetof(struct dma_desc, des2));
 	if (mode == STMMAC_CHAIN_MODE)
-		enh_desc_end_tx_desc_on_chain(p);
+		enh_desc_end_tx_desc_on_chain(p, np, hwts_tx);
 	else
 		enh_desc_end_tx_desc_on_ring(p, ter);
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h
index b1654e86179a..ed961ea1f939 100644
--- a/drivers/net/ethernet/stmicro/stmmac/hwif.h
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h
@@ -53,7 +53,8 @@ struct stmmac_desc_ops {
 	void (*set_tx_owner)(struct dma_desc *p);
 	int (*get_tx_owner)(struct dma_desc *p);
 	/* Clean the tx descriptor as soon as the tx irq is received */
-	void (*release_tx_desc)(struct dma_desc *p, int mode);
+	void (*release_tx_desc)(struct dma_desc *p, int mode,
+				dma_addr_t np, bool hwts_tx);
 	/* Clear interrupt on tx frame completion. When this bit is
 	 * set an interrupt happens as soon as the frame is transmitted */
 	void (*set_tx_ic)(struct dma_desc *p);
@@ -553,7 +554,6 @@ struct stmmac_mode_ops {
 	int (*set_16kib_bfsize)(int mtu);
 	void (*init_desc3)(struct dma_desc *p);
 	void (*refill_desc3)(struct stmmac_rx_queue *rx_q, struct dma_desc *p);
-	void (*clean_desc3)(struct stmmac_tx_queue *tx_q, struct dma_desc *p);
 };
 
 #define stmmac_mode_init(__priv, __args...) \
@@ -568,8 +568,6 @@ struct stmmac_mode_ops {
 	stmmac_do_void_callback(__priv, mode, init_desc3, __args)
 #define stmmac_refill_desc3(__priv, __args...) \
 	stmmac_do_void_callback(__priv, mode, refill_desc3, __args)
-#define stmmac_clean_desc3(__priv, __args...) \
-	stmmac_do_void_callback(__priv, mode, clean_desc3, __args)
 
 struct tc_cls_u32_offload;
 struct tc_cbs_qopt_offload;
diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
index a26b58f9b573..6f90b0b6f54c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
@@ -183,7 +183,7 @@ static void ndesc_init_tx_desc(struct dma_desc *p, int mode, int end)
 {
 	memset(p, 0, offsetof(struct dma_desc, des2));
 	if (mode == STMMAC_CHAIN_MODE)
-		ndesc_tx_set_on_chain(p);
+		ndesc_end_tx_desc_on_chain(p, 0, false);
 	else
 		ndesc_end_tx_desc_on_ring(p, end);
 }
@@ -208,13 +208,14 @@ static int ndesc_get_tx_ls(struct dma_desc *p)
 	return (le32_to_cpu(p->des1) & TDES1_LAST_SEGMENT) >> 30;
 }
 
-static void ndesc_release_tx_desc(struct dma_desc *p, int mode)
+static void ndesc_release_tx_desc(struct dma_desc *p, int mode,
+				  dma_addr_t np, bool hwts_tx)
 {
 	int ter = (le32_to_cpu(p->des1) & TDES1_END_RING) >> 25;
 
 	memset(p, 0, offsetof(struct dma_desc, des2));
 	if (mode == STMMAC_CHAIN_MODE)
-		ndesc_tx_set_on_chain(p);
+		ndesc_end_tx_desc_on_chain(p, np, hwts_tx);
 	else
 		ndesc_end_tx_desc_on_ring(p, ter);
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
index 382d94a3b972..3110b10675cd 100644
--- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
@@ -111,18 +111,6 @@ static void init_desc3(struct dma_desc *p)
 	p->des3 = cpu_to_le32(le32_to_cpu(p->des2) + BUF_SIZE_8KiB);
 }
 
-static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p)
-{
-	struct stmmac_priv *priv = tx_q->priv_data;
-	unsigned int entry = tx_q->dirty_tx;
-
-	/* des3 is only used for jumbo frames tx or time stamping */
-	if (unlikely(tx_q->tx_skbuff_dma[entry].is_jumbo ||
-		     (tx_q->tx_skbuff_dma[entry].last_segment &&
-		      !priv->extend_desc && priv->hwts_tx_en)))
-		p->des3 = 0;
-}
-
 static int set_16kib_bfsize(int mtu)
 {
 	int ret = 0;
@@ -136,6 +124,5 @@ const struct stmmac_mode_ops ring_mode_ops = {
 	.jumbo_frm = jumbo_frm,
 	.refill_desc3 = refill_desc3,
 	.init_desc3 = init_desc3,
-	.clean_desc3 = clean_desc3,
 	.set_16kib_bfsize = set_16kib_bfsize,
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index b4ee92afe8c8..529a9bf40699 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -2062,38 +2062,32 @@ static int __init_dma_tx_desc_rings(struct stmmac_priv *priv,
 		  "(%s) dma_tx_phy=0x%08x\n", __func__,
 		  (u32)tx_q->dma_tx_phy);
 
-	/* Setup the chained descriptor addresses */
-	if (priv->mode == STMMAC_CHAIN_MODE) {
-		if (priv->extend_desc)
-			stmmac_mode_init(priv, tx_q->dma_etx,
-					 tx_q->dma_tx_phy,
-					 dma_conf->dma_tx_size, 1);
-		else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
-			stmmac_mode_init(priv, tx_q->dma_tx,
-					 tx_q->dma_tx_phy,
-					 dma_conf->dma_tx_size, 0);
-	}
-
 	/* This must be performed on each SKB/XSK mode switch */
 	tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
 
 	for (i = 0; i < dma_conf->dma_tx_size; i++) {
 		struct dma_desc *p;
+		dma_addr_t np;
 
-		if (priv->extend_desc)
-			p = &((tx_q->dma_etx + i)->basic);
-		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-			p = &((tx_q->dma_entx + i)->basic);
-		else
-			p = tx_q->dma_tx + i;
-
-		stmmac_clear_desc(priv, p);
+		np = STMMAC_GET_ENTRY(i, dma_conf->dma_tx_size);
+		if (priv->extend_desc) {
+			p = &tx_q->dma_etx[i].basic;
+			np = tx_q->dma_tx_phy + np * sizeof(struct dma_extended_desc);
+		} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
+			p = &tx_q->dma_entx[i].basic;
+			np = tx_q->dma_tx_phy + np * sizeof(struct dma_edesc);
+		} else {
+			p = &tx_q->dma_tx[i];
+			np = tx_q->dma_tx_phy + np * sizeof(struct dma_desc);
+		}
 
 		tx_q->tx_skbuff_dma[i].buf = 0;
 		tx_q->tx_skbuff_dma[i].map_as_page = false;
 		tx_q->tx_skbuff_dma[i].len = 0;
 		tx_q->tx_skbuff_dma[i].last_segment = false;
 		tx_q->tx_skbuff[i] = NULL;
+
+		stmmac_release_tx_desc(priv, p, priv->mode, np, true);
 	}
 
 	return 0;
@@ -2938,6 +2932,7 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
 		struct xdp_frame *xdpf;
 		struct sk_buff *skb;
 		struct dma_desc *p;
+		dma_addr_t np;
 		int status;
 
 		if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX ||
@@ -2952,12 +2947,17 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
 			skb = NULL;
 		}
 
-		if (priv->extend_desc)
-			p = (struct dma_desc *)(tx_q->dma_etx + entry);
-		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		np = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+		if (priv->extend_desc) {
+			p = &tx_q->dma_etx[entry].basic;
+			np = tx_q->dma_tx_phy + np * sizeof(struct dma_extended_desc);
+		} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
 			p = &tx_q->dma_entx[entry].basic;
-		else
-			p = tx_q->dma_tx + entry;
+			np = tx_q->dma_tx_phy + np * sizeof(struct dma_edesc);
+		} else {
+			p = &tx_q->dma_tx[entry];
+			np = tx_q->dma_tx_phy + np * sizeof(struct dma_desc);
+		}
 
 		status = stmmac_tx_status(priv,	&priv->xstats, p);
 		/* Check if the descriptor is owned by the DMA */
@@ -3011,8 +3011,6 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
 			tx_q->tx_skbuff_dma[entry].map_as_page = false;
 		}
 
-		stmmac_clean_desc3(priv, tx_q, p);
-
 		tx_q->tx_skbuff_dma[entry].last_segment = false;
 		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
 
@@ -3040,7 +3038,8 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
 			}
 		}
 
-		stmmac_release_tx_desc(priv, p, priv->mode);
+		stmmac_release_tx_desc(priv, p, priv->mode, np,
+				       priv->hwts_tx_en);
 
 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
 	}