net: stmmac: Add unified Rx descriptor release method

Currently the DMA-descriptor release method is implemented for the Tx
descriptors and specific for the related code semantics. The Rx
DMA-descriptors are being prepared by means of the
stmmac_desc_ops::set_rx_owner() methods, which besides to setting the
DMA-owner flag also initialize the Interrupt-on-Completion flag and the
buffer1-valid flag. This makes the functions body noncoherent and thus
the callee code - harder to read/comprehend. Instead let's introduce the
stmmac_desc_ops::release_rx_desc() methods for all currently supported DW
MAC IP-cores, which would implement the semantics required for the
Rx DMA-descriptors being released for DMA.

Note this change also moves the DMA-Wr memory barrier to the new
release_rx_desc() and makes sure it's executed only ones on the
initial descriptor release. This shall speed-up the procedure a bit.

Besides the commit happens to fix a problem specific for the DW MAC
IP-cores with the fully written back Rx DMA-descriptors (DW GMAC >= v4.x
and DW XGMAC >= v2.x). Since the DW MAC DMA-engine of these IP-cores
overwrites all the descriptors fields it also clears/sets the
Interrupt-on-Completion flag depending on the RDES3.CTX bit state. So the
release procedure must correctly restore the IOC flag in accordance with
the disable_rx_ic argument value (Rx WDT status) including the possible
clearance.

Fixes: 753a71090f33 ("stmmac: add descriptors function for GMAC 4.xx")
Fixes: 874dfb65a484 ("net: stmmac: Add descriptor related callbacks for XGMAC2")
Signed-off-by: Serge Semin <fancer.lancer@gmail.com>

6 files changed, 124 insertions, 8 deletions

diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
index 31e1c2ad9079..86eb6c01a7d0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
@@ -323,7 +323,12 @@ exit:
 static void dwmac4_rd_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
 				   int mode, int end, int bfsize)
 {
-	dwmac4_set_rx_owner(p, disable_rx_ic);
+	u32 flags = (RDES3_OWN | RDES3_BUFFER1_VALID_ADDR);
+
+	if (!disable_rx_ic)
+		flags |= RDES3_INT_ON_COMPLETION_EN;
+
+	p->des3 |= cpu_to_le32(flags);
 }
 
 static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end)
@@ -339,6 +344,30 @@ static unsigned int dwmac4_rd_get_rx_len(int mode)
 	return STMMAC_RX_BUF_ALIGN(FIELD_MAX(DMA_RBSZ_MASK));
 }
 
+static void dwmac4_release_rx_desc(struct dma_desc *p, int disable_rx_ic,
+				   int is_fs, bool rx_own)
+{
+	unsigned int rdes3 = le32_to_cpu(p->des3);
+
+	rdes3 |= RDES3_BUFFER1_VALID_ADDR;
+
+	if (disable_rx_ic)
+		rdes3 &= cpu_to_le32(~RDES3_INT_ON_COMPLETION_EN);
+	else
+		rdes3 |= cpu_to_le32(RDES3_INT_ON_COMPLETION_EN);
+
+	if (rx_own)
+		rdes3 |= cpu_to_le32(RDES3_OWN);
+
+	/* Before releasing the initial descriptor make sure that all
+	 * the previous writes are visible to the controller.
+	 */
+	if (is_fs && rx_own)
+		dma_wmb();
+
+	p->des3 = cpu_to_le32(rdes3);
+}
+
 static unsigned int dwmac4_rd_get_tx_len(int mode)
 {
 	return FIELD_MAX(TDES2_BUFFER1_SIZE_MASK);
@@ -583,6 +612,7 @@ const struct stmmac_desc_ops dwmac4_desc_ops = {
 	.get_rx_timestamp_status = dwmac4_wrback_get_rx_timestamp_status,
 	.get_timestamp = dwmac4_get_timestamp,
 	.set_tx_ic = dwmac4_rd_set_tx_ic,
+	.release_rx_desc = dwmac4_release_rx_desc,
 	.prepare_tx_desc = dwmac4_rd_prepare_tx_desc,
 	.prepare_tso_tx_desc = dwmac4_rd_prepare_tso_tx_desc,
 	.release_tx_desc = dwmac4_release_tx_desc,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
index 14f2da1e5f02..192860b639f6 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
@@ -231,7 +231,12 @@ static int dwxgmac2_get_rx_timestamp_status(void *desc, void *next_desc,
 static void dwxgmac2_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
 				  int mode, int end, int bfsize)
 {
-	dwxgmac2_set_rx_owner(p, disable_rx_ic);
+	u32 flags = XGMAC_RDES3_OWN;
+
+	if (!disable_rx_ic)
+		flags |= XGMAC_RDES3_IOC;
+
+	p->des3 |= cpu_to_le32(flags);
 }
 
 static void dwxgmac2_init_tx_desc(struct dma_desc *p, int mode, int end)
@@ -247,6 +252,28 @@ static unsigned int dwxgmac2_get_rx_len(int mode)
 	return STMMAC_RX_BUF_ALIGN(FIELD_MAX(XGMAC_RBSZ));
 }
 
+static void dwxgmac2_release_rx_desc(struct dma_desc *p, int disable_rx_ic,
+				     int is_fs, bool rx_own)
+{
+	unsigned int rdes3 = le32_to_cpu(p->des3);
+
+	if (disable_rx_ic)
+		rdes3 &= cpu_to_le32(~XGMAC_RDES3_IOC);
+	else
+		rdes3 |= cpu_to_le32(XGMAC_RDES3_IOC);
+
+	if (rx_own)
+		rdes3 |= cpu_to_le32(XGMAC_RDES3_OWN);
+
+	/* Before releasing the initial descriptor make sure that all
+	 * the previous writes are visible to the controller.
+	 */
+	if (is_fs && rx_own)
+		dma_wmb();
+
+	p->des3 = cpu_to_le32(rdes3);
+}
+
 static unsigned int dwxgmac2_get_tx_len(int mode)
 {
 	return FIELD_MAX(XGMAC_TDES2_B1L);
@@ -464,6 +491,7 @@ const struct stmmac_desc_ops dwxgmac210_desc_ops = {
 	.get_rx_timestamp_status = dwxgmac2_get_rx_timestamp_status,
 	.get_timestamp = dwxgmac2_get_timestamp,
 	.set_tx_ic = dwxgmac2_set_tx_ic,
+	.release_rx_desc = dwxgmac2_release_rx_desc,
 	.prepare_tx_desc = dwxgmac2_prepare_tx_desc,
 	.prepare_tso_tx_desc = dwxgmac2_prepare_tso_tx_desc,
 	.release_tx_desc = dwxgmac2_release_tx_desc,
diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
index 91d63659f108..c11a06af7dbf 100644
--- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
@@ -327,6 +327,28 @@ static unsigned int enh_desc_get_rx_len(int mode)
 		return enh_desc_rx_desc_len_on_ring();
 }
 
+static void enh_desc_release_rx_desc(struct dma_desc *p, int disable_rx_ic,
+				     int is_fs, bool rx_own)
+{
+	unsigned int rdes1 = le32_to_cpu(p->des1);
+
+	if (disable_rx_ic)
+		rdes1 |= cpu_to_le32(ERDES1_DISABLE_IC);
+	else
+		rdes1 &= cpu_to_le32(~ERDES1_DISABLE_IC);
+
+	p->des1 = cpu_to_le32(rdes1);
+
+	/* Before releasing the initial descriptor make sure that all
+	 * the previous writes are visible to the controller.
+	 */
+	if (is_fs && rx_own)
+		dma_wmb();
+
+	if (rx_own)
+		p->des0 = cpu_to_le32(RDES0_OWN);
+}
+
 static void enh_desc_prepare_rx_desc(struct dma_desc *p, int mode,
 				     dma_addr_t np, bool hwts_rx, int bfsize)
 {
@@ -493,6 +515,7 @@ const struct stmmac_desc_ops enh_desc_ops = {
 	.init_rx_desc = enh_desc_init_rx_desc,
 	.init_tx_desc = enh_desc_init_tx_desc,
 	.get_tx_owner = enh_desc_get_tx_owner,
+	.release_rx_desc = enh_desc_release_rx_desc,
 	.prepare_rx_desc = enh_desc_prepare_rx_desc,
 	.release_tx_desc = enh_desc_release_tx_desc,
 	.prepare_tx_desc = enh_desc_prepare_tx_desc,
@@ -518,6 +541,7 @@ const struct stmmac_desc_ops enh_desc_noext_ops = {
 	.init_rx_desc = enh_desc_init_rx_desc,
 	.init_tx_desc = enh_desc_init_tx_desc,
 	.get_tx_owner = enh_desc_get_tx_owner,
+	.release_rx_desc = enh_desc_release_rx_desc,
 	.prepare_rx_desc = enh_desc_prepare_rx_desc,
 	.release_tx_desc = enh_desc_release_tx_desc,
 	.prepare_tx_desc = enh_desc_prepare_tx_desc,
@@ -543,6 +567,7 @@ const struct stmmac_desc_ops enh_desc_ext_ops = {
 	.init_rx_desc = enh_desc_init_rx_desc,
 	.init_tx_desc = enh_desc_init_tx_desc,
 	.get_tx_owner = enh_desc_get_tx_owner,
+	.release_rx_desc = enh_desc_release_rx_desc,
 	.prepare_rx_desc = enh_desc_prepare_rx_desc,
 	.release_tx_desc = enh_desc_release_tx_desc,
 	.prepare_tx_desc = enh_desc_prepare_tx_desc,
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h
index 2eb1e03375f1..f523ef567eab 100644
--- a/drivers/net/ethernet/stmicro/stmmac/hwif.h
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h
@@ -55,6 +55,9 @@ struct stmmac_desc_ops {
 	/* Set/get the owner of the descriptor */
 	void (*set_tx_owner)(struct dma_desc *p);
 	int (*get_tx_owner)(struct dma_desc *p);
+	/* Clean the rx descriptor as soon as the rx flow is handled */
+	void (*release_rx_desc)(struct dma_desc *p, int disable_rx_ic,
+				int is_fs, bool rx_own);
 	/* Clean the tx descriptor as soon as the tx irq is received */
 	void (*release_tx_desc)(struct dma_desc *p, int mode,
 				dma_addr_t np, bool hwts_tx);
@@ -122,6 +125,8 @@ struct stmmac_desc_ops {
 	stmmac_do_void_callback(__priv, desc, set_tx_owner, __args)
 #define stmmac_get_tx_owner(__priv, __args...) \
 	stmmac_do_callback(__priv, desc, get_tx_owner, __args)
+#define stmmac_release_rx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, release_rx_desc, __args)
 #define stmmac_release_tx_desc(__priv, __args...) \
 	stmmac_do_void_callback(__priv, desc, release_tx_desc, __args)
 #define stmmac_set_tx_ic(__priv, __args...) \
diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
index d24324bd2058..e90184f3f136 100644
--- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
@@ -201,6 +201,28 @@ static unsigned int ndesc_get_rx_len(int mode)
 		return ndesc_rx_desc_len_on_ring();
 }
 
+static void ndesc_release_rx_desc(struct dma_desc *p, int disable_rx_ic,
+				  int is_fs, bool rx_own)
+{
+	unsigned int rdes1 = le32_to_cpu(p->des1);
+
+	if (disable_rx_ic)
+		rdes1 |= cpu_to_le32(RDES1_DISABLE_IC);
+	else
+		rdes1 &= cpu_to_le32(~RDES1_DISABLE_IC);
+
+	p->des1 = cpu_to_le32(rdes1);
+
+	/* Before releasing the initial descriptor make sure that all
+	 * the previous writes are visible to the controller.
+	 */
+	if (is_fs && rx_own)
+		dma_wmb();
+
+	if (rx_own)
+		p->des0 = cpu_to_le32(RDES0_OWN);
+}
+
 static void ndesc_prepare_rx_desc(struct dma_desc *p, int mode,
 				  dma_addr_t np, bool hwts_rx, int bfsize)
 {
@@ -348,6 +370,7 @@ const struct stmmac_desc_ops ndesc_ops = {
 	.init_rx_desc = ndesc_init_rx_desc,
 	.init_tx_desc = ndesc_init_tx_desc,
 	.get_tx_owner = ndesc_get_tx_owner,
+	.release_rx_desc = ndesc_release_rx_desc,
 	.prepare_rx_desc = ndesc_prepare_rx_desc,
 	.release_tx_desc = ndesc_release_tx_desc,
 	.prepare_tx_desc = ndesc_prepare_tx_desc,
@@ -373,6 +396,7 @@ const struct stmmac_desc_ops ndesc_rxcoe2_ops = {
 	.init_rx_desc = ndesc_init_rx_desc,
 	.init_tx_desc = ndesc_init_tx_desc,
 	.get_tx_owner = ndesc_get_tx_owner,
+	.release_rx_desc = ndesc_release_rx_desc,
 	.prepare_rx_desc = ndesc_prepare_rx_desc,
 	.release_tx_desc = ndesc_release_tx_desc,
 	.prepare_tx_desc = ndesc_prepare_tx_desc,
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index 5fbafd8e53ce..0ec2cb760301 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -5139,11 +5139,13 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
 	int dirty = stmmac_rx_dirty(priv, queue);
 	unsigned int entry = rx_q->dirty_rx;
 	gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
-	unsigned int size;
+	unsigned int size, first_rx;
 
 	if (priv->dma_cap.host_dma_width <= 32)
 		gfp |= GFP_DMA32;
 
+	first_rx = rx_q->dirty_rx;
+
 	while (dirty-- > 0) {
 		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
 		struct dma_desc *p;
@@ -5197,8 +5199,8 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
 		if (!priv->use_riwt)
 			use_rx_wd = false;
 
-		dma_wmb();
-		stmmac_set_rx_owner(priv, p, use_rx_wd);
+		stmmac_release_rx_desc(priv, p, use_rx_wd,
+				       entry == first_rx, true);
 
 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);
 	}
@@ -5576,13 +5578,15 @@ static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,
 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 {
 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
-	unsigned int entry = rx_q->dirty_rx;
+	unsigned int entry = rx_q->dirty_rx, first_rx;
 	struct dma_desc *rx_desc = NULL;
 	dma_addr_t rx_next;
 	bool ret = true;
 
 	budget = min(budget, stmmac_rx_dirty(priv, queue));
 
+	first_rx = rx_q->dirty_rx;
+
 	while (budget-- > 0 && entry != rx_q->cur_rx) {
 		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
 		dma_addr_t dma_addr;
@@ -5622,8 +5626,8 @@ static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 		if (!priv->use_riwt)
 			use_rx_wd = false;
 
-		dma_wmb();
-		stmmac_set_rx_owner(priv, rx_desc, use_rx_wd);
+		stmmac_release_rx_desc(priv, rx_desc, use_rx_wd,
+				       entry == first_rx, true);
 
 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);
 	}