b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1) // SPDX-License-Identifier: GPL-2.0
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 2) // Copyright 2014-2015 Freescale
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 3) // Copyright 2018 NXP
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 4)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 5) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 6) * Driver for NXP Layerscape Queue Direct Memory Access Controller
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 7) *
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 8) * Author:
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 9) * Wen He <wen.he_1@nxp.com>
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 10) * Jiaheng Fan <jiaheng.fan@nxp.com>
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 11) *
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 12) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 13)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 14) #include <linux/module.h>
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 15) #include <linux/delay.h>
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 16) #include <linux/of_irq.h>
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 17) #include <linux/of_platform.h>
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 18) #include <linux/of_dma.h>
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 19) #include <linux/dma-mapping.h>
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 20)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 21) #include "virt-dma.h"
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 22) #include "fsldma.h"
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 23)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 24) /* Register related definition */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 25) #define FSL_QDMA_DMR 0x0
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 26) #define FSL_QDMA_DSR 0x4
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 27) #define FSL_QDMA_DEIER 0xe00
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 28) #define FSL_QDMA_DEDR 0xe04
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 29) #define FSL_QDMA_DECFDW0R 0xe10
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 30) #define FSL_QDMA_DECFDW1R 0xe14
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 31) #define FSL_QDMA_DECFDW2R 0xe18
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 32) #define FSL_QDMA_DECFDW3R 0xe1c
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 33) #define FSL_QDMA_DECFQIDR 0xe30
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 34) #define FSL_QDMA_DECBR 0xe34
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 35)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 36) #define FSL_QDMA_BCQMR(x) (0xc0 + 0x100 * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 37) #define FSL_QDMA_BCQSR(x) (0xc4 + 0x100 * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 38) #define FSL_QDMA_BCQEDPA_SADDR(x) (0xc8 + 0x100 * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 39) #define FSL_QDMA_BCQDPA_SADDR(x) (0xcc + 0x100 * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 40) #define FSL_QDMA_BCQEEPA_SADDR(x) (0xd0 + 0x100 * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 41) #define FSL_QDMA_BCQEPA_SADDR(x) (0xd4 + 0x100 * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 42) #define FSL_QDMA_BCQIER(x) (0xe0 + 0x100 * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 43) #define FSL_QDMA_BCQIDR(x) (0xe4 + 0x100 * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 44)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 45) #define FSL_QDMA_SQDPAR 0x80c
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 46) #define FSL_QDMA_SQEPAR 0x814
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 47) #define FSL_QDMA_BSQMR 0x800
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 48) #define FSL_QDMA_BSQSR 0x804
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 49) #define FSL_QDMA_BSQICR 0x828
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 50) #define FSL_QDMA_CQMR 0xa00
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 51) #define FSL_QDMA_CQDSCR1 0xa08
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 52) #define FSL_QDMA_CQDSCR2 0xa0c
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 53) #define FSL_QDMA_CQIER 0xa10
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 54) #define FSL_QDMA_CQEDR 0xa14
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 55) #define FSL_QDMA_SQCCMR 0xa20
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 56)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 57) /* Registers for bit and genmask */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 58) #define FSL_QDMA_CQIDR_SQT BIT(15)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 59) #define QDMA_CCDF_FORMAT BIT(29)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 60) #define QDMA_CCDF_SER BIT(30)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 61) #define QDMA_SG_FIN BIT(30)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 62) #define QDMA_SG_LEN_MASK GENMASK(29, 0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 63) #define QDMA_CCDF_MASK GENMASK(28, 20)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 64)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 65) #define FSL_QDMA_DEDR_CLEAR GENMASK(31, 0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 66) #define FSL_QDMA_BCQIDR_CLEAR GENMASK(31, 0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 67) #define FSL_QDMA_DEIER_CLEAR GENMASK(31, 0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 68)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 69) #define FSL_QDMA_BCQIER_CQTIE BIT(15)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 70) #define FSL_QDMA_BCQIER_CQPEIE BIT(23)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 71) #define FSL_QDMA_BSQICR_ICEN BIT(31)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 72)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 73) #define FSL_QDMA_BSQICR_ICST(x) ((x) << 16)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 74) #define FSL_QDMA_CQIER_MEIE BIT(31)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 75) #define FSL_QDMA_CQIER_TEIE BIT(0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 76) #define FSL_QDMA_SQCCMR_ENTER_WM BIT(21)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 77)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 78) #define FSL_QDMA_BCQMR_EN BIT(31)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 79) #define FSL_QDMA_BCQMR_EI BIT(30)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 80) #define FSL_QDMA_BCQMR_CD_THLD(x) ((x) << 20)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 81) #define FSL_QDMA_BCQMR_CQ_SIZE(x) ((x) << 16)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 82)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 83) #define FSL_QDMA_BCQSR_QF BIT(16)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 84) #define FSL_QDMA_BCQSR_XOFF BIT(0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 85)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 86) #define FSL_QDMA_BSQMR_EN BIT(31)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 87) #define FSL_QDMA_BSQMR_DI BIT(30)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 88) #define FSL_QDMA_BSQMR_CQ_SIZE(x) ((x) << 16)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 89)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 90) #define FSL_QDMA_BSQSR_QE BIT(17)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 91)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 92) #define FSL_QDMA_DMR_DQD BIT(30)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 93) #define FSL_QDMA_DSR_DB BIT(31)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 94)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 95) /* Size related definition */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 96) #define FSL_QDMA_QUEUE_MAX 8
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 97) #define FSL_QDMA_COMMAND_BUFFER_SIZE 64
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 98) #define FSL_QDMA_DESCRIPTOR_BUFFER_SIZE 32
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 99) #define FSL_QDMA_CIRCULAR_DESC_SIZE_MIN 64
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 100) #define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX 16384
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 101) #define FSL_QDMA_QUEUE_NUM_MAX 8
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 102)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 103) /* Field definition for CMD */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 104) #define FSL_QDMA_CMD_RWTTYPE 0x4
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 105) #define FSL_QDMA_CMD_LWC 0x2
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 106) #define FSL_QDMA_CMD_RWTTYPE_OFFSET 28
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 107) #define FSL_QDMA_CMD_NS_OFFSET 27
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 108) #define FSL_QDMA_CMD_DQOS_OFFSET 24
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 109) #define FSL_QDMA_CMD_WTHROTL_OFFSET 20
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 110) #define FSL_QDMA_CMD_DSEN_OFFSET 19
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 111) #define FSL_QDMA_CMD_LWC_OFFSET 16
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 112)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 113) /* Field definition for Descriptor status */
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 114) #define QDMA_CCDF_STATUS_RTE BIT(5)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 115) #define QDMA_CCDF_STATUS_WTE BIT(4)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 116) #define QDMA_CCDF_STATUS_CDE BIT(2)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 117) #define QDMA_CCDF_STATUS_SDE BIT(1)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 118) #define QDMA_CCDF_STATUS_DDE BIT(0)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 119) #define QDMA_CCDF_STATUS_MASK (QDMA_CCDF_STATUS_RTE | \
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 120) QDMA_CCDF_STATUS_WTE | \
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 121) QDMA_CCDF_STATUS_CDE | \
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 122) QDMA_CCDF_STATUS_SDE | \
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 123) QDMA_CCDF_STATUS_DDE)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 124)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 125) /* Field definition for Descriptor offset */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 126) #define QDMA_CCDF_OFFSET 20
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 127) #define QDMA_SDDF_CMD(x) (((u64)(x)) << 32)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 128)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 129) /* Field definition for safe loop count*/
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 130) #define FSL_QDMA_HALT_COUNT 1500
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 131) #define FSL_QDMA_MAX_SIZE 16385
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 132) #define FSL_QDMA_COMP_TIMEOUT 1000
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 133) #define FSL_COMMAND_QUEUE_OVERFLLOW 10
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 134)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 135) #define FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma_engine, x) \
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 136) (((fsl_qdma_engine)->block_offset) * (x))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 137)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 138) /**
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 139) * struct fsl_qdma_format - This is the struct holding describing compound
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 140) * descriptor format with qDMA.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 141) * @status: Command status and enqueue status notification.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 142) * @cfg: Frame offset and frame format.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 143) * @addr_lo: Holding the compound descriptor of the lower
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 144) * 32-bits address in memory 40-bit address.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 145) * @addr_hi: Same as above member, but point high 8-bits in
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 146) * memory 40-bit address.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 147) * @__reserved1: Reserved field.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 148) * @cfg8b_w1: Compound descriptor command queue origin produced
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 149) * by qDMA and dynamic debug field.
041c4646003e2 (Lee Jones 2020-07-14 12:15:39 +0100 150) * @data: Pointer to the memory 40-bit address, describes DMA
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 151) * source information and DMA destination information.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 152) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 153) struct fsl_qdma_format {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 154) __le32 status;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 155) __le32 cfg;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 156) union {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 157) struct {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 158) __le32 addr_lo;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 159) u8 addr_hi;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 160) u8 __reserved1[2];
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 161) u8 cfg8b_w1;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 162) } __packed;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 163) __le64 data;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 164) };
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 165) } __packed;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 166)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 167) /* qDMA status notification pre information */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 168) struct fsl_pre_status {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 169) u64 addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 170) u8 queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 171) };
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 172)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 173) static DEFINE_PER_CPU(struct fsl_pre_status, pre);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 174)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 175) struct fsl_qdma_chan {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 176) struct virt_dma_chan vchan;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 177) struct virt_dma_desc vdesc;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 178) enum dma_status status;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 179) struct fsl_qdma_engine *qdma;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 180) struct fsl_qdma_queue *queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 181) };
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 182)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 183) struct fsl_qdma_queue {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 184) struct fsl_qdma_format *virt_head;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 185) struct fsl_qdma_format *virt_tail;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 186) struct list_head comp_used;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 187) struct list_head comp_free;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 188) struct dma_pool *comp_pool;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 189) struct dma_pool *desc_pool;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 190) spinlock_t queue_lock;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 191) dma_addr_t bus_addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 192) u32 n_cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 193) u32 id;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 194) struct fsl_qdma_format *cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 195) void __iomem *block_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 196) };
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 197)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 198) struct fsl_qdma_comp {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 199) dma_addr_t bus_addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 200) dma_addr_t desc_bus_addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 201) struct fsl_qdma_format *virt_addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 202) struct fsl_qdma_format *desc_virt_addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 203) struct fsl_qdma_chan *qchan;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 204) struct virt_dma_desc vdesc;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 205) struct list_head list;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 206) };
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 207)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 208) struct fsl_qdma_engine {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 209) struct dma_device dma_dev;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 210) void __iomem *ctrl_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 211) void __iomem *status_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 212) void __iomem *block_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 213) u32 n_chans;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 214) u32 n_queues;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 215) struct mutex fsl_qdma_mutex;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 216) int error_irq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 217) int *queue_irq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 218) u32 feature;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 219) struct fsl_qdma_queue *queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 220) struct fsl_qdma_queue **status;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 221) struct fsl_qdma_chan *chans;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 222) int block_number;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 223) int block_offset;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 224) int irq_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 225) int desc_allocated;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 226)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 227) };
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 228)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 229) static inline u64
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 230) qdma_ccdf_addr_get64(const struct fsl_qdma_format *ccdf)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 231) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 232) return le64_to_cpu(ccdf->data) & (U64_MAX >> 24);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 233) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 234)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 235) static inline void
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 236) qdma_desc_addr_set64(struct fsl_qdma_format *ccdf, u64 addr)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 237) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 238) ccdf->addr_hi = upper_32_bits(addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 239) ccdf->addr_lo = cpu_to_le32(lower_32_bits(addr));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 240) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 241)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 242) static inline u8
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 243) qdma_ccdf_get_queue(const struct fsl_qdma_format *ccdf)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 244) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 245) return ccdf->cfg8b_w1 & U8_MAX;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 246) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 247)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 248) static inline int
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 249) qdma_ccdf_get_offset(const struct fsl_qdma_format *ccdf)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 250) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 251) return (le32_to_cpu(ccdf->cfg) & QDMA_CCDF_MASK) >> QDMA_CCDF_OFFSET;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 252) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 253)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 254) static inline void
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 255) qdma_ccdf_set_format(struct fsl_qdma_format *ccdf, int offset)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 256) {
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 257) ccdf->cfg = cpu_to_le32(QDMA_CCDF_FORMAT |
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 258) (offset << QDMA_CCDF_OFFSET));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 259) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 260)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 261) static inline int
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 262) qdma_ccdf_get_status(const struct fsl_qdma_format *ccdf)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 263) {
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 264) return (le32_to_cpu(ccdf->status) & QDMA_CCDF_STATUS_MASK);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 265) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 266)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 267) static inline void
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 268) qdma_ccdf_set_ser(struct fsl_qdma_format *ccdf, int status)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 269) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 270) ccdf->status = cpu_to_le32(QDMA_CCDF_SER | status);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 271) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 272)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 273) static inline void qdma_csgf_set_len(struct fsl_qdma_format *csgf, int len)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 274) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 275) csgf->cfg = cpu_to_le32(len & QDMA_SG_LEN_MASK);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 276) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 277)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 278) static inline void qdma_csgf_set_f(struct fsl_qdma_format *csgf, int len)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 279) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 280) csgf->cfg = cpu_to_le32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 281) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 282)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 283) static u32 qdma_readl(struct fsl_qdma_engine *qdma, void __iomem *addr)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 284) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 285) return FSL_DMA_IN(qdma, addr, 32);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 286) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 287)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 288) static void qdma_writel(struct fsl_qdma_engine *qdma, u32 val,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 289) void __iomem *addr)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 290) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 291) FSL_DMA_OUT(qdma, addr, val, 32);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 292) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 293)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 294) static struct fsl_qdma_chan *to_fsl_qdma_chan(struct dma_chan *chan)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 295) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 296) return container_of(chan, struct fsl_qdma_chan, vchan.chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 297) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 298)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 299) static struct fsl_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 300) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 301) return container_of(vd, struct fsl_qdma_comp, vdesc);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 302) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 303)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 304) static void fsl_qdma_free_chan_resources(struct dma_chan *chan)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 305) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 306) struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 307) struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 308) struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 309) struct fsl_qdma_comp *comp_temp, *_comp_temp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 310) unsigned long flags;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 311) LIST_HEAD(head);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 312)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 313) spin_lock_irqsave(&fsl_chan->vchan.lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 314) vchan_get_all_descriptors(&fsl_chan->vchan, &head);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 315) spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 316)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 317) vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 318)
4b048178854da (Chen Zhou 2020-01-20 20:58:43 +0800 319) if (!fsl_queue->comp_pool && !fsl_queue->desc_pool)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 320) return;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 321)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 322) list_for_each_entry_safe(comp_temp, _comp_temp,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 323) &fsl_queue->comp_used, list) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 324) dma_pool_free(fsl_queue->comp_pool,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 325) comp_temp->virt_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 326) comp_temp->bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 327) dma_pool_free(fsl_queue->desc_pool,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 328) comp_temp->desc_virt_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 329) comp_temp->desc_bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 330) list_del(&comp_temp->list);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 331) kfree(comp_temp);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 332) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 333)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 334) list_for_each_entry_safe(comp_temp, _comp_temp,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 335) &fsl_queue->comp_free, list) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 336) dma_pool_free(fsl_queue->comp_pool,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 337) comp_temp->virt_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 338) comp_temp->bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 339) dma_pool_free(fsl_queue->desc_pool,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 340) comp_temp->desc_virt_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 341) comp_temp->desc_bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 342) list_del(&comp_temp->list);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 343) kfree(comp_temp);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 344) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 345)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 346) dma_pool_destroy(fsl_queue->comp_pool);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 347) dma_pool_destroy(fsl_queue->desc_pool);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 348)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 349) fsl_qdma->desc_allocated--;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 350) fsl_queue->comp_pool = NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 351) fsl_queue->desc_pool = NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 352) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 353)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 354) static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 355) dma_addr_t dst, dma_addr_t src, u32 len)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 356) {
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 357) u32 cmd;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 358) struct fsl_qdma_format *sdf, *ddf;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 359) struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 360)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 361) ccdf = fsl_comp->virt_addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 362) csgf_desc = fsl_comp->virt_addr + 1;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 363) csgf_src = fsl_comp->virt_addr + 2;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 364) csgf_dest = fsl_comp->virt_addr + 3;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 365) sdf = fsl_comp->desc_virt_addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 366) ddf = fsl_comp->desc_virt_addr + 1;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 367)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 368) memset(fsl_comp->virt_addr, 0, FSL_QDMA_COMMAND_BUFFER_SIZE);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 369) memset(fsl_comp->desc_virt_addr, 0, FSL_QDMA_DESCRIPTOR_BUFFER_SIZE);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 370) /* Head Command Descriptor(Frame Descriptor) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 371) qdma_desc_addr_set64(ccdf, fsl_comp->bus_addr + 16);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 372) qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(ccdf));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 373) qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(ccdf));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 374) /* Status notification is enqueued to status queue. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 375) /* Compound Command Descriptor(Frame List Table) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 376) qdma_desc_addr_set64(csgf_desc, fsl_comp->desc_bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 377) /* It must be 32 as Compound S/G Descriptor */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 378) qdma_csgf_set_len(csgf_desc, 32);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 379) qdma_desc_addr_set64(csgf_src, src);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 380) qdma_csgf_set_len(csgf_src, len);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 381) qdma_desc_addr_set64(csgf_dest, dst);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 382) qdma_csgf_set_len(csgf_dest, len);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 383) /* This entry is the last entry. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 384) qdma_csgf_set_f(csgf_dest, len);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 385) /* Descriptor Buffer */
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 386) cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE <<
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 387) FSL_QDMA_CMD_RWTTYPE_OFFSET);
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 388) sdf->data = QDMA_SDDF_CMD(cmd);
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 389)
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 390) cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE <<
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 391) FSL_QDMA_CMD_RWTTYPE_OFFSET);
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 392) cmd |= cpu_to_le32(FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET);
8f95adcf3a5aa (Peng Ma 2019-05-22 03:21:02 +0000 393) ddf->data = QDMA_SDDF_CMD(cmd);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 394) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 395)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 396) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 397) * Pre-request full command descriptor for enqueue.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 398) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 399) static int fsl_qdma_pre_request_enqueue_desc(struct fsl_qdma_queue *queue)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 400) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 401) int i;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 402) struct fsl_qdma_comp *comp_temp, *_comp_temp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 403)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 404) for (i = 0; i < queue->n_cq + FSL_COMMAND_QUEUE_OVERFLLOW; i++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 405) comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 406) if (!comp_temp)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 407) goto err_alloc;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 408) comp_temp->virt_addr =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 409) dma_pool_alloc(queue->comp_pool, GFP_KERNEL,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 410) &comp_temp->bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 411) if (!comp_temp->virt_addr)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 412) goto err_dma_alloc;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 413)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 414) comp_temp->desc_virt_addr =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 415) dma_pool_alloc(queue->desc_pool, GFP_KERNEL,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 416) &comp_temp->desc_bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 417) if (!comp_temp->desc_virt_addr)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 418) goto err_desc_dma_alloc;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 419)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 420) list_add_tail(&comp_temp->list, &queue->comp_free);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 421) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 422)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 423) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 424)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 425) err_desc_dma_alloc:
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 426) dma_pool_free(queue->comp_pool, comp_temp->virt_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 427) comp_temp->bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 428)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 429) err_dma_alloc:
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 430) kfree(comp_temp);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 431)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 432) err_alloc:
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 433) list_for_each_entry_safe(comp_temp, _comp_temp,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 434) &queue->comp_free, list) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 435) if (comp_temp->virt_addr)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 436) dma_pool_free(queue->comp_pool,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 437) comp_temp->virt_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 438) comp_temp->bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 439) if (comp_temp->desc_virt_addr)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 440) dma_pool_free(queue->desc_pool,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 441) comp_temp->desc_virt_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 442) comp_temp->desc_bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 443)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 444) list_del(&comp_temp->list);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 445) kfree(comp_temp);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 446) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 447)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 448) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 449) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 450)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 451) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 452) * Request a command descriptor for enqueue.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 453) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 454) static struct fsl_qdma_comp
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 455) *fsl_qdma_request_enqueue_desc(struct fsl_qdma_chan *fsl_chan)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 456) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 457) unsigned long flags;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 458) struct fsl_qdma_comp *comp_temp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 459) int timeout = FSL_QDMA_COMP_TIMEOUT;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 460) struct fsl_qdma_queue *queue = fsl_chan->queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 461)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 462) while (timeout--) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 463) spin_lock_irqsave(&queue->queue_lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 464) if (!list_empty(&queue->comp_free)) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 465) comp_temp = list_first_entry(&queue->comp_free,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 466) struct fsl_qdma_comp,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 467) list);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 468) list_del(&comp_temp->list);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 469)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 470) spin_unlock_irqrestore(&queue->queue_lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 471) comp_temp->qchan = fsl_chan;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 472) return comp_temp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 473) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 474) spin_unlock_irqrestore(&queue->queue_lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 475) udelay(1);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 476) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 477)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 478) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 479) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 480)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 481) static struct fsl_qdma_queue
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 482) *fsl_qdma_alloc_queue_resources(struct platform_device *pdev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 483) struct fsl_qdma_engine *fsl_qdma)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 484) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 485) int ret, len, i, j;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 486) int queue_num, block_number;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 487) unsigned int queue_size[FSL_QDMA_QUEUE_MAX];
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 488) struct fsl_qdma_queue *queue_head, *queue_temp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 489)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 490) queue_num = fsl_qdma->n_queues;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 491) block_number = fsl_qdma->block_number;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 492)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 493) if (queue_num > FSL_QDMA_QUEUE_MAX)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 494) queue_num = FSL_QDMA_QUEUE_MAX;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 495) len = sizeof(*queue_head) * queue_num * block_number;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 496) queue_head = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 497) if (!queue_head)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 498) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 499)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 500) ret = device_property_read_u32_array(&pdev->dev, "queue-sizes",
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 501) queue_size, queue_num);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 502) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 503) dev_err(&pdev->dev, "Can't get queue-sizes.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 504) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 505) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 506) for (j = 0; j < block_number; j++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 507) for (i = 0; i < queue_num; i++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 508) if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 509) queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 510) dev_err(&pdev->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 511) "Get wrong queue-sizes.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 512) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 513) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 514) queue_temp = queue_head + i + (j * queue_num);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 515)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 516) queue_temp->cq =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 517) dma_alloc_coherent(&pdev->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 518) sizeof(struct fsl_qdma_format) *
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 519) queue_size[i],
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 520) &queue_temp->bus_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 521) GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 522) if (!queue_temp->cq)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 523) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 524) queue_temp->block_base = fsl_qdma->block_base +
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 525) FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 526) queue_temp->n_cq = queue_size[i];
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 527) queue_temp->id = i;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 528) queue_temp->virt_head = queue_temp->cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 529) queue_temp->virt_tail = queue_temp->cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 530) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 531) * List for queue command buffer
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 532) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 533) INIT_LIST_HEAD(&queue_temp->comp_used);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 534) spin_lock_init(&queue_temp->queue_lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 535) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 536) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 537) return queue_head;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 538) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 539)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 540) static struct fsl_qdma_queue
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 541) *fsl_qdma_prep_status_queue(struct platform_device *pdev)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 542) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 543) int ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 544) unsigned int status_size;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 545) struct fsl_qdma_queue *status_head;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 546) struct device_node *np = pdev->dev.of_node;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 547)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 548) ret = of_property_read_u32(np, "status-sizes", &status_size);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 549) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 550) dev_err(&pdev->dev, "Can't get status-sizes.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 551) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 552) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 553) if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 554) status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 555) dev_err(&pdev->dev, "Get wrong status_size.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 556) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 557) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 558) status_head = devm_kzalloc(&pdev->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 559) sizeof(*status_head), GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 560) if (!status_head)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 561) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 562)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 563) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 564) * Buffer for queue command
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 565) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 566) status_head->cq = dma_alloc_coherent(&pdev->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 567) sizeof(struct fsl_qdma_format) *
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 568) status_size,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 569) &status_head->bus_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 570) GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 571) if (!status_head->cq) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 572) devm_kfree(&pdev->dev, status_head);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 573) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 574) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 575) status_head->n_cq = status_size;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 576) status_head->virt_head = status_head->cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 577) status_head->virt_tail = status_head->cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 578) status_head->comp_pool = NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 579)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 580) return status_head;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 581) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 582)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 583) static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 584) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 585) u32 reg;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 586) int i, j, count = FSL_QDMA_HALT_COUNT;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 587) void __iomem *block, *ctrl = fsl_qdma->ctrl_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 588)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 589) /* Disable the command queue and wait for idle state. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 590) reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 591) reg |= FSL_QDMA_DMR_DQD;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 592) qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 593) for (j = 0; j < fsl_qdma->block_number; j++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 594) block = fsl_qdma->block_base +
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 595) FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 596) for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 597) qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQMR(i));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 598) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 599) while (1) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 600) reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DSR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 601) if (!(reg & FSL_QDMA_DSR_DB))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 602) break;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 603) if (count-- < 0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 604) return -EBUSY;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 605) udelay(100);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 606) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 607)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 608) for (j = 0; j < fsl_qdma->block_number; j++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 609) block = fsl_qdma->block_base +
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 610) FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 611)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 612) /* Disable status queue. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 613) qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BSQMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 614)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 615) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 616) * clear the command queue interrupt detect register for
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 617) * all queues.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 618) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 619) qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 620) block + FSL_QDMA_BCQIDR(0));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 621) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 622)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 623) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 624) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 625)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 626) static int
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 627) fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 628) void *block,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 629) int id)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 630) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 631) bool duplicate;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 632) u32 reg, i, count;
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 633) u8 completion_status;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 634) struct fsl_qdma_queue *temp_queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 635) struct fsl_qdma_format *status_addr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 636) struct fsl_qdma_comp *fsl_comp = NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 637) struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 638) struct fsl_qdma_queue *fsl_status = fsl_qdma->status[id];
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 639)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 640) count = FSL_QDMA_MAX_SIZE;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 641)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 642) while (count--) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 643) duplicate = 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 644) reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQSR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 645) if (reg & FSL_QDMA_BSQSR_QE)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 646) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 647)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 648) status_addr = fsl_status->virt_head;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 649)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 650) if (qdma_ccdf_get_queue(status_addr) ==
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 651) __this_cpu_read(pre.queue) &&
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 652) qdma_ccdf_addr_get64(status_addr) ==
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 653) __this_cpu_read(pre.addr))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 654) duplicate = 1;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 655) i = qdma_ccdf_get_queue(status_addr) +
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 656) id * fsl_qdma->n_queues;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 657) __this_cpu_write(pre.addr, qdma_ccdf_addr_get64(status_addr));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 658) __this_cpu_write(pre.queue, qdma_ccdf_get_queue(status_addr));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 659) temp_queue = fsl_queue + i;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 660)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 661) spin_lock(&temp_queue->queue_lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 662) if (list_empty(&temp_queue->comp_used)) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 663) if (!duplicate) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 664) spin_unlock(&temp_queue->queue_lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 665) return -EAGAIN;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 666) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 667) } else {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 668) fsl_comp = list_first_entry(&temp_queue->comp_used,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 669) struct fsl_qdma_comp, list);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 670) if (fsl_comp->bus_addr + 16 !=
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 671) __this_cpu_read(pre.addr)) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 672) if (!duplicate) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 673) spin_unlock(&temp_queue->queue_lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 674) return -EAGAIN;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 675) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 676) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 677) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 678)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 679) if (duplicate) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 680) reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 681) reg |= FSL_QDMA_BSQMR_DI;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 682) qdma_desc_addr_set64(status_addr, 0x0);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 683) fsl_status->virt_head++;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 684) if (fsl_status->virt_head == fsl_status->cq
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 685) + fsl_status->n_cq)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 686) fsl_status->virt_head = fsl_status->cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 687) qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 688) spin_unlock(&temp_queue->queue_lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 689) continue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 690) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 691) list_del(&fsl_comp->list);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 692)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 693) completion_status = qdma_ccdf_get_status(status_addr);
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 694)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 695) reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 696) reg |= FSL_QDMA_BSQMR_DI;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 697) qdma_desc_addr_set64(status_addr, 0x0);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 698) fsl_status->virt_head++;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 699) if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 700) fsl_status->virt_head = fsl_status->cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 701) qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 702) spin_unlock(&temp_queue->queue_lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 703)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 704) /* The completion_status is evaluated here
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 705) * (outside of spin lock)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 706) */
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 707) if (completion_status) {
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 708) /* A completion error occurred! */
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 709) if (completion_status & QDMA_CCDF_STATUS_WTE) {
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 710) /* Write transaction error */
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 711) fsl_comp->vdesc.tx_result.result =
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 712) DMA_TRANS_WRITE_FAILED;
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 713) } else if (completion_status & QDMA_CCDF_STATUS_RTE) {
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 714) /* Read transaction error */
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 715) fsl_comp->vdesc.tx_result.result =
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 716) DMA_TRANS_READ_FAILED;
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 717) } else {
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 718) /* Command/source/destination
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 719) * description error
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 720) */
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 721) fsl_comp->vdesc.tx_result.result =
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 722) DMA_TRANS_ABORTED;
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 723) dev_err(fsl_qdma->dma_dev.dev,
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 724) "DMA status descriptor error %x\n",
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 725) completion_status);
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 726) }
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 727) }
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 728)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 729) spin_lock(&fsl_comp->qchan->vchan.lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 730) vchan_cookie_complete(&fsl_comp->vdesc);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 731) fsl_comp->qchan->status = DMA_COMPLETE;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 732) spin_unlock(&fsl_comp->qchan->vchan.lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 733) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 734)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 735) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 736) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 737)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 738) static irqreturn_t fsl_qdma_error_handler(int irq, void *dev_id)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 739) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 740) unsigned int intr;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 741) struct fsl_qdma_engine *fsl_qdma = dev_id;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 742) void __iomem *status = fsl_qdma->status_base;
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 743) unsigned int decfdw0r;
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 744) unsigned int decfdw1r;
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 745) unsigned int decfdw2r;
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 746) unsigned int decfdw3r;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 747)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 748) intr = qdma_readl(fsl_qdma, status + FSL_QDMA_DEDR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 749)
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 750) if (intr) {
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 751) decfdw0r = qdma_readl(fsl_qdma, status + FSL_QDMA_DECFDW0R);
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 752) decfdw1r = qdma_readl(fsl_qdma, status + FSL_QDMA_DECFDW1R);
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 753) decfdw2r = qdma_readl(fsl_qdma, status + FSL_QDMA_DECFDW2R);
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 754) decfdw3r = qdma_readl(fsl_qdma, status + FSL_QDMA_DECFDW3R);
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 755) dev_err(fsl_qdma->dma_dev.dev,
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 756) "DMA transaction error! (%x: %x-%x-%x-%x)\n",
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 757) intr, decfdw0r, decfdw1r, decfdw2r, decfdw3r);
ab6041e409001 (Koehrer Mathias (ETAS/EES-SL) 2020-07-15 08:41:19 +0000 758) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 759)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 760) qdma_writel(fsl_qdma, FSL_QDMA_DEDR_CLEAR, status + FSL_QDMA_DEDR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 761) return IRQ_HANDLED;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 762) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 763)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 764) static irqreturn_t fsl_qdma_queue_handler(int irq, void *dev_id)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 765) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 766) int id;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 767) unsigned int intr, reg;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 768) struct fsl_qdma_engine *fsl_qdma = dev_id;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 769) void __iomem *block, *ctrl = fsl_qdma->ctrl_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 770)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 771) id = irq - fsl_qdma->irq_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 772) if (id < 0 && id > fsl_qdma->block_number) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 773) dev_err(fsl_qdma->dma_dev.dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 774) "irq %d is wrong irq_base is %d\n",
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 775) irq, fsl_qdma->irq_base);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 776) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 777)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 778) block = fsl_qdma->block_base +
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 779) FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 780)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 781) intr = qdma_readl(fsl_qdma, block + FSL_QDMA_BCQIDR(0));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 782)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 783) if ((intr & FSL_QDMA_CQIDR_SQT) != 0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 784) intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 785)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 786) if (intr != 0) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 787) reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 788) reg |= FSL_QDMA_DMR_DQD;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 789) qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 790) qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQIER(0));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 791) dev_err(fsl_qdma->dma_dev.dev, "QDMA: status err!\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 792) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 793)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 794) /* Clear all detected events and interrupts. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 795) qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 796) block + FSL_QDMA_BCQIDR(0));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 797)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 798) return IRQ_HANDLED;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 799) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 800)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 801) static int
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 802) fsl_qdma_irq_init(struct platform_device *pdev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 803) struct fsl_qdma_engine *fsl_qdma)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 804) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 805) int i;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 806) int cpu;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 807) int ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 808) char irq_name[20];
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 809)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 810) fsl_qdma->error_irq =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 811) platform_get_irq_byname(pdev, "qdma-error");
e17be6e1b713f (Stephen Boyd 2019-07-30 11:15:10 -0700 812) if (fsl_qdma->error_irq < 0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 813) return fsl_qdma->error_irq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 814)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 815) ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 816) fsl_qdma_error_handler, 0,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 817) "qDMA error", fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 818) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 819) dev_err(&pdev->dev, "Can't register qDMA controller IRQ.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 820) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 821) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 822)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 823) for (i = 0; i < fsl_qdma->block_number; i++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 824) sprintf(irq_name, "qdma-queue%d", i);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 825) fsl_qdma->queue_irq[i] =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 826) platform_get_irq_byname(pdev, irq_name);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 827)
e17be6e1b713f (Stephen Boyd 2019-07-30 11:15:10 -0700 828) if (fsl_qdma->queue_irq[i] < 0)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 829) return fsl_qdma->queue_irq[i];
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 830)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 831) ret = devm_request_irq(&pdev->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 832) fsl_qdma->queue_irq[i],
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 833) fsl_qdma_queue_handler,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 834) 0,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 835) "qDMA queue",
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 836) fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 837) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 838) dev_err(&pdev->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 839) "Can't register qDMA queue IRQ.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 840) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 841) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 842)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 843) cpu = i % num_online_cpus();
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 844) ret = irq_set_affinity_hint(fsl_qdma->queue_irq[i],
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 845) get_cpu_mask(cpu));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 846) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 847) dev_err(&pdev->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 848) "Can't set cpu %d affinity to IRQ %d.\n",
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 849) cpu,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 850) fsl_qdma->queue_irq[i]);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 851) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 852) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 853) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 854)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 855) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 856) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 857)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 858) static void fsl_qdma_irq_exit(struct platform_device *pdev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 859) struct fsl_qdma_engine *fsl_qdma)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 860) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 861) int i;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 862)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 863) devm_free_irq(&pdev->dev, fsl_qdma->error_irq, fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 864) for (i = 0; i < fsl_qdma->block_number; i++)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 865) devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[i], fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 866) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 867)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 868) static int fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 869) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 870) u32 reg;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 871) int i, j, ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 872) struct fsl_qdma_queue *temp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 873) void __iomem *status = fsl_qdma->status_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 874) void __iomem *block, *ctrl = fsl_qdma->ctrl_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 875) struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 876)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 877) /* Try to halt the qDMA engine first. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 878) ret = fsl_qdma_halt(fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 879) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 880) dev_err(fsl_qdma->dma_dev.dev, "DMA halt failed!");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 881) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 882) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 883)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 884) for (i = 0; i < fsl_qdma->block_number; i++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 885) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 886) * Clear the command queue interrupt detect register for
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 887) * all queues.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 888) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 889)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 890) block = fsl_qdma->block_base +
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 891) FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, i);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 892) qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 893) block + FSL_QDMA_BCQIDR(0));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 894) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 895)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 896) for (j = 0; j < fsl_qdma->block_number; j++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 897) block = fsl_qdma->block_base +
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 898) FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 899) for (i = 0; i < fsl_qdma->n_queues; i++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 900) temp = fsl_queue + i + (j * fsl_qdma->n_queues);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 901) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 902) * Initialize Command Queue registers to
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 903) * point to the first
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 904) * command descriptor in memory.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 905) * Dequeue Pointer Address Registers
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 906) * Enqueue Pointer Address Registers
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 907) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 908)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 909) qdma_writel(fsl_qdma, temp->bus_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 910) block + FSL_QDMA_BCQDPA_SADDR(i));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 911) qdma_writel(fsl_qdma, temp->bus_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 912) block + FSL_QDMA_BCQEPA_SADDR(i));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 913)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 914) /* Initialize the queue mode. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 915) reg = FSL_QDMA_BCQMR_EN;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 916) reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq) - 4);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 917) reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq) - 6);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 918) qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BCQMR(i));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 919) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 920)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 921) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 922) * Workaround for erratum: ERR010812.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 923) * We must enable XOFF to avoid the enqueue rejection occurs.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 924) * Setting SQCCMR ENTER_WM to 0x20.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 925) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 926)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 927) qdma_writel(fsl_qdma, FSL_QDMA_SQCCMR_ENTER_WM,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 928) block + FSL_QDMA_SQCCMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 929)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 930) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 931) * Initialize status queue registers to point to the first
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 932) * command descriptor in memory.
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 933) * Dequeue Pointer Address Registers
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 934) * Enqueue Pointer Address Registers
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 935) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 936)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 937) qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 938) block + FSL_QDMA_SQEPAR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 939) qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 940) block + FSL_QDMA_SQDPAR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 941) /* Initialize status queue interrupt. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 942) qdma_writel(fsl_qdma, FSL_QDMA_BCQIER_CQTIE,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 943) block + FSL_QDMA_BCQIER(0));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 944) qdma_writel(fsl_qdma, FSL_QDMA_BSQICR_ICEN |
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 945) FSL_QDMA_BSQICR_ICST(5) | 0x8000,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 946) block + FSL_QDMA_BSQICR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 947) qdma_writel(fsl_qdma, FSL_QDMA_CQIER_MEIE |
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 948) FSL_QDMA_CQIER_TEIE,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 949) block + FSL_QDMA_CQIER);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 950)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 951) /* Initialize the status queue mode. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 952) reg = FSL_QDMA_BSQMR_EN;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 953) reg |= FSL_QDMA_BSQMR_CQ_SIZE(ilog2
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 954) (fsl_qdma->status[j]->n_cq) - 6);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 955)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 956) qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 957) reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 958) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 959)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 960) /* Initialize controller interrupt register. */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 961) qdma_writel(fsl_qdma, FSL_QDMA_DEDR_CLEAR, status + FSL_QDMA_DEDR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 962) qdma_writel(fsl_qdma, FSL_QDMA_DEIER_CLEAR, status + FSL_QDMA_DEIER);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 963)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 964) reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 965) reg &= ~FSL_QDMA_DMR_DQD;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 966) qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 967)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 968) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 969) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 970)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 971) static struct dma_async_tx_descriptor *
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 972) fsl_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 973) dma_addr_t src, size_t len, unsigned long flags)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 974) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 975) struct fsl_qdma_comp *fsl_comp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 976) struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 977)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 978) fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 979)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 980) if (!fsl_comp)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 981) return NULL;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 982)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 983) fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 984)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 985) return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 986) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 987)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 988) static void fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 989) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 990) u32 reg;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 991) struct virt_dma_desc *vdesc;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 992) struct fsl_qdma_comp *fsl_comp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 993) struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 994) void __iomem *block = fsl_queue->block_base;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 995)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 996) reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQSR(fsl_queue->id));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 997) if (reg & (FSL_QDMA_BCQSR_QF | FSL_QDMA_BCQSR_XOFF))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 998) return;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 999) vdesc = vchan_next_desc(&fsl_chan->vchan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1000) if (!vdesc)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1001) return;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1002) list_del(&vdesc->node);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1003) fsl_comp = to_fsl_qdma_comp(vdesc);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1004)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1005) memcpy(fsl_queue->virt_head++,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1006) fsl_comp->virt_addr, sizeof(struct fsl_qdma_format));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1007) if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1008) fsl_queue->virt_head = fsl_queue->cq;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1009)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1010) list_add_tail(&fsl_comp->list, &fsl_queue->comp_used);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1011) barrier();
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1012) reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQMR(fsl_queue->id));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1013) reg |= FSL_QDMA_BCQMR_EI;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1014) qdma_writel(fsl_chan->qdma, reg, block + FSL_QDMA_BCQMR(fsl_queue->id));
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1015) fsl_chan->status = DMA_IN_PROGRESS;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1016) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1017)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1018) static void fsl_qdma_free_desc(struct virt_dma_desc *vdesc)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1019) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1020) unsigned long flags;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1021) struct fsl_qdma_comp *fsl_comp;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1022) struct fsl_qdma_queue *fsl_queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1023)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1024) fsl_comp = to_fsl_qdma_comp(vdesc);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1025) fsl_queue = fsl_comp->qchan->queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1026)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1027) spin_lock_irqsave(&fsl_queue->queue_lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1028) list_add_tail(&fsl_comp->list, &fsl_queue->comp_free);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1029) spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1030) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1031)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1032) static void fsl_qdma_issue_pending(struct dma_chan *chan)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1033) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1034) unsigned long flags;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1035) struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1036) struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1037)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1038) spin_lock_irqsave(&fsl_queue->queue_lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1039) spin_lock(&fsl_chan->vchan.lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1040) if (vchan_issue_pending(&fsl_chan->vchan))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1041) fsl_qdma_enqueue_desc(fsl_chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1042) spin_unlock(&fsl_chan->vchan.lock);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1043) spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1044) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1045)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1046) static void fsl_qdma_synchronize(struct dma_chan *chan)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1047) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1048) struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1049)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1050) vchan_synchronize(&fsl_chan->vchan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1051) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1052)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1053) static int fsl_qdma_terminate_all(struct dma_chan *chan)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1054) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1055) LIST_HEAD(head);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1056) unsigned long flags;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1057) struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1058)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1059) spin_lock_irqsave(&fsl_chan->vchan.lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1060) vchan_get_all_descriptors(&fsl_chan->vchan, &head);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1061) spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1062) vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1063) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1064) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1065)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1066) static int fsl_qdma_alloc_chan_resources(struct dma_chan *chan)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1067) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1068) int ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1069) struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1070) struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1071) struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1072)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1073) if (fsl_queue->comp_pool && fsl_queue->desc_pool)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1074) return fsl_qdma->desc_allocated;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1075)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1076) INIT_LIST_HEAD(&fsl_queue->comp_free);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1077)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1078) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1079) * The dma pool for queue command buffer
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1080) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1081) fsl_queue->comp_pool =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1082) dma_pool_create("comp_pool",
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1083) chan->device->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1084) FSL_QDMA_COMMAND_BUFFER_SIZE,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1085) 64, 0);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1086) if (!fsl_queue->comp_pool)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1087) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1088)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1089) /*
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1090) * The dma pool for Descriptor(SD/DD) buffer
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1091) */
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1092) fsl_queue->desc_pool =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1093) dma_pool_create("desc_pool",
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1094) chan->device->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1095) FSL_QDMA_DESCRIPTOR_BUFFER_SIZE,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1096) 32, 0);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1097) if (!fsl_queue->desc_pool)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1098) goto err_desc_pool;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1099)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1100) ret = fsl_qdma_pre_request_enqueue_desc(fsl_queue);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1101) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1102) dev_err(chan->device->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1103) "failed to alloc dma buffer for S/G descriptor\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1104) goto err_mem;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1105) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1106)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1107) fsl_qdma->desc_allocated++;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1108) return fsl_qdma->desc_allocated;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1109)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1110) err_mem:
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1111) dma_pool_destroy(fsl_queue->desc_pool);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1112) err_desc_pool:
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1113) dma_pool_destroy(fsl_queue->comp_pool);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1114) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1115) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1116)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1117) static int fsl_qdma_probe(struct platform_device *pdev)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1118) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1119) int ret, i;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1120) int blk_num, blk_off;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1121) u32 len, chans, queues;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1122) struct resource *res;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1123) struct fsl_qdma_chan *fsl_chan;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1124) struct fsl_qdma_engine *fsl_qdma;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1125) struct device_node *np = pdev->dev.of_node;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1126)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1127) ret = of_property_read_u32(np, "dma-channels", &chans);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1128) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1129) dev_err(&pdev->dev, "Can't get dma-channels.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1130) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1131) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1132)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1133) ret = of_property_read_u32(np, "block-offset", &blk_off);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1134) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1135) dev_err(&pdev->dev, "Can't get block-offset.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1136) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1137) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1138)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1139) ret = of_property_read_u32(np, "block-number", &blk_num);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1140) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1141) dev_err(&pdev->dev, "Can't get block-number.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1142) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1143) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1144)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1145) blk_num = min_t(int, blk_num, num_online_cpus());
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1146)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1147) len = sizeof(*fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1148) fsl_qdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1149) if (!fsl_qdma)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1150) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1151)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1152) len = sizeof(*fsl_chan) * chans;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1153) fsl_qdma->chans = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1154) if (!fsl_qdma->chans)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1155) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1156)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1157) len = sizeof(struct fsl_qdma_queue *) * blk_num;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1158) fsl_qdma->status = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1159) if (!fsl_qdma->status)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1160) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1161)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1162) len = sizeof(int) * blk_num;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1163) fsl_qdma->queue_irq = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1164) if (!fsl_qdma->queue_irq)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1165) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1166)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1167) ret = of_property_read_u32(np, "fsl,dma-queues", &queues);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1168) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1169) dev_err(&pdev->dev, "Can't get queues.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1170) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1171) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1172)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1173) fsl_qdma->desc_allocated = 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1174) fsl_qdma->n_chans = chans;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1175) fsl_qdma->n_queues = queues;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1176) fsl_qdma->block_number = blk_num;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1177) fsl_qdma->block_offset = blk_off;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1178)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1179) mutex_init(&fsl_qdma->fsl_qdma_mutex);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1180)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1181) for (i = 0; i < fsl_qdma->block_number; i++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1182) fsl_qdma->status[i] = fsl_qdma_prep_status_queue(pdev);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1183) if (!fsl_qdma->status[i])
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1184) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1185) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1186) res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1187) fsl_qdma->ctrl_base = devm_ioremap_resource(&pdev->dev, res);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1188) if (IS_ERR(fsl_qdma->ctrl_base))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1189) return PTR_ERR(fsl_qdma->ctrl_base);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1190)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1191) res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1192) fsl_qdma->status_base = devm_ioremap_resource(&pdev->dev, res);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1193) if (IS_ERR(fsl_qdma->status_base))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1194) return PTR_ERR(fsl_qdma->status_base);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1195)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1196) res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1197) fsl_qdma->block_base = devm_ioremap_resource(&pdev->dev, res);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1198) if (IS_ERR(fsl_qdma->block_base))
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1199) return PTR_ERR(fsl_qdma->block_base);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1200) fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1201) if (!fsl_qdma->queue)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1202) return -ENOMEM;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1203)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1204) ret = fsl_qdma_irq_init(pdev, fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1205) if (ret)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1206) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1207)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1208) fsl_qdma->irq_base = platform_get_irq_byname(pdev, "qdma-queue0");
41814c4eadf8a (Krzysztof Kozlowski 2019-10-04 17:08:26 +0200 1209) if (fsl_qdma->irq_base < 0)
41814c4eadf8a (Krzysztof Kozlowski 2019-10-04 17:08:26 +0200 1210) return fsl_qdma->irq_base;
41814c4eadf8a (Krzysztof Kozlowski 2019-10-04 17:08:26 +0200 1211)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1212) fsl_qdma->feature = of_property_read_bool(np, "big-endian");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1213) INIT_LIST_HEAD(&fsl_qdma->dma_dev.channels);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1214)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1215) for (i = 0; i < fsl_qdma->n_chans; i++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1216) struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1217)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1218) fsl_chan->qdma = fsl_qdma;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1219) fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues *
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1220) fsl_qdma->block_number);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1221) fsl_chan->vchan.desc_free = fsl_qdma_free_desc;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1222) vchan_init(&fsl_chan->vchan, &fsl_qdma->dma_dev);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1223) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1224)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1225) dma_cap_set(DMA_MEMCPY, fsl_qdma->dma_dev.cap_mask);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1226)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1227) fsl_qdma->dma_dev.dev = &pdev->dev;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1228) fsl_qdma->dma_dev.device_free_chan_resources =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1229) fsl_qdma_free_chan_resources;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1230) fsl_qdma->dma_dev.device_alloc_chan_resources =
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1231) fsl_qdma_alloc_chan_resources;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1232) fsl_qdma->dma_dev.device_tx_status = dma_cookie_status;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1233) fsl_qdma->dma_dev.device_prep_dma_memcpy = fsl_qdma_prep_memcpy;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1234) fsl_qdma->dma_dev.device_issue_pending = fsl_qdma_issue_pending;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1235) fsl_qdma->dma_dev.device_synchronize = fsl_qdma_synchronize;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1236) fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1237)
f0c07993af0ac (Robin Gong 2021-04-26 16:59:09 +0800 1238) ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));
f0c07993af0ac (Robin Gong 2021-04-26 16:59:09 +0800 1239) if (ret) {
f0c07993af0ac (Robin Gong 2021-04-26 16:59:09 +0800 1240) dev_err(&pdev->dev, "dma_set_mask failure.\n");
f0c07993af0ac (Robin Gong 2021-04-26 16:59:09 +0800 1241) return ret;
f0c07993af0ac (Robin Gong 2021-04-26 16:59:09 +0800 1242) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1243)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1244) platform_set_drvdata(pdev, fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1245)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1246) ret = dma_async_device_register(&fsl_qdma->dma_dev);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1247) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1248) dev_err(&pdev->dev,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1249) "Can't register NXP Layerscape qDMA engine.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1250) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1251) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1252)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1253) ret = fsl_qdma_reg_init(fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1254) if (ret) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1255) dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1256) return ret;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1257) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1258)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1259) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1260) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1261)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1262) static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1263) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1264) struct fsl_qdma_chan *chan, *_chan;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1265)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1266) list_for_each_entry_safe(chan, _chan,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1267) &dmadev->channels, vchan.chan.device_node) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1268) list_del(&chan->vchan.chan.device_node);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1269) tasklet_kill(&chan->vchan.task);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1270) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1271) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1272)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1273) static int fsl_qdma_remove(struct platform_device *pdev)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1274) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1275) int i;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1276) struct fsl_qdma_queue *status;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1277) struct device_node *np = pdev->dev.of_node;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1278) struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1279)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1280) fsl_qdma_irq_exit(pdev, fsl_qdma);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1281) fsl_qdma_cleanup_vchan(&fsl_qdma->dma_dev);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1282) of_dma_controller_free(np);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1283) dma_async_device_unregister(&fsl_qdma->dma_dev);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1284)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1285) for (i = 0; i < fsl_qdma->block_number; i++) {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1286) status = fsl_qdma->status[i];
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1287) dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_format) *
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1288) status->n_cq, status->cq, status->bus_addr);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1289) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1290) return 0;
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1291) }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1292)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1293) static const struct of_device_id fsl_qdma_dt_ids[] = {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1294) { .compatible = "fsl,ls1021a-qdma", },
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1295) { /* sentinel */ }
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1296) };
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1297) MODULE_DEVICE_TABLE(of, fsl_qdma_dt_ids);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1298)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1299) static struct platform_driver fsl_qdma_driver = {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1300) .driver = {
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1301) .name = "fsl-qdma",
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1302) .of_match_table = fsl_qdma_dt_ids,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1303) },
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1304) .probe = fsl_qdma_probe,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1305) .remove = fsl_qdma_remove,
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1306) };
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1307)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1308) module_platform_driver(fsl_qdma_driver);
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1309)
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1310) MODULE_ALIAS("platform:fsl-qdma");
279cc97c01a20 (Arnd Bergmann 2018-12-10 21:55:02 +0100 1311) MODULE_LICENSE("GPL v2");
b092529e0aa09 (Peng Ma 2018-10-30 10:36:00 +0800 1312) MODULE_DESCRIPTION("NXP Layerscape qDMA engine driver");
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
33 Branches
55 Tags