905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 1) // SPDX-License-Identifier: GPL-2.0
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 2) /*
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 3) * Microsemi Switchtec(tm) PCIe Management Driver
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 4) * Copyright (c) 2019, Logan Gunthorpe <logang@deltatee.com>
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 5) * Copyright (c) 2019, GigaIO Networks, Inc
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 6) */
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 7)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 8) #include "dmaengine.h"
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 9)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 10) #include <linux/circ_buf.h>
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 11) #include <linux/dmaengine.h>
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 12) #include <linux/kref.h>
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 13) #include <linux/list.h>
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 14) #include <linux/module.h>
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 15) #include <linux/pci.h>
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 16)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 17) MODULE_DESCRIPTION("PLX ExpressLane PEX PCI Switch DMA Engine");
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 18) MODULE_VERSION("0.1");
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 19) MODULE_LICENSE("GPL");
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 20) MODULE_AUTHOR("Logan Gunthorpe");
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 21)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 22) #define PLX_REG_DESC_RING_ADDR 0x214
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 23) #define PLX_REG_DESC_RING_ADDR_HI 0x218
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 24) #define PLX_REG_DESC_RING_NEXT_ADDR 0x21C
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 25) #define PLX_REG_DESC_RING_COUNT 0x220
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 26) #define PLX_REG_DESC_RING_LAST_ADDR 0x224
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 27) #define PLX_REG_DESC_RING_LAST_SIZE 0x228
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 28) #define PLX_REG_PREF_LIMIT 0x234
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 29) #define PLX_REG_CTRL 0x238
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 30) #define PLX_REG_CTRL2 0x23A
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 31) #define PLX_REG_INTR_CTRL 0x23C
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 32) #define PLX_REG_INTR_STATUS 0x23E
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 33)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 34) #define PLX_REG_PREF_LIMIT_PREF_FOUR 8
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 35)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 36) #define PLX_REG_CTRL_GRACEFUL_PAUSE BIT(0)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 37) #define PLX_REG_CTRL_ABORT BIT(1)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 38) #define PLX_REG_CTRL_WRITE_BACK_EN BIT(2)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 39) #define PLX_REG_CTRL_START BIT(3)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 40) #define PLX_REG_CTRL_RING_STOP_MODE BIT(4)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 41) #define PLX_REG_CTRL_DESC_MODE_BLOCK (0 << 5)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 42) #define PLX_REG_CTRL_DESC_MODE_ON_CHIP (1 << 5)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 43) #define PLX_REG_CTRL_DESC_MODE_OFF_CHIP (2 << 5)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 44) #define PLX_REG_CTRL_DESC_INVALID BIT(8)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 45) #define PLX_REG_CTRL_GRACEFUL_PAUSE_DONE BIT(9)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 46) #define PLX_REG_CTRL_ABORT_DONE BIT(10)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 47) #define PLX_REG_CTRL_IMM_PAUSE_DONE BIT(12)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 48) #define PLX_REG_CTRL_IN_PROGRESS BIT(30)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 49)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 50) #define PLX_REG_CTRL_RESET_VAL (PLX_REG_CTRL_DESC_INVALID | \
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 51) PLX_REG_CTRL_GRACEFUL_PAUSE_DONE | \
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 52) PLX_REG_CTRL_ABORT_DONE | \
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 53) PLX_REG_CTRL_IMM_PAUSE_DONE)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 54)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 55) #define PLX_REG_CTRL_START_VAL (PLX_REG_CTRL_WRITE_BACK_EN | \
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 56) PLX_REG_CTRL_DESC_MODE_OFF_CHIP | \
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 57) PLX_REG_CTRL_START | \
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 58) PLX_REG_CTRL_RESET_VAL)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 59)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 60) #define PLX_REG_CTRL2_MAX_TXFR_SIZE_64B 0
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 61) #define PLX_REG_CTRL2_MAX_TXFR_SIZE_128B 1
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 62) #define PLX_REG_CTRL2_MAX_TXFR_SIZE_256B 2
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 63) #define PLX_REG_CTRL2_MAX_TXFR_SIZE_512B 3
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 64) #define PLX_REG_CTRL2_MAX_TXFR_SIZE_1KB 4
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 65) #define PLX_REG_CTRL2_MAX_TXFR_SIZE_2KB 5
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 66) #define PLX_REG_CTRL2_MAX_TXFR_SIZE_4B 7
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 67)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 68) #define PLX_REG_INTR_CRTL_ERROR_EN BIT(0)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 69) #define PLX_REG_INTR_CRTL_INV_DESC_EN BIT(1)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 70) #define PLX_REG_INTR_CRTL_ABORT_DONE_EN BIT(3)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 71) #define PLX_REG_INTR_CRTL_PAUSE_DONE_EN BIT(4)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 72) #define PLX_REG_INTR_CRTL_IMM_PAUSE_DONE_EN BIT(5)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 73)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 74) #define PLX_REG_INTR_STATUS_ERROR BIT(0)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 75) #define PLX_REG_INTR_STATUS_INV_DESC BIT(1)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 76) #define PLX_REG_INTR_STATUS_DESC_DONE BIT(2)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 77) #define PLX_REG_INTR_CRTL_ABORT_DONE BIT(3)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 78)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 79) struct plx_dma_hw_std_desc {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 80) __le32 flags_and_size;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 81) __le16 dst_addr_hi;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 82) __le16 src_addr_hi;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 83) __le32 dst_addr_lo;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 84) __le32 src_addr_lo;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 85) };
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 86)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 87) #define PLX_DESC_SIZE_MASK 0x7ffffff
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 88) #define PLX_DESC_FLAG_VALID BIT(31)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 89) #define PLX_DESC_FLAG_INT_WHEN_DONE BIT(30)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 90)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 91) #define PLX_DESC_WB_SUCCESS BIT(30)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 92) #define PLX_DESC_WB_RD_FAIL BIT(29)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 93) #define PLX_DESC_WB_WR_FAIL BIT(28)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 94)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 95) #define PLX_DMA_RING_COUNT 2048
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 96)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 97) struct plx_dma_desc {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 98) struct dma_async_tx_descriptor txd;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 99) struct plx_dma_hw_std_desc *hw;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 100) u32 orig_size;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 101) };
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 102)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 103) struct plx_dma_dev {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 104) struct dma_device dma_dev;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 105) struct dma_chan dma_chan;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 106) struct pci_dev __rcu *pdev;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 107) void __iomem *bar;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 108) struct tasklet_struct desc_task;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 109)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 110) spinlock_t ring_lock;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 111) bool ring_active;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 112) int head;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 113) int tail;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 114) struct plx_dma_hw_std_desc *hw_ring;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 115) dma_addr_t hw_ring_dma;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 116) struct plx_dma_desc **desc_ring;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 117) };
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 118)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 119) static struct plx_dma_dev *chan_to_plx_dma_dev(struct dma_chan *c)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 120) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 121) return container_of(c, struct plx_dma_dev, dma_chan);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 122) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 123)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 124) static struct plx_dma_desc *to_plx_desc(struct dma_async_tx_descriptor *txd)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 125) {
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 126) return container_of(txd, struct plx_dma_desc, txd);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 127) }
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 128)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 129) static struct plx_dma_desc *plx_dma_get_desc(struct plx_dma_dev *plxdev, int i)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 130) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 131) return plxdev->desc_ring[i & (PLX_DMA_RING_COUNT - 1)];
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 132) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 133)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 134) static void plx_dma_process_desc(struct plx_dma_dev *plxdev)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 135) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 136) struct dmaengine_result res;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 137) struct plx_dma_desc *desc;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 138) u32 flags;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 139)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 140) spin_lock_bh(&plxdev->ring_lock);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 141)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 142) while (plxdev->tail != plxdev->head) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 143) desc = plx_dma_get_desc(plxdev, plxdev->tail);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 144)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 145) flags = le32_to_cpu(READ_ONCE(desc->hw->flags_and_size));
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 146)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 147) if (flags & PLX_DESC_FLAG_VALID)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 148) break;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 149)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 150) res.residue = desc->orig_size - (flags & PLX_DESC_SIZE_MASK);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 151)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 152) if (flags & PLX_DESC_WB_SUCCESS)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 153) res.result = DMA_TRANS_NOERROR;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 154) else if (flags & PLX_DESC_WB_WR_FAIL)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 155) res.result = DMA_TRANS_WRITE_FAILED;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 156) else
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 157) res.result = DMA_TRANS_READ_FAILED;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 158)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 159) dma_cookie_complete(&desc->txd);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 160) dma_descriptor_unmap(&desc->txd);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 161) dmaengine_desc_get_callback_invoke(&desc->txd, &res);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 162) desc->txd.callback = NULL;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 163) desc->txd.callback_result = NULL;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 164)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 165) plxdev->tail++;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 166) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 167)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 168) spin_unlock_bh(&plxdev->ring_lock);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 169) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 170)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 171) static void plx_dma_abort_desc(struct plx_dma_dev *plxdev)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 172) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 173) struct dmaengine_result res;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 174) struct plx_dma_desc *desc;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 175)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 176) plx_dma_process_desc(plxdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 177)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 178) spin_lock_bh(&plxdev->ring_lock);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 179)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 180) while (plxdev->tail != plxdev->head) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 181) desc = plx_dma_get_desc(plxdev, plxdev->tail);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 182)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 183) res.residue = desc->orig_size;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 184) res.result = DMA_TRANS_ABORTED;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 185)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 186) dma_cookie_complete(&desc->txd);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 187) dma_descriptor_unmap(&desc->txd);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 188) dmaengine_desc_get_callback_invoke(&desc->txd, &res);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 189) desc->txd.callback = NULL;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 190) desc->txd.callback_result = NULL;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 191)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 192) plxdev->tail++;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 193) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 194)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 195) spin_unlock_bh(&plxdev->ring_lock);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 196) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 197)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 198) static void __plx_dma_stop(struct plx_dma_dev *plxdev)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 199) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 200) unsigned long timeout = jiffies + msecs_to_jiffies(1000);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 201) u32 val;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 202)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 203) val = readl(plxdev->bar + PLX_REG_CTRL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 204) if (!(val & ~PLX_REG_CTRL_GRACEFUL_PAUSE))
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 205) return;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 206)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 207) writel(PLX_REG_CTRL_RESET_VAL | PLX_REG_CTRL_GRACEFUL_PAUSE,
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 208) plxdev->bar + PLX_REG_CTRL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 209)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 210) while (!time_after(jiffies, timeout)) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 211) val = readl(plxdev->bar + PLX_REG_CTRL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 212) if (val & PLX_REG_CTRL_GRACEFUL_PAUSE_DONE)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 213) break;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 214)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 215) cpu_relax();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 216) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 217)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 218) if (!(val & PLX_REG_CTRL_GRACEFUL_PAUSE_DONE))
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 219) dev_err(plxdev->dma_dev.dev,
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 220) "Timeout waiting for graceful pause!\n");
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 221)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 222) writel(PLX_REG_CTRL_RESET_VAL | PLX_REG_CTRL_GRACEFUL_PAUSE,
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 223) plxdev->bar + PLX_REG_CTRL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 224)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 225) writel(0, plxdev->bar + PLX_REG_DESC_RING_COUNT);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 226) writel(0, plxdev->bar + PLX_REG_DESC_RING_ADDR);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 227) writel(0, plxdev->bar + PLX_REG_DESC_RING_ADDR_HI);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 228) writel(0, plxdev->bar + PLX_REG_DESC_RING_NEXT_ADDR);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 229) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 230)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 231) static void plx_dma_stop(struct plx_dma_dev *plxdev)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 232) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 233) rcu_read_lock();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 234) if (!rcu_dereference(plxdev->pdev)) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 235) rcu_read_unlock();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 236) return;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 237) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 238)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 239) __plx_dma_stop(plxdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 240)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 241) rcu_read_unlock();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 242) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 243)
6c1fd9ad2c5b7 (Allen Pais 2020-08-31 16:05:40 +0530 244) static void plx_dma_desc_task(struct tasklet_struct *t)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 245) {
6c1fd9ad2c5b7 (Allen Pais 2020-08-31 16:05:40 +0530 246) struct plx_dma_dev *plxdev = from_tasklet(plxdev, t, desc_task);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 247)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 248) plx_dma_process_desc(plxdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 249) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 250)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 251) static struct dma_async_tx_descriptor *plx_dma_prep_memcpy(struct dma_chan *c,
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 252) dma_addr_t dma_dst, dma_addr_t dma_src, size_t len,
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 253) unsigned long flags)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 254) __acquires(plxdev->ring_lock)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 255) {
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 256) struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(c);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 257) struct plx_dma_desc *plxdesc;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 258)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 259) spin_lock_bh(&plxdev->ring_lock);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 260) if (!plxdev->ring_active)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 261) goto err_unlock;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 262)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 263) if (!CIRC_SPACE(plxdev->head, plxdev->tail, PLX_DMA_RING_COUNT))
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 264) goto err_unlock;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 265)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 266) if (len > PLX_DESC_SIZE_MASK)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 267) goto err_unlock;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 268)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 269) plxdesc = plx_dma_get_desc(plxdev, plxdev->head);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 270) plxdev->head++;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 271)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 272) plxdesc->hw->dst_addr_lo = cpu_to_le32(lower_32_bits(dma_dst));
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 273) plxdesc->hw->dst_addr_hi = cpu_to_le16(upper_32_bits(dma_dst));
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 274) plxdesc->hw->src_addr_lo = cpu_to_le32(lower_32_bits(dma_src));
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 275) plxdesc->hw->src_addr_hi = cpu_to_le16(upper_32_bits(dma_src));
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 276)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 277) plxdesc->orig_size = len;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 278)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 279) if (flags & DMA_PREP_INTERRUPT)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 280) len |= PLX_DESC_FLAG_INT_WHEN_DONE;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 281)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 282) plxdesc->hw->flags_and_size = cpu_to_le32(len);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 283) plxdesc->txd.flags = flags;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 284)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 285) /* return with the lock held, it will be released in tx_submit */
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 286)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 287) return &plxdesc->txd;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 288)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 289) err_unlock:
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 290) /*
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 291) * Keep sparse happy by restoring an even lock count on
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 292) * this lock.
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 293) */
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 294) __acquire(plxdev->ring_lock);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 295)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 296) spin_unlock_bh(&plxdev->ring_lock);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 297) return NULL;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 298) }
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 299)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 300) static dma_cookie_t plx_dma_tx_submit(struct dma_async_tx_descriptor *desc)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 301) __releases(plxdev->ring_lock)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 302) {
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 303) struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(desc->chan);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 304) struct plx_dma_desc *plxdesc = to_plx_desc(desc);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 305) dma_cookie_t cookie;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 306)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 307) cookie = dma_cookie_assign(desc);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 308)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 309) /*
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 310) * Ensure the descriptor updates are visible to the dma device
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 311) * before setting the valid bit.
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 312) */
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 313) wmb();
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 314)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 315) plxdesc->hw->flags_and_size |= cpu_to_le32(PLX_DESC_FLAG_VALID);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 316)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 317) spin_unlock_bh(&plxdev->ring_lock);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 318)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 319) return cookie;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 320) }
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 321)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 322) static enum dma_status plx_dma_tx_status(struct dma_chan *chan,
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 323) dma_cookie_t cookie, struct dma_tx_state *txstate)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 324) {
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 325) struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(chan);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 326) enum dma_status ret;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 327)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 328) ret = dma_cookie_status(chan, cookie, txstate);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 329) if (ret == DMA_COMPLETE)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 330) return ret;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 331)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 332) plx_dma_process_desc(plxdev);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 333)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 334) return dma_cookie_status(chan, cookie, txstate);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 335) }
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 336)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 337) static void plx_dma_issue_pending(struct dma_chan *chan)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 338) {
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 339) struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(chan);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 340)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 341) rcu_read_lock();
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 342) if (!rcu_dereference(plxdev->pdev)) {
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 343) rcu_read_unlock();
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 344) return;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 345) }
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 346)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 347) /*
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 348) * Ensure the valid bits are visible before starting the
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 349) * DMA engine.
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 350) */
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 351) wmb();
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 352)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 353) writew(PLX_REG_CTRL_START_VAL, plxdev->bar + PLX_REG_CTRL);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 354)
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 355) rcu_read_unlock();
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 356) }
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 357)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 358) static irqreturn_t plx_dma_isr(int irq, void *devid)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 359) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 360) struct plx_dma_dev *plxdev = devid;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 361) u32 status;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 362)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 363) status = readw(plxdev->bar + PLX_REG_INTR_STATUS);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 364)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 365) if (!status)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 366) return IRQ_NONE;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 367)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 368) if (status & PLX_REG_INTR_STATUS_DESC_DONE && plxdev->ring_active)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 369) tasklet_schedule(&plxdev->desc_task);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 370)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 371) writew(status, plxdev->bar + PLX_REG_INTR_STATUS);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 372)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 373) return IRQ_HANDLED;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 374) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 375)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 376) static int plx_dma_alloc_desc(struct plx_dma_dev *plxdev)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 377) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 378) struct plx_dma_desc *desc;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 379) int i;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 380)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 381) plxdev->desc_ring = kcalloc(PLX_DMA_RING_COUNT,
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 382) sizeof(*plxdev->desc_ring), GFP_KERNEL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 383) if (!plxdev->desc_ring)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 384) return -ENOMEM;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 385)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 386) for (i = 0; i < PLX_DMA_RING_COUNT; i++) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 387) desc = kzalloc(sizeof(*desc), GFP_KERNEL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 388) if (!desc)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 389) goto free_and_exit;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 390)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 391) dma_async_tx_descriptor_init(&desc->txd, &plxdev->dma_chan);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 392) desc->txd.tx_submit = plx_dma_tx_submit;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 393) desc->hw = &plxdev->hw_ring[i];
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 394)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 395) plxdev->desc_ring[i] = desc;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 396) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 397)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 398) return 0;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 399)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 400) free_and_exit:
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 401) for (i = 0; i < PLX_DMA_RING_COUNT; i++)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 402) kfree(plxdev->desc_ring[i]);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 403) kfree(plxdev->desc_ring);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 404) return -ENOMEM;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 405) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 406)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 407) static int plx_dma_alloc_chan_resources(struct dma_chan *chan)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 408) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 409) struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(chan);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 410) size_t ring_sz = PLX_DMA_RING_COUNT * sizeof(*plxdev->hw_ring);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 411) int rc;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 412)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 413) plxdev->head = plxdev->tail = 0;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 414) plxdev->hw_ring = dma_alloc_coherent(plxdev->dma_dev.dev, ring_sz,
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 415) &plxdev->hw_ring_dma, GFP_KERNEL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 416) if (!plxdev->hw_ring)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 417) return -ENOMEM;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 418)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 419) rc = plx_dma_alloc_desc(plxdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 420) if (rc)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 421) goto out_free_hw_ring;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 422)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 423) rcu_read_lock();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 424) if (!rcu_dereference(plxdev->pdev)) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 425) rcu_read_unlock();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 426) rc = -ENODEV;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 427) goto out_free_hw_ring;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 428) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 429)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 430) writel(PLX_REG_CTRL_RESET_VAL, plxdev->bar + PLX_REG_CTRL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 431) writel(lower_32_bits(plxdev->hw_ring_dma),
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 432) plxdev->bar + PLX_REG_DESC_RING_ADDR);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 433) writel(upper_32_bits(plxdev->hw_ring_dma),
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 434) plxdev->bar + PLX_REG_DESC_RING_ADDR_HI);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 435) writel(lower_32_bits(plxdev->hw_ring_dma),
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 436) plxdev->bar + PLX_REG_DESC_RING_NEXT_ADDR);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 437) writel(PLX_DMA_RING_COUNT, plxdev->bar + PLX_REG_DESC_RING_COUNT);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 438) writel(PLX_REG_PREF_LIMIT_PREF_FOUR, plxdev->bar + PLX_REG_PREF_LIMIT);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 439)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 440) plxdev->ring_active = true;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 441)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 442) rcu_read_unlock();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 443)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 444) return PLX_DMA_RING_COUNT;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 445)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 446) out_free_hw_ring:
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 447) dma_free_coherent(plxdev->dma_dev.dev, ring_sz, plxdev->hw_ring,
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 448) plxdev->hw_ring_dma);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 449) return rc;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 450) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 451)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 452) static void plx_dma_free_chan_resources(struct dma_chan *chan)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 453) {
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 454) struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(chan);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 455) size_t ring_sz = PLX_DMA_RING_COUNT * sizeof(*plxdev->hw_ring);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 456) struct pci_dev *pdev;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 457) int irq = -1;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 458) int i;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 459)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 460) spin_lock_bh(&plxdev->ring_lock);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 461) plxdev->ring_active = false;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 462) spin_unlock_bh(&plxdev->ring_lock);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 463)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 464) plx_dma_stop(plxdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 465)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 466) rcu_read_lock();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 467) pdev = rcu_dereference(plxdev->pdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 468) if (pdev)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 469) irq = pci_irq_vector(pdev, 0);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 470) rcu_read_unlock();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 471)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 472) if (irq > 0)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 473) synchronize_irq(irq);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 474)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 475) tasklet_kill(&plxdev->desc_task);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 476)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 477) plx_dma_abort_desc(plxdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 478)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 479) for (i = 0; i < PLX_DMA_RING_COUNT; i++)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 480) kfree(plxdev->desc_ring[i]);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 481)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 482) kfree(plxdev->desc_ring);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 483) dma_free_coherent(plxdev->dma_dev.dev, ring_sz, plxdev->hw_ring,
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 484) plxdev->hw_ring_dma);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 485)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 486) }
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 487)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 488) static void plx_dma_release(struct dma_device *dma_dev)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 489) {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 490) struct plx_dma_dev *plxdev =
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 491) container_of(dma_dev, struct plx_dma_dev, dma_dev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 492)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 493) put_device(dma_dev->dev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 494) kfree(plxdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 495) }
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 496)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 497) static int plx_dma_create(struct pci_dev *pdev)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 498) {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 499) struct plx_dma_dev *plxdev;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 500) struct dma_device *dma;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 501) struct dma_chan *chan;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 502) int rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 503)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 504) plxdev = kzalloc(sizeof(*plxdev), GFP_KERNEL);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 505) if (!plxdev)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 506) return -ENOMEM;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 507)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 508) rc = request_irq(pci_irq_vector(pdev, 0), plx_dma_isr, 0,
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 509) KBUILD_MODNAME, plxdev);
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 510) if (rc)
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 511) goto free_plx;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 512)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 513) spin_lock_init(&plxdev->ring_lock);
6c1fd9ad2c5b7 (Allen Pais 2020-08-31 16:05:40 +0530 514) tasklet_setup(&plxdev->desc_task, plx_dma_desc_task);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 515)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 516) RCU_INIT_POINTER(plxdev->pdev, pdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 517) plxdev->bar = pcim_iomap_table(pdev)[0];
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 518)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 519) dma = &plxdev->dma_dev;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 520) dma->chancnt = 1;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 521) INIT_LIST_HEAD(&dma->channels);
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 522) dma_cap_set(DMA_MEMCPY, dma->cap_mask);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 523) dma->copy_align = DMAENGINE_ALIGN_1_BYTE;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 524) dma->dev = get_device(&pdev->dev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 525)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 526) dma->device_alloc_chan_resources = plx_dma_alloc_chan_resources;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 527) dma->device_free_chan_resources = plx_dma_free_chan_resources;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 528) dma->device_prep_dma_memcpy = plx_dma_prep_memcpy;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 529) dma->device_issue_pending = plx_dma_issue_pending;
4d3df16840a2b (Logan Gunthorpe 2020-01-03 14:20:21 -0700 530) dma->device_tx_status = plx_dma_tx_status;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 531) dma->device_release = plx_dma_release;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 532)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 533) chan = &plxdev->dma_chan;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 534) chan->device = dma;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 535) dma_cookie_init(chan);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 536) list_add_tail(&chan->device_node, &dma->channels);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 537)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 538) rc = dma_async_device_register(dma);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 539) if (rc) {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 540) pci_err(pdev, "Failed to register dma device: %d\n", rc);
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 541) goto put_device;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 542) }
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 543)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 544) pci_set_drvdata(pdev, plxdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 545)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 546) return 0;
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 547)
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 548) put_device:
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 549) put_device(&pdev->dev);
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 550) free_irq(pci_irq_vector(pdev, 0), plxdev);
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 551) free_plx:
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 552) kfree(plxdev);
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 553)
07503e6aefe4a (Dan Carpenter 2021-03-23 16:19:59 +0300 554) return rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 555) }
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 556)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 557) static int plx_dma_probe(struct pci_dev *pdev,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 558) const struct pci_device_id *id)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 559) {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 560) int rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 561)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 562) rc = pcim_enable_device(pdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 563) if (rc)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 564) return rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 565)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 566) rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 567) if (rc)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 568) rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 569) if (rc)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 570) return rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 571)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 572) rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 573) if (rc)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 574) rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 575) if (rc)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 576) return rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 577)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 578) rc = pcim_iomap_regions(pdev, 1, KBUILD_MODNAME);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 579) if (rc)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 580) return rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 581)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 582) rc = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 583) if (rc <= 0)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 584) return rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 585)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 586) pci_set_master(pdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 587)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 588) rc = plx_dma_create(pdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 589) if (rc)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 590) goto err_free_irq_vectors;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 591)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 592) pci_info(pdev, "PLX DMA Channel Registered\n");
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 593)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 594) return 0;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 595)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 596) err_free_irq_vectors:
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 597) pci_free_irq_vectors(pdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 598) return rc;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 599) }
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 600)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 601) static void plx_dma_remove(struct pci_dev *pdev)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 602) {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 603) struct plx_dma_dev *plxdev = pci_get_drvdata(pdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 604)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 605) free_irq(pci_irq_vector(pdev, 0), plxdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 606)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 607) rcu_assign_pointer(plxdev->pdev, NULL);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 608) synchronize_rcu();
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 609)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 610) spin_lock_bh(&plxdev->ring_lock);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 611) plxdev->ring_active = false;
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 612) spin_unlock_bh(&plxdev->ring_lock);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 613)
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 614) __plx_dma_stop(plxdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 615) plx_dma_abort_desc(plxdev);
c2dbcaa8c672d (Logan Gunthorpe 2020-01-03 14:20:20 -0700 616)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 617) plxdev->bar = NULL;
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 618) dma_async_device_unregister(&plxdev->dma_dev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 619)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 620) pci_free_irq_vectors(pdev);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 621) }
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 622)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 623) static const struct pci_device_id plx_dma_pci_tbl[] = {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 624) {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 625) .vendor = PCI_VENDOR_ID_PLX,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 626) .device = 0x87D0,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 627) .subvendor = PCI_ANY_ID,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 628) .subdevice = PCI_ANY_ID,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 629) .class = PCI_CLASS_SYSTEM_OTHER << 8,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 630) .class_mask = 0xFFFFFFFF,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 631) },
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 632) {0}
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 633) };
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 634) MODULE_DEVICE_TABLE(pci, plx_dma_pci_tbl);
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 635)
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 636) static struct pci_driver plx_dma_pci_driver = {
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 637) .name = KBUILD_MODNAME,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 638) .id_table = plx_dma_pci_tbl,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 639) .probe = plx_dma_probe,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 640) .remove = plx_dma_remove,
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 641) };
905ca51e63be7 (Logan Gunthorpe 2020-01-03 14:20:19 -0700 642) module_pci_driver(plx_dma_pci_driver);
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
33 Branches
55 Tags