On Fri, Jul 11, 2014 at 12:50:11AM +0300, Oded Gabbay wrote:Squashed and restructured in v2 of the patchset.
This patch adds the code base of the scheduler, which handles queue
creation, deletion and scheduling on the CP of the GPU.
Signed-off-by: Oded Gabbay <oded.gabbay@xxxxxxx>
I would rather see all this squashed, this gave feeling that driver
can access register which is latter remove. I know jungling with
patch squashing can be daunting but really it makes reviewing hard
here because i have to jump back and forth to see if thing i am looking
at really matter in the final version.
Cheers,
Jérôme
---
drivers/gpu/hsa/radeon/Makefile | 3 +-
drivers/gpu/hsa/radeon/cik_regs.h | 213 +++++++
drivers/gpu/hsa/radeon/kfd_device.c | 1 +
drivers/gpu/hsa/radeon/kfd_registers.c | 50 ++
drivers/gpu/hsa/radeon/kfd_sched_cik_static.c | 800 ++++++++++++++++++++++++++
drivers/gpu/hsa/radeon/kfd_vidmem.c | 61 ++
6 files changed, 1127 insertions(+), 1 deletion(-)
create mode 100644 drivers/gpu/hsa/radeon/cik_regs.h
create mode 100644 drivers/gpu/hsa/radeon/kfd_registers.c
create mode 100644 drivers/gpu/hsa/radeon/kfd_sched_cik_static.c
create mode 100644 drivers/gpu/hsa/radeon/kfd_vidmem.c
diff --git a/drivers/gpu/hsa/radeon/Makefile b/drivers/gpu/hsa/radeon/Makefile
index 989518a..28da10c 100644
--- a/drivers/gpu/hsa/radeon/Makefile
+++ b/drivers/gpu/hsa/radeon/Makefile
@@ -4,6 +4,7 @@
radeon_kfd-y := kfd_module.o kfd_device.o kfd_chardev.o \
kfd_pasid.o kfd_topology.o kfd_process.o \
- kfd_doorbell.o
+ kfd_doorbell.o kfd_sched_cik_static.o kfd_registers.o \
+ kfd_vidmem.o
obj-$(CONFIG_HSA_RADEON) += radeon_kfd.o
diff --git a/drivers/gpu/hsa/radeon/cik_regs.h b/drivers/gpu/hsa/radeon/cik_regs.h
new file mode 100644
index 0000000..d0cdc57
--- /dev/null
+++ b/drivers/gpu/hsa/radeon/cik_regs.h
@@ -0,0 +1,213 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef CIK_REGS_H
+#define CIK_REGS_H
+
+#define BIF_DOORBELL_CNTL 0x530Cu
+
+#define SRBM_GFX_CNTL 0xE44
+#define PIPEID(x) ((x) << 0)
+#define MEID(x) ((x) << 2)
+#define VMID(x) ((x) << 4)
+#define QUEUEID(x) ((x) << 8)
+
+#define SQ_CONFIG 0x8C00
+
+#define SH_MEM_BASES 0x8C28
+/* if PTR32, these are the bases for scratch and lds */
+#define PRIVATE_BASE(x) ((x) << 0) /* scratch */
+#define SHARED_BASE(x) ((x) << 16) /* LDS */
+#define SH_MEM_APE1_BASE 0x8C2C
+/* if PTR32, this is the base location of GPUVM */
+#define SH_MEM_APE1_LIMIT 0x8C30
+/* if PTR32, this is the upper limit of GPUVM */
+#define SH_MEM_CONFIG 0x8C34
+#define PTR32 (1 << 0)
+#define ALIGNMENT_MODE(x) ((x) << 2)
+#define SH_MEM_ALIGNMENT_MODE_DWORD 0
+#define SH_MEM_ALIGNMENT_MODE_DWORD_STRICT 1
+#define SH_MEM_ALIGNMENT_MODE_STRICT 2
+#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3
+#define DEFAULT_MTYPE(x) ((x) << 4)
+#define APE1_MTYPE(x) ((x) << 7)
+
+/* valid for both DEFAULT_MTYPE and APE1_MTYPE */
+#define MTYPE_NONCACHED 3
+
+
+#define SH_STATIC_MEM_CONFIG 0x9604u
+
+#define TC_CFG_L1_LOAD_POLICY0 0xAC68
+#define TC_CFG_L1_LOAD_POLICY1 0xAC6C
+#define TC_CFG_L1_STORE_POLICY 0xAC70
+#define TC_CFG_L2_LOAD_POLICY0 0xAC74
+#define TC_CFG_L2_LOAD_POLICY1 0xAC78
+#define TC_CFG_L2_STORE_POLICY0 0xAC7C
+#define TC_CFG_L2_STORE_POLICY1 0xAC80
+#define TC_CFG_L2_ATOMIC_POLICY 0xAC84
+#define TC_CFG_L1_VOLATILE 0xAC88
+#define TC_CFG_L2_VOLATILE 0xAC8C
+
+#define CP_PQ_WPTR_POLL_CNTL 0xC20C
+#define WPTR_POLL_EN (1 << 31)
+
+#define CP_ME1_PIPE0_INT_CNTL 0xC214
+#define CP_ME1_PIPE1_INT_CNTL 0xC218
+#define CP_ME1_PIPE2_INT_CNTL 0xC21C
+#define CP_ME1_PIPE3_INT_CNTL 0xC220
+#define CP_ME2_PIPE0_INT_CNTL 0xC224
+#define CP_ME2_PIPE1_INT_CNTL 0xC228
+#define CP_ME2_PIPE2_INT_CNTL 0xC22C
+#define CP_ME2_PIPE3_INT_CNTL 0xC230
+#define DEQUEUE_REQUEST_INT_ENABLE (1 << 13)
+#define WRM_POLL_TIMEOUT_INT_ENABLE (1 << 17)
+#define PRIV_REG_INT_ENABLE (1 << 23)
+#define TIME_STAMP_INT_ENABLE (1 << 26)
+#define GENERIC2_INT_ENABLE (1 << 29)
+#define GENERIC1_INT_ENABLE (1 << 30)
+#define GENERIC0_INT_ENABLE (1 << 31)
+#define CP_ME1_PIPE0_INT_STATUS 0xC214
+#define CP_ME1_PIPE1_INT_STATUS 0xC218
+#define CP_ME1_PIPE2_INT_STATUS 0xC21C
+#define CP_ME1_PIPE3_INT_STATUS 0xC220
+#define CP_ME2_PIPE0_INT_STATUS 0xC224
+#define CP_ME2_PIPE1_INT_STATUS 0xC228
+#define CP_ME2_PIPE2_INT_STATUS 0xC22C
+#define CP_ME2_PIPE3_INT_STATUS 0xC230
+#define DEQUEUE_REQUEST_INT_STATUS (1 << 13)
+#define WRM_POLL_TIMEOUT_INT_STATUS (1 << 17)
+#define PRIV_REG_INT_STATUS (1 << 23)
+#define TIME_STAMP_INT_STATUS (1 << 26)
+#define GENERIC2_INT_STATUS (1 << 29)
+#define GENERIC1_INT_STATUS (1 << 30)
+#define GENERIC0_INT_STATUS (1 << 31)
+
+#define CP_HPD_EOP_BASE_ADDR 0xC904
+#define CP_HPD_EOP_BASE_ADDR_HI 0xC908
+#define CP_HPD_EOP_VMID 0xC90C
+#define CP_HPD_EOP_CONTROL 0xC910
+#define EOP_SIZE(x) ((x) << 0)
+#define EOP_SIZE_MASK (0x3f << 0)
+#define CP_MQD_BASE_ADDR 0xC914
+#define CP_MQD_BASE_ADDR_HI 0xC918
+#define CP_HQD_ACTIVE 0xC91C
+#define CP_HQD_VMID 0xC920
+
+#define CP_HQD_PERSISTENT_STATE 0xC924u
+#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8)
+
+#define CP_HQD_PIPE_PRIORITY 0xC928u
+#define CP_HQD_QUEUE_PRIORITY 0xC92Cu
+#define CP_HQD_QUANTUM 0xC930u
+#define QUANTUM_EN 1U
+#define QUANTUM_SCALE_1MS (1U << 4)
+#define QUANTUM_DURATION(x) ((x) << 8)
+
+#define CP_HQD_PQ_BASE 0xC934
+#define CP_HQD_PQ_BASE_HI 0xC938
+#define CP_HQD_PQ_RPTR 0xC93C
+#define CP_HQD_PQ_RPTR_REPORT_ADDR 0xC940
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI 0xC944
+#define CP_HQD_PQ_WPTR_POLL_ADDR 0xC948
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI 0xC94C
+#define CP_HQD_PQ_DOORBELL_CONTROL 0xC950
+#define DOORBELL_OFFSET(x) ((x) << 2)
+#define DOORBELL_OFFSET_MASK (0x1fffff << 2)
+#define DOORBELL_SOURCE (1 << 28)
+#define DOORBELL_SCHD_HIT (1 << 29)
+#define DOORBELL_EN (1 << 30)
+#define DOORBELL_HIT (1 << 31)
+#define CP_HQD_PQ_WPTR 0xC954
+#define CP_HQD_PQ_CONTROL 0xC958
+#define QUEUE_SIZE(x) ((x) << 0)
+#define QUEUE_SIZE_MASK (0x3f << 0)
+#define RPTR_BLOCK_SIZE(x) ((x) << 8)
+#define RPTR_BLOCK_SIZE_MASK (0x3f << 8)
+#define MIN_AVAIL_SIZE(x) ((x) << 20)
+#define PQ_ATC_EN (1 << 23)
+#define PQ_VOLATILE (1 << 26)
+#define NO_UPDATE_RPTR (1 << 27)
+#define UNORD_DISPATCH (1 << 28)
+#define ROQ_PQ_IB_FLIP (1 << 29)
+#define PRIV_STATE (1 << 30)
+#define KMD_QUEUE (1 << 31)
+
+#define DEFAULT_RPTR_BLOCK_SIZE RPTR_BLOCK_SIZE(5)
+#define DEFAULT_MIN_AVAIL_SIZE MIN_AVAIL_SIZE(3)
+
+#define CP_HQD_IB_BASE_ADDR 0xC95Cu
+#define CP_HQD_IB_BASE_ADDR_HI 0xC960u
+#define CP_HQD_IB_RPTR 0xC964u
+#define CP_HQD_IB_CONTROL 0xC968u
+#define IB_ATC_EN (1U << 23)
+#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20)
+
+#define CP_HQD_DEQUEUE_REQUEST 0xC974
+#define DEQUEUE_REQUEST_DRAIN 1
+
+#define CP_HQD_SEMA_CMD 0xC97Cu
+#define CP_HQD_MSG_TYPE 0xC980u
+#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u
+#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u
+#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu
+#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u
+#define CP_HQD_HQ_SCHEDULER0 0xC994u
+#define CP_HQD_HQ_SCHEDULER1 0xC998u
+
+
+#define CP_MQD_CONTROL 0xC99C
+#define MQD_VMID(x) ((x) << 0)
+#define MQD_VMID_MASK (0xf << 0)
+#define MQD_CONTROL_PRIV_STATE_EN (1U << 8)
+
+#define GRBM_GFX_INDEX 0x30800
+#define INSTANCE_INDEX(x) ((x) << 0)
+#define SH_INDEX(x) ((x) << 8)
+#define SE_INDEX(x) ((x) << 16)
+#define SH_BROADCAST_WRITES (1 << 29)
+#define INSTANCE_BROADCAST_WRITES (1 << 30)
+#define SE_BROADCAST_WRITES (1 << 31)
+
+#define SQC_CACHES 0x30d20
+#define SQC_POLICY 0x8C38u
+#define SQC_VOLATILE 0x8C3Cu
+
+#define CP_PERFMON_CNTL 0x36020
+
+#define ATC_VMID0_PASID_MAPPING 0x339Cu
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u
+#define ATC_VMID_PASID_MAPPING_VALID (1U << 31)
+
+#define ATC_VM_APERTURE0_CNTL 0x3310u
+#define ATS_ACCESS_MODE_NEVER 0
+#define ATS_ACCESS_MODE_ALWAYS 1
+
+#define ATC_VM_APERTURE0_CNTL2 0x3318u
+#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u
+#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u
+#define ATC_VM_APERTURE1_CNTL 0x3314u
+#define ATC_VM_APERTURE1_CNTL2 0x331Cu
+#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu
+#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u
+
+#endif
diff --git a/drivers/gpu/hsa/radeon/kfd_device.c b/drivers/gpu/hsa/radeon/kfd_device.c
index 4e9fe6c..465c822 100644
--- a/drivers/gpu/hsa/radeon/kfd_device.c
+++ b/drivers/gpu/hsa/radeon/kfd_device.c
@@ -28,6 +28,7 @@
#include "kfd_scheduler.h"
static const struct kfd_device_info bonaire_device_info = {
+ .scheduler_class = &radeon_kfd_cik_static_scheduler_class,
.max_pasid_bits = 16,
};
diff --git a/drivers/gpu/hsa/radeon/kfd_registers.c b/drivers/gpu/hsa/radeon/kfd_registers.c
new file mode 100644
index 0000000..223debd
--- /dev/null
+++ b/drivers/gpu/hsa/radeon/kfd_registers.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/io.h>
+#include "kfd_priv.h"
+
+/* In KFD, "reg" is the byte offset of the register. */
+static void __iomem *reg_address(struct kfd_dev *dev, uint32_t reg)
+{
+ return dev->regs + reg;
+}
+
+void radeon_kfd_write_reg(struct kfd_dev *dev, uint32_t reg, uint32_t value)
+{
+ writel(value, reg_address(dev, reg));
+}
+
+uint32_t radeon_kfd_read_reg(struct kfd_dev *dev, uint32_t reg)
+{
+ return readl(reg_address(dev, reg));
+}
+
+void radeon_kfd_lock_srbm_index(struct kfd_dev *dev)
+{
+ kfd2kgd->lock_srbm_gfx_cntl(dev->kgd);
+}
+
+void radeon_kfd_unlock_srbm_index(struct kfd_dev *dev)
+{
+ kfd2kgd->unlock_srbm_gfx_cntl(dev->kgd);
+}
diff --git a/drivers/gpu/hsa/radeon/kfd_sched_cik_static.c b/drivers/gpu/hsa/radeon/kfd_sched_cik_static.c
new file mode 100644
index 0000000..b986ff9
--- /dev/null
+++ b/drivers/gpu/hsa/radeon/kfd_sched_cik_static.c
@@ -0,0 +1,800 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/log2.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include "kfd_priv.h"
+#include "kfd_scheduler.h"
+#include "cik_regs.h"
+
+/* CIK CP hardware is arranged with 8 queues per pipe and 8 pipes per MEC (microengine for compute).
+ * The first MEC is ME 1 with the GFX ME as ME 0.
+ * We split the CP with the KGD, they take the first N pipes and we take the rest.
+ */
+#define CIK_QUEUES_PER_PIPE 8
+#define CIK_PIPES_PER_MEC 4
+
+#define CIK_MAX_PIPES (2 * CIK_PIPES_PER_MEC)
+
+#define CIK_NUM_VMID 16
+
+#define CIK_HPD_SIZE_LOG2 11
+#define CIK_HPD_SIZE (1U << CIK_HPD_SIZE_LOG2)
+#define CIK_HPD_ALIGNMENT 256
+#define CIK_MQD_ALIGNMENT 4
+
+#pragma pack(push, 4)
+
+struct cik_hqd_registers {
+ u32 cp_mqd_base_addr;
+ u32 cp_mqd_base_addr_hi;
+ u32 cp_hqd_active;
+ u32 cp_hqd_vmid;
+ u32 cp_hqd_persistent_state;
+ u32 cp_hqd_pipe_priority;
+ u32 cp_hqd_queue_priority;
+ u32 cp_hqd_quantum;
+ u32 cp_hqd_pq_base;
+ u32 cp_hqd_pq_base_hi;
+ u32 cp_hqd_pq_rptr;
+ u32 cp_hqd_pq_rptr_report_addr;
+ u32 cp_hqd_pq_rptr_report_addr_hi;
+ u32 cp_hqd_pq_wptr_poll_addr;
+ u32 cp_hqd_pq_wptr_poll_addr_hi;
+ u32 cp_hqd_pq_doorbell_control;
+ u32 cp_hqd_pq_wptr;
+ u32 cp_hqd_pq_control;
+ u32 cp_hqd_ib_base_addr;
+ u32 cp_hqd_ib_base_addr_hi;
+ u32 cp_hqd_ib_rptr;
+ u32 cp_hqd_ib_control;
+ u32 cp_hqd_iq_timer;
+ u32 cp_hqd_iq_rptr;
+ u32 cp_hqd_dequeue_request;
+ u32 cp_hqd_dma_offload;
+ u32 cp_hqd_sema_cmd;
+ u32 cp_hqd_msg_type;
+ u32 cp_hqd_atomic0_preop_lo;
+ u32 cp_hqd_atomic0_preop_hi;
+ u32 cp_hqd_atomic1_preop_lo;
+ u32 cp_hqd_atomic1_preop_hi;
+ u32 cp_hqd_hq_scheduler0;
+ u32 cp_hqd_hq_scheduler1;
+ u32 cp_mqd_control;
+};
+
+struct cik_mqd {
+ u32 header;
+ u32 dispatch_initiator;
+ u32 dimensions[3];
+ u32 start_idx[3];
+ u32 num_threads[3];
+ u32 pipeline_stat_enable;
+ u32 perf_counter_enable;
+ u32 pgm[2];
+ u32 tba[2];
+ u32 tma[2];
+ u32 pgm_rsrc[2];
+ u32 vmid;
+ u32 resource_limits;
+ u32 static_thread_mgmt01[2];
+ u32 tmp_ring_size;
+ u32 static_thread_mgmt23[2];
+ u32 restart[3];
+ u32 thread_trace_enable;
+ u32 reserved1;
+ u32 user_data[16];
+ u32 vgtcs_invoke_count[2];
+ struct cik_hqd_registers queue_state;
+ u32 dequeue_cntr;
+ u32 interrupt_queue[64];
+};
+
+struct cik_mqd_padded {
+ struct cik_mqd mqd;
+ u8 padding[1024 - sizeof(struct cik_mqd)]; /* Pad MQD out to 1KB. (HW requires 4-byte alignment.) */
+};
+
+#pragma pack(pop)
+
+struct cik_static_private {
+ struct kfd_dev *dev;
+
+ struct mutex mutex;
+
+ unsigned int first_pipe;
+ unsigned int num_pipes;
+
+ unsigned long free_vmid_mask; /* unsigned long to make set/clear_bit happy */
+
+ /* Everything below here is offset by first_pipe. E.g. bit 0 in
+ * free_queues is queue 0 in pipe first_pipe
+ */
+
+ /* Queue q on pipe p is at bit QUEUES_PER_PIPE * p + q. */
+ unsigned long free_queues[DIV_ROUND_UP(CIK_MAX_PIPES * CIK_QUEUES_PER_PIPE, BITS_PER_LONG)];
+
+ kfd_mem_obj hpd_mem; /* Single allocation for HPDs for all KFD pipes. */
+ kfd_mem_obj mqd_mem; /* Single allocation for all MQDs for all KFD
+ * pipes. This is actually struct cik_mqd_padded. */
+ uint64_t hpd_addr; /* GPU address for hpd_mem. */
+ uint64_t mqd_addr; /* GPU address for mqd_mem. */
+ /*
+ * Pointer for mqd_mem.
+ * We keep this mapped because multiple processes may need to access it
+ * in parallel and this is simpler than controlling concurrent kmaps
+ */
+ struct cik_mqd_padded *mqds;
+};
+
+struct cik_static_process {
+ unsigned int vmid;
+ pasid_t pasid;
+};
+
+struct cik_static_queue {
+ unsigned int queue; /* + first_pipe * QUEUES_PER_PIPE */
+
+ uint64_t mqd_addr;
+ struct cik_mqd *mqd;
+
+ void __user *pq_addr;
+ void __user *rptr_address;
+ doorbell_t __user *wptr_address;
+ uint32_t doorbell_index;
+
+ uint32_t queue_size_encoded; /* CP_HQD_PQ_CONTROL.QUEUE_SIZE takes the queue size as log2(size) - 3. */
+};
+
+static uint32_t lower_32(uint64_t x)
+{
+ return (uint32_t)x;
+}
+
+static uint32_t upper_32(uint64_t x)
+{
+ return (uint32_t)(x >> 32);
+}
+
+/* SRBM_GFX_CNTL provides the MEC/pipe/queue and vmid for many registers that are
+ * In particular, CP_HQD_* and CP_MQD_* are instanced for each queue. CP_HPD_* are instanced for each pipe.
+ * SH_MEM_* are instanced per-VMID.
+ *
+ * We provide queue_select, pipe_select and vmid_select helpers that should be used before accessing
+ * registers from those groups. Note that these overwrite each other, e.g. after vmid_select the current
+ * selected MEC/pipe/queue is undefined.
+ *
+ * SRBM_GFX_CNTL and the registers it indexes are shared with KGD. You must be holding the srbm_gfx_cntl
+ * lock via lock_srbm_index before setting SRBM_GFX_CNTL or accessing any of the instanced registers.
+ */
+static uint32_t make_srbm_gfx_cntl_mpqv(unsigned int me, unsigned int pipe, unsigned int queue, unsigned int vmid)
+{
+ return QUEUEID(queue) | VMID(vmid) | MEID(me) | PIPEID(pipe);
+}
+
+static void pipe_select(struct cik_static_private *priv, unsigned int pipe)
+{
+ unsigned int pipe_in_mec = (pipe + priv->first_pipe) % CIK_PIPES_PER_MEC;
+ unsigned int mec = (pipe + priv->first_pipe) / CIK_PIPES_PER_MEC;
+
+ WRITE_REG(priv->dev, SRBM_GFX_CNTL, make_srbm_gfx_cntl_mpqv(mec+1, pipe_in_mec, 0, 0));
+}
+
+static void queue_select(struct cik_static_private *priv, unsigned int queue)
+{
+ unsigned int queue_in_pipe = queue % CIK_QUEUES_PER_PIPE;
+ unsigned int pipe = queue / CIK_QUEUES_PER_PIPE + priv->first_pipe;
+ unsigned int pipe_in_mec = pipe % CIK_PIPES_PER_MEC;
+ unsigned int mec = pipe / CIK_PIPES_PER_MEC;
+
+#if 0
+ dev_err(radeon_kfd_chardev(), "queue select %d = %u/%u/%u = 0x%08x\n", queue, mec+1, pipe_in_mec, queue_in_pipe,
+ make_srbm_gfx_cntl_mpqv(mec+1, pipe_in_mec, queue_in_pipe, 0));
+#endif
+
+ WRITE_REG(priv->dev, SRBM_GFX_CNTL, make_srbm_gfx_cntl_mpqv(mec+1, pipe_in_mec, queue_in_pipe, 0));
+}
+
+static void vmid_select(struct cik_static_private *priv, unsigned int vmid)
+{
+ WRITE_REG(priv->dev, SRBM_GFX_CNTL, make_srbm_gfx_cntl_mpqv(0, 0, 0, vmid));
+}
+
+static void lock_srbm_index(struct cik_static_private *priv)
+{
+ radeon_kfd_lock_srbm_index(priv->dev);
+}
+
+static void unlock_srbm_index(struct cik_static_private *priv)
+{
+ WRITE_REG(priv->dev, SRBM_GFX_CNTL, 0); /* Be nice to KGD, reset indexed CP registers to the GFX pipe. */
+ radeon_kfd_unlock_srbm_index(priv->dev);
+}
+
+/* One-time setup for all compute pipes. They need to be programmed with the address & size of the HPD EOP buffer. */
+static void init_pipes(struct cik_static_private *priv)
+{
+ unsigned int i;
+
+ lock_srbm_index(priv);
+
+ for (i = 0; i < priv->num_pipes; i++) {
+ uint64_t pipe_hpd_addr = priv->hpd_addr + i * CIK_HPD_SIZE;
+
+ pipe_select(priv, i);
+
+ WRITE_REG(priv->dev, CP_HPD_EOP_BASE_ADDR, lower_32(pipe_hpd_addr >> 8));
+ WRITE_REG(priv->dev, CP_HPD_EOP_BASE_ADDR_HI, upper_32(pipe_hpd_addr >> 8));
+ WRITE_REG(priv->dev, CP_HPD_EOP_VMID, 0);
+ WRITE_REG(priv->dev, CP_HPD_EOP_CONTROL, CIK_HPD_SIZE_LOG2 - 1);
+ }
+
+ unlock_srbm_index(priv);
+}
+
+/* Program the VMID -> PASID mapping for one VMID.
+ * PASID 0 is special: it means to associate no PASID with that VMID.
+ * This function waits for the VMID/PASID mapping to complete.
+ */
+static void set_vmid_pasid_mapping(struct cik_static_private *priv, unsigned int vmid, pasid_t pasid)
+{
+ /* We have to assume that there is no outstanding mapping.
+ * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because a mapping
+ * is in progress or because a mapping finished and the SW cleared it.
+ * So the protocol is to always wait & clear.
+ */
+
+ uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID;
+
+ WRITE_REG(priv->dev, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t), pasid_mapping);
+
+ while (!(READ_REG(priv->dev, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid)))
+ cpu_relax();
+ WRITE_REG(priv->dev, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
+}
+
+static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
+{
+ /* In 64-bit mode, we can only control the top 3 bits of the LDS, scratch and GPUVM apertures.
+ * The hardware fills in the remaining 59 bits according to the following pattern:
+ * LDS: X0000000'00000000 - X0000001'00000000 (4GB)
+ * Scratch: X0000001'00000000 - X0000002'00000000 (4GB)
+ * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB)
+ *
+ * (where X/Y is the configurable nybble with the low-bit 0)
+ *
+ * LDS and scratch will have the same top nybble programmed in the top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
+ * GPUVM can have a different top nybble programmed in the top 3 bits of SH_MEM_BASES.SHARED_BASE.
+ * We don't bother to support different top nybbles for LDS/Scratch and GPUVM.
+ */
+
+ BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE);
+
+ return PRIVATE_BASE(top_address_nybble << 12) | SHARED_BASE(top_address_nybble << 12);
+}
+
+/* Initial programming for all ATS registers.
+ * - enable ATS for all compute VMIDs
+ * - clear the VMID/PASID mapping for all compute VMIDS
+ * - program the shader core flat address settings:
+ * -- 64-bit mode
+ * -- unaligned access allowed
+ * -- noncached (this is the only CPU-coherent mode in CIK)
+ * -- APE 1 disabled
+ */
+static void init_ats(struct cik_static_private *priv)
+{
+ unsigned int i;
+
+ /* Enable self-ringing doorbell recognition and direct the BIF to send
+ * untranslated writes to the IOMMU before comparing to the aperture.*/
+ WRITE_REG(priv->dev, BIF_DOORBELL_CNTL, 0);
+
+ WRITE_REG(priv->dev, ATC_VM_APERTURE0_CNTL, ATS_ACCESS_MODE_ALWAYS);
+ WRITE_REG(priv->dev, ATC_VM_APERTURE0_CNTL2, priv->free_vmid_mask);
+ WRITE_REG(priv->dev, ATC_VM_APERTURE0_LOW_ADDR, 0);
+ WRITE_REG(priv->dev, ATC_VM_APERTURE0_HIGH_ADDR, 0);
+
+ WRITE_REG(priv->dev, ATC_VM_APERTURE1_CNTL, 0);
+ WRITE_REG(priv->dev, ATC_VM_APERTURE1_CNTL2, 0);
+ WRITE_REG(priv->dev, ATC_VM_APERTURE1_LOW_ADDR, 0);
+ WRITE_REG(priv->dev, ATC_VM_APERTURE1_HIGH_ADDR, 0);
+
+ lock_srbm_index(priv);
+
+ for (i = 0; i < CIK_NUM_VMID; i++) {
+ if (priv->free_vmid_mask & (1U << i)) {
+ uint32_t sh_mem_config;
+
+ set_vmid_pasid_mapping(priv, i, 0);
+
+ vmid_select(priv, i);
+
+ sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
+ sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
+
+ WRITE_REG(priv->dev, SH_MEM_CONFIG, sh_mem_config);
+
+ /* Configure apertures:
+ * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
+ * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
+ * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
+ */
+ WRITE_REG(priv->dev, SH_MEM_BASES, compute_sh_mem_bases_64bit(6));
+
+ /* Scratch aperture is not supported for now. */
+ WRITE_REG(priv->dev, SH_STATIC_MEM_CONFIG, 0);
+
+ /* APE1 disabled for now. */
+ WRITE_REG(priv->dev, SH_MEM_APE1_BASE, 1);
+ WRITE_REG(priv->dev, SH_MEM_APE1_LIMIT, 0);
+ }
+ }
+
+ unlock_srbm_index(priv);
+}
+
+static void exit_ats(struct cik_static_private *priv)
+{
+ unsigned int i;
+
+ for (i = 0; i < CIK_NUM_VMID; i++)
+ if (priv->free_vmid_mask & (1U << i))
+ set_vmid_pasid_mapping(priv, i, 0);
+
+ WRITE_REG(priv->dev, ATC_VM_APERTURE0_CNTL, ATS_ACCESS_MODE_NEVER);
+ WRITE_REG(priv->dev, ATC_VM_APERTURE0_CNTL2, 0);
+}
+
+static struct cik_static_private *kfd_scheduler_to_private(struct kfd_scheduler *scheduler)
+{
+ return (struct cik_static_private *)scheduler;
+}
+
+static struct cik_static_process *kfd_process_to_private(struct kfd_scheduler_process *process)
+{
+ return (struct cik_static_process *)process;
+}
+
+static struct cik_static_queue *kfd_queue_to_private(struct kfd_scheduler_queue *queue)
+{
+ return (struct cik_static_queue *)queue;
+}
+
+static int cik_static_create(struct kfd_dev *dev, struct kfd_scheduler **scheduler)
+{
+ struct cik_static_private *priv;
+ unsigned int i;
+ int err;
+ void *hpdptr;
+
+ priv = kmalloc(sizeof(*priv), GFP_KERNEL);
+ if (priv == NULL)
+ return -ENOMEM;
+
+ mutex_init(&priv->mutex);
+
+ priv->dev = dev;
+
+ priv->first_pipe = dev->shared_resources.first_compute_pipe;
+ priv->num_pipes = dev->shared_resources.compute_pipe_count;
+
+ for (i = 0; i < priv->num_pipes * CIK_QUEUES_PER_PIPE; i++)
+ __set_bit(i, priv->free_queues);
+
+ priv->free_vmid_mask = dev->shared_resources.compute_vmid_bitmap;
+
+ /*
+ * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
+ * The driver never accesses this memory after zeroing it. It doesn't even have
+ * to be saved/restored on suspend/resume because it contains no data when there
+ * are no active queues.
+ */
+ err = radeon_kfd_vidmem_alloc(dev,
+ CIK_HPD_SIZE * priv->num_pipes * 2,
+ PAGE_SIZE,
+ KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
+ &priv->hpd_mem);
+ if (err)
+ goto err_hpd_alloc;
+
+ err = radeon_kfd_vidmem_kmap(dev, priv->hpd_mem, &hpdptr);
+ if (err)
+ goto err_hpd_kmap;
+ memset(hpdptr, 0, CIK_HPD_SIZE * priv->num_pipes);
+ radeon_kfd_vidmem_unkmap(dev, priv->hpd_mem);
+
+ /*
+ * Allocate memory for all the MQDs.
+ * These are per-queue data that is hardware owned but with driver init.
+ * The driver has to copy this data into HQD registers when a
+ * pipe is (re)activated.
+ */
+ err = radeon_kfd_vidmem_alloc(dev,
+ sizeof(struct cik_mqd_padded) * priv->num_pipes * CIK_QUEUES_PER_PIPE,
+ PAGE_SIZE,
+ KFD_MEMPOOL_SYSTEM_CACHEABLE,
+ &priv->mqd_mem);
+ if (err)
+ goto err_mqd_alloc;
+ radeon_kfd_vidmem_kmap(dev, priv->mqd_mem, (void **)&priv->mqds);
+ if (err)
+ goto err_mqd_kmap;
+
+ *scheduler = (struct kfd_scheduler *)priv;
+
+ return 0;
+
+err_mqd_kmap:
+ radeon_kfd_vidmem_free(dev, priv->mqd_mem);
+err_mqd_alloc:
+err_hpd_kmap:
+ radeon_kfd_vidmem_free(dev, priv->hpd_mem);
+err_hpd_alloc:
+ mutex_destroy(&priv->mutex);
+ kfree(priv);
+ return err;
+}
+
+static void cik_static_destroy(struct kfd_scheduler *scheduler)
+{
+ struct cik_static_private *priv = kfd_scheduler_to_private(scheduler);
+
+ radeon_kfd_vidmem_unkmap(priv->dev, priv->mqd_mem);
+ radeon_kfd_vidmem_free(priv->dev, priv->mqd_mem);
+ radeon_kfd_vidmem_free(priv->dev, priv->hpd_mem);
+
+ mutex_destroy(&priv->mutex);
+
+ kfree(priv);
+}
+
+static void cik_static_start(struct kfd_scheduler *scheduler)
+{
+ struct cik_static_private *priv = kfd_scheduler_to_private(scheduler);
+
+ radeon_kfd_vidmem_gpumap(priv->dev, priv->hpd_mem, &priv->hpd_addr);
+ radeon_kfd_vidmem_gpumap(priv->dev, priv->mqd_mem, &priv->mqd_addr);
+
+ init_pipes(priv);
+ init_ats(priv);
+}
+
+static void cik_static_stop(struct kfd_scheduler *scheduler)
+{
+ struct cik_static_private *priv = kfd_scheduler_to_private(scheduler);
+
+ exit_ats(priv);
+
+ radeon_kfd_vidmem_ungpumap(priv->dev, priv->hpd_mem);
+ radeon_kfd_vidmem_ungpumap(priv->dev, priv->mqd_mem);
+}
+
+static bool allocate_vmid(struct cik_static_private *priv, unsigned int *vmid)
+{
+ bool ok = false;
+
+ mutex_lock(&priv->mutex);
+
+ if (priv->free_vmid_mask != 0) {
+ unsigned int v = __ffs64(priv->free_vmid_mask);
+
+ clear_bit(v, &priv->free_vmid_mask);
+ *vmid = v;
+
+ ok = true;
+ }
+
+ mutex_unlock(&priv->mutex);
+
+ return ok;
+}
+
+static void release_vmid(struct cik_static_private *priv, unsigned int vmid)
+{
+ /* It's okay to race against allocate_vmid because this only adds bits to free_vmid_mask.
+ * And set_bit/clear_bit are atomic wrt each other. */
+ set_bit(vmid, &priv->free_vmid_mask);
+}
+
+static void setup_vmid_for_process(struct cik_static_private *priv, struct cik_static_process *p)
+{
+ set_vmid_pasid_mapping(priv, p->vmid, p->pasid);
+
+ /*
+ * SH_MEM_CONFIG and others need to be programmed differently
+ * for 32/64-bit processes. And maybe other reasons.
+ */
+}
+
+static int
+cik_static_register_process(struct kfd_scheduler *scheduler, struct kfd_process *process,
+ struct kfd_scheduler_process **scheduler_process)
+{
+ struct cik_static_private *priv = kfd_scheduler_to_private(scheduler);
+
+ struct cik_static_process *hwp;
+
+ hwp = kmalloc(sizeof(*hwp), GFP_KERNEL);
+ if (hwp == NULL)
+ return -ENOMEM;
+
+ if (!allocate_vmid(priv, &hwp->vmid)) {
+ kfree(hwp);
+ return -ENOMEM;
+ }
+
+ hwp->pasid = process->pasid;
+
+ setup_vmid_for_process(priv, hwp);
+
+ *scheduler_process = (struct kfd_scheduler_process *)hwp;
+
+ return 0;
+}
+
+static void cik_static_deregister_process(struct kfd_scheduler *scheduler,
+ struct kfd_scheduler_process *scheduler_process)
+{
+ struct cik_static_private *priv = kfd_scheduler_to_private(scheduler);
+ struct cik_static_process *pp = kfd_process_to_private(scheduler_process);
+
+ release_vmid(priv, pp->vmid);
+ kfree(pp);
+}
+
+static bool allocate_hqd(struct cik_static_private *priv, unsigned int *queue)
+{
+ bool ok = false;
+ unsigned int q;
+
+ mutex_lock(&priv->mutex);
+
+ q = find_first_bit(priv->free_queues, priv->num_pipes * CIK_QUEUES_PER_PIPE);
+
+ if (q != priv->num_pipes * CIK_QUEUES_PER_PIPE) {
+ clear_bit(q, priv->free_queues);
+ *queue = q;
+
+ ok = true;
+ }
+
+ mutex_unlock(&priv->mutex);
+
+ return ok;
+}
+
+static void release_hqd(struct cik_static_private *priv, unsigned int queue)
+{
+ /* It's okay to race against allocate_hqd because this only adds bits to free_queues.
+ * And set_bit/clear_bit are atomic wrt each other. */
+ set_bit(queue, priv->free_queues);
+}
+
+static void init_mqd(const struct cik_static_queue *queue, const struct cik_static_process *process)
+{
+ struct cik_mqd *mqd = queue->mqd;
+
+ memset(mqd, 0, sizeof(*mqd));
+
+ mqd->header = 0xC0310800;
+ mqd->pipeline_stat_enable = 1;
+ mqd->static_thread_mgmt01[0] = 0xffffffff;
+ mqd->static_thread_mgmt01[1] = 0xffffffff;
+ mqd->static_thread_mgmt23[0] = 0xffffffff;
+ mqd->static_thread_mgmt23[1] = 0xffffffff;
+
+ mqd->queue_state.cp_mqd_base_addr = lower_32(queue->mqd_addr);
+ mqd->queue_state.cp_mqd_base_addr_hi = upper_32(queue->mqd_addr);
+ mqd->queue_state.cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
+
+ mqd->queue_state.cp_hqd_pq_base = lower_32((uintptr_t)queue->pq_addr >> 8);
+ mqd->queue_state.cp_hqd_pq_base_hi = upper_32((uintptr_t)queue->pq_addr >> 8);
+ mqd->queue_state.cp_hqd_pq_control = QUEUE_SIZE(queue->queue_size_encoded) | DEFAULT_RPTR_BLOCK_SIZE
+ | DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
+ mqd->queue_state.cp_hqd_pq_rptr_report_addr = lower_32((uintptr_t)queue->rptr_address);
+ mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = upper_32((uintptr_t)queue->rptr_address);
+ mqd->queue_state.cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(queue->doorbell_index) | DOORBELL_EN;
+ mqd->queue_state.cp_hqd_vmid = process->vmid;
+ mqd->queue_state.cp_hqd_active = 1;
+
+ mqd->queue_state.cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE;
+
+ /* The values for these 3 are from WinKFD. */
+ mqd->queue_state.cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | QUANTUM_DURATION(10);
+ mqd->queue_state.cp_hqd_pipe_priority = 1;
+ mqd->queue_state.cp_hqd_queue_priority = 15;
+
+ mqd->queue_state.cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE;
+}
+
+/* Write the HQD registers and activate the queue.
+ * Requires that SRBM_GFX_CNTL has already been programmed for the queue.
+ */
+static void load_hqd(struct cik_static_private *priv, struct cik_static_queue *queue)
+{
+ struct kfd_dev *dev = priv->dev;
+ const struct cik_hqd_registers *qs = &queue->mqd->queue_state;
+
+ WRITE_REG(dev, CP_MQD_BASE_ADDR, qs->cp_mqd_base_addr);
+ WRITE_REG(dev, CP_MQD_BASE_ADDR_HI, qs->cp_mqd_base_addr_hi);
+ WRITE_REG(dev, CP_MQD_CONTROL, qs->cp_mqd_control);
+
+ WRITE_REG(dev, CP_HQD_PQ_BASE, qs->cp_hqd_pq_base);
+ WRITE_REG(dev, CP_HQD_PQ_BASE_HI, qs->cp_hqd_pq_base_hi);
+ WRITE_REG(dev, CP_HQD_PQ_CONTROL, qs->cp_hqd_pq_control);
+ /* DOORBELL_CONTROL before WPTR because WPTR writes are dropped if DOORBELL_HIT is set. */
+ WRITE_REG(dev, CP_HQD_PQ_DOORBELL_CONTROL, qs->cp_hqd_pq_doorbell_control);
+ WRITE_REG(dev, CP_HQD_PQ_WPTR, qs->cp_hqd_pq_wptr);
+ WRITE_REG(dev, CP_HQD_PQ_RPTR, qs->cp_hqd_pq_rptr);
+ WRITE_REG(dev, CP_HQD_PQ_RPTR_REPORT_ADDR, qs->cp_hqd_pq_rptr_report_addr);
+ WRITE_REG(dev, CP_HQD_PQ_RPTR_REPORT_ADDR_HI, qs->cp_hqd_pq_rptr_report_addr_hi);
+
+ WRITE_REG(dev, CP_HQD_VMID, qs->cp_hqd_vmid);
+ WRITE_REG(dev, CP_HQD_PERSISTENT_STATE, qs->cp_hqd_persistent_state);
+ WRITE_REG(dev, CP_HQD_QUANTUM, qs->cp_hqd_quantum);
+ WRITE_REG(dev, CP_HQD_PIPE_PRIORITY, qs->cp_hqd_pipe_priority);
+ WRITE_REG(dev, CP_HQD_QUEUE_PRIORITY, qs->cp_hqd_queue_priority);
+
+ WRITE_REG(dev, CP_HQD_IB_CONTROL, qs->cp_hqd_ib_control);
+ WRITE_REG(dev, CP_HQD_IB_BASE_ADDR, qs->cp_hqd_ib_base_addr);
+ WRITE_REG(dev, CP_HQD_IB_BASE_ADDR_HI, qs->cp_hqd_ib_base_addr_hi);
+ WRITE_REG(dev, CP_HQD_IB_RPTR, qs->cp_hqd_ib_rptr);
+ WRITE_REG(dev, CP_HQD_SEMA_CMD, qs->cp_hqd_sema_cmd);
+ WRITE_REG(dev, CP_HQD_MSG_TYPE, qs->cp_hqd_msg_type);
+ WRITE_REG(dev, CP_HQD_ATOMIC0_PREOP_LO, qs->cp_hqd_atomic0_preop_lo);
+ WRITE_REG(dev, CP_HQD_ATOMIC0_PREOP_HI, qs->cp_hqd_atomic0_preop_hi);
+ WRITE_REG(dev, CP_HQD_ATOMIC1_PREOP_LO, qs->cp_hqd_atomic1_preop_lo);
+ WRITE_REG(dev, CP_HQD_ATOMIC1_PREOP_HI, qs->cp_hqd_atomic1_preop_hi);
+ WRITE_REG(dev, CP_HQD_HQ_SCHEDULER0, qs->cp_hqd_hq_scheduler0);
+ WRITE_REG(dev, CP_HQD_HQ_SCHEDULER1, qs->cp_hqd_hq_scheduler1);
+
+ WRITE_REG(dev, CP_HQD_ACTIVE, 1);
+}
+
+static void activate_queue(struct cik_static_private *priv, struct cik_static_queue *queue)
+{
+ bool wptr_shadow_valid;
+ doorbell_t wptr_shadow;
+
+ /* Avoid sleeping while holding the SRBM lock. */
+ wptr_shadow_valid = !get_user(wptr_shadow, queue->wptr_address);
+
+ lock_srbm_index(priv);
+ queue_select(priv, queue->queue);
+
+ load_hqd(priv, queue);
+
+ /* Doorbell and wptr are special because there is a race when reactivating a queue.
+ * Since doorbell writes to deactivated queues are ignored by hardware, the application
+ * shadows the doorbell into memory at queue->wptr_address.
+ *
+ * We want the queue to automatically resume processing as if it were always active,
+ * so we want to copy from queue->wptr_address into the wptr/doorbell.
+ *
+ * The race is that the app could write a new wptr into the doorbell before we
+ * write the shadowed wptr, resulting in an old wptr written later.
+ *
+ * The hardware solves this ignoring CP_HQD_WPTR writes after a doorbell write.
+ * So the KFD can activate the doorbell then write the shadow wptr to CP_HQD_WPTR
+ * knowing it will be ignored if the user has written a more-recent doorbell.
+ */
+ if (wptr_shadow_valid)
+ WRITE_REG(priv->dev, CP_HQD_PQ_WPTR, wptr_shadow);
+
+ unlock_srbm_index(priv);
+}
+
+static void drain_hqd(struct cik_static_private *priv)
+{
+ WRITE_REG(priv->dev, CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQUEST_DRAIN);
+}
+
+static void wait_hqd_inactive(struct cik_static_private *priv)
+{
+ while (READ_REG(priv->dev, CP_HQD_ACTIVE) != 0)
+ cpu_relax();
+}
+
+static void deactivate_queue(struct cik_static_private *priv, struct cik_static_queue *queue)
+{
+ lock_srbm_index(priv);
+ queue_select(priv, queue->queue);
+
+ drain_hqd(priv);
+ wait_hqd_inactive(priv);
+
+ unlock_srbm_index(priv);
+}
+
+#define BIT_MASK_64(high, low) (((1ULL << (high)) - 1) & ~((1ULL << (low)) - 1))
+#define RING_ADDRESS_BAD_BIT_MASK (~BIT_MASK_64(48, 8))
+#define RWPTR_ADDRESS_BAD_BIT_MASK (~BIT_MASK_64(48, 2))
+
+#define MAX_QUEUE_SIZE (1ULL << 32)
+#define MIN_QUEUE_SIZE (1ULL << 10)
+
+static int
+cik_static_create_queue(struct kfd_scheduler *scheduler,
+ struct kfd_scheduler_process *process,
+ struct kfd_scheduler_queue *queue,
+ void __user *ring_address,
+ uint64_t ring_size,
+ void __user *rptr_address,
+ void __user *wptr_address,
+ unsigned int doorbell)
+{
+ struct cik_static_private *priv = kfd_scheduler_to_private(scheduler);
+ struct cik_static_process *hwp = kfd_process_to_private(process);
+ struct cik_static_queue *hwq = kfd_queue_to_private(queue);
+
+ if ((uint64_t)ring_address & RING_ADDRESS_BAD_BIT_MASK
+ || (uint64_t)rptr_address & RWPTR_ADDRESS_BAD_BIT_MASK
+ || (uint64_t)wptr_address & RWPTR_ADDRESS_BAD_BIT_MASK)
+ return -EINVAL;
+
+ if (ring_size > MAX_QUEUE_SIZE || ring_size < MIN_QUEUE_SIZE || !is_power_of_2(ring_size))
+ return -EINVAL;
+
+ if (!allocate_hqd(priv, &hwq->queue))
+ return -ENOMEM;
+
+ hwq->mqd_addr = priv->mqd_addr + sizeof(struct cik_mqd_padded) * hwq->queue;
+ hwq->mqd = &priv->mqds[hwq->queue].mqd;
+ hwq->pq_addr = ring_address;
+ hwq->rptr_address = rptr_address;
+ hwq->wptr_address = wptr_address;
+ hwq->doorbell_index = doorbell;
+ hwq->queue_size_encoded = ilog2(ring_size) - 3;
+
+ init_mqd(hwq, hwp);
+ activate_queue(priv, hwq);
+
+ return 0;
+}
+
+static void
+cik_static_destroy_queue(struct kfd_scheduler *scheduler, struct kfd_scheduler_queue *queue)
+{
+ struct cik_static_private *priv = kfd_scheduler_to_private(scheduler);
+ struct cik_static_queue *hwq = kfd_queue_to_private(queue);
+
+ deactivate_queue(priv, hwq);
+
+ release_hqd(priv, hwq->queue);
+}
+
+const struct kfd_scheduler_class radeon_kfd_cik_static_scheduler_class = {
+ .name = "CIK static scheduler",
+ .create = cik_static_create,
+ .destroy = cik_static_destroy,
+ .start = cik_static_start,
+ .stop = cik_static_stop,
+ .register_process = cik_static_register_process,
+ .deregister_process = cik_static_deregister_process,
+ .queue_size = sizeof(struct cik_static_queue),
+ .create_queue = cik_static_create_queue,
+ .destroy_queue = cik_static_destroy_queue,
+};
diff --git a/drivers/gpu/hsa/radeon/kfd_vidmem.c b/drivers/gpu/hsa/radeon/kfd_vidmem.c
new file mode 100644
index 0000000..c8d3770
--- /dev/null
+++ b/drivers/gpu/hsa/radeon/kfd_vidmem.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "kfd_priv.h"
+
+int radeon_kfd_vidmem_alloc(struct kfd_dev *kfd, size_t size, size_t alignment,
+ enum kfd_mempool pool, kfd_mem_obj *mem_obj)
+{
+ return kfd2kgd->allocate_mem(kfd->kgd,
+ size,
+ alignment,
+ (enum kgd_memory_pool)pool,
+ (struct kgd_mem **)mem_obj);
+}
+
+void radeon_kfd_vidmem_free(struct kfd_dev *kfd, kfd_mem_obj mem_obj)
+{
+ kfd2kgd->free_mem(kfd->kgd, (struct kgd_mem *)mem_obj);
+}
+
+int radeon_kfd_vidmem_gpumap(struct kfd_dev *kfd, kfd_mem_obj mem_obj,
+ uint64_t *vmid0_address)
+{
+ return kfd2kgd->gpumap_mem(kfd->kgd,
+ (struct kgd_mem *)mem_obj,
+ vmid0_address);
+}
+
+void radeon_kfd_vidmem_ungpumap(struct kfd_dev *kfd, kfd_mem_obj mem_obj)
+{
+ kfd2kgd->ungpumap_mem(kfd->kgd, (struct kgd_mem *)mem_obj);
+}
+
+int radeon_kfd_vidmem_kmap(struct kfd_dev *kfd, kfd_mem_obj mem_obj, void **ptr)
+{
+ return kfd2kgd->kmap_mem(kfd->kgd, (struct kgd_mem *)mem_obj, ptr);
+}
+
+void radeon_kfd_vidmem_unkmap(struct kfd_dev *kfd, kfd_mem_obj mem_obj)
+{
+ kfd2kgd->unkmap_mem(kfd->kgd, (struct kgd_mem *)mem_obj);
+}
--
1.9.1