[PATCH 1/4] cdx: add support for bus enable and disable
From: Abhijit Gangurde
Date: Tue Jul 11 2023 - 08:13:05 EST
CDX bus needs to be disabled before updating/writing devices
in the FPGA. Once the devices are written, the bus shall be
enabled. This change provides sysfs entry to enable/disable the
CDX bus.
Co-developed-by: Nipun Gupta <nipun.gupta@xxxxxxx>
Signed-off-by: Nipun Gupta <nipun.gupta@xxxxxxx>
Signed-off-by: Abhijit Gangurde <abhijit.gangurde@xxxxxxx>
Reviewed-by: Pieter Jansen van Vuuren <pieter.jansen-van-vuuren@xxxxxxx>
Tested-by: Nikhil Agarwal <nikhil.agarwal@xxxxxxx>
---
Documentation/ABI/testing/sysfs-bus-cdx | 11 +++++
drivers/cdx/cdx.c | 26 ++++++++++++
drivers/cdx/controller/cdx_controller.c | 27 +++++++++++++
drivers/cdx/controller/mc_cdx_pcol.h | 54 +++++++++++++++++++++++++
drivers/cdx/controller/mcdi_functions.c | 24 +++++++++++
drivers/cdx/controller/mcdi_functions.h | 16 ++++++++
include/linux/cdx/cdx_bus.h | 6 +++
7 files changed, 164 insertions(+)
diff --git a/Documentation/ABI/testing/sysfs-bus-cdx b/Documentation/ABI/testing/sysfs-bus-cdx
index 7af477f49998..0afa85b3c63b 100644
--- a/Documentation/ABI/testing/sysfs-bus-cdx
+++ b/Documentation/ABI/testing/sysfs-bus-cdx
@@ -11,6 +11,17 @@ Description:
# echo 1 > /sys/bus/cdx/rescan
+What: /sys/bus/cdx/enable
+Date: July 2023
+Contact: nipun.gupta@xxxxxxx
+Description:
+ Writing y/1/on to this file enables the CDX bus and
+ writing n/0/off disables the bus.
+
+ For example to disable CDX bus::
+
+ # echo 0 > /sys/bus/cdx/enable
+
What: /sys/bus/cdx/devices/.../vendor
Date: March 2023
Contact: nipun.gupta@xxxxxxx
diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c
index d2cad4c670a0..48c493a43491 100644
--- a/drivers/cdx/cdx.c
+++ b/drivers/cdx/cdx.c
@@ -380,6 +380,30 @@ static struct attribute *cdx_dev_attrs[] = {
};
ATTRIBUTE_GROUPS(cdx_dev);
+static ssize_t enable_store(const struct bus_type *bus,
+ const char *buf, size_t count)
+{
+ struct cdx_controller *cdx;
+ unsigned long index;
+ bool enable;
+ int ret;
+
+ if (kstrtobool(buf, &enable) < 0)
+ return -EINVAL;
+
+ xa_for_each(&cdx_controllers, index, cdx) {
+ if (cdx->enabled == enable)
+ continue;
+
+ ret = cdx->ops->enable(cdx, enable);
+ if (ret)
+ dev_err(cdx->dev, "cdx bus enable/disable failed\n");
+ }
+
+ return count;
+}
+static BUS_ATTR_WO(enable);
+
static ssize_t rescan_store(const struct bus_type *bus,
const char *buf, size_t count)
{
@@ -410,6 +434,7 @@ static ssize_t rescan_store(const struct bus_type *bus,
static BUS_ATTR_WO(rescan);
static struct attribute *cdx_bus_attrs[] = {
+ &bus_attr_enable.attr,
&bus_attr_rescan.attr,
NULL,
};
@@ -541,6 +566,7 @@ void cdx_unregister_controller(struct cdx_controller *cdx)
if (cdx->id >= MAX_CDX_CONTROLLERS)
return;
+ cdx->ops->enable(cdx, false);
device_for_each_child(cdx->dev, NULL, cdx_unregister_device);
xa_erase(&cdx_controllers, cdx->id);
}
diff --git a/drivers/cdx/controller/cdx_controller.c b/drivers/cdx/controller/cdx_controller.c
index dc52f95f8978..ac8081f23cbe 100644
--- a/drivers/cdx/controller/cdx_controller.c
+++ b/drivers/cdx/controller/cdx_controller.c
@@ -45,6 +45,21 @@ void cdx_rpmsg_pre_remove(struct cdx_controller *cdx)
cdx_mcdi_wait_for_quiescence(cdx->priv, MCDI_RPC_TIMEOUT);
}
+static int cdx_bus_enable(struct cdx_controller *cdx, bool enable)
+{
+ int ret;
+
+ if (enable)
+ ret = cdx_mcdi_bus_enable(cdx->priv);
+ else
+ ret = cdx_mcdi_bus_disable(cdx->priv);
+
+ if (!ret)
+ cdx->enabled = enable;
+
+ return ret;
+}
+
static int cdx_configure_device(struct cdx_controller *cdx,
u8 bus_num, u8 dev_num,
struct cdx_device_config *dev_config)
@@ -80,11 +95,22 @@ static int cdx_scan_devices(struct cdx_controller *cdx)
for (bus_num = 0; bus_num < num_cdx_bus; bus_num++) {
u8 num_cdx_dev;
+ ret = cdx_mcdi_bus_enable(cdx_mcdi);
+ if (ret && ret != -EALREADY) {
+ dev_err(cdx->dev,
+ "CDX bus %d enable failed: %d\n", bus_num, ret);
+ continue;
+ }
+
/* MCDI FW Read: Fetch the number of devices present */
ret = cdx_mcdi_get_num_devs(cdx_mcdi, bus_num);
if (ret < 0) {
dev_err(cdx->dev,
"Get devices on CDX bus %d failed: %d\n", bus_num, ret);
+ ret = cdx_mcdi_bus_disable(cdx_mcdi);
+ if (ret)
+ dev_err(cdx->dev,
+ "CDX bus %d disable failed: %d\n", bus_num, ret);
continue;
}
num_cdx_dev = (u8)ret;
@@ -120,6 +146,7 @@ static int cdx_scan_devices(struct cdx_controller *cdx)
}
static struct cdx_ops cdx_ops = {
+ .enable = cdx_bus_enable,
.scan = cdx_scan_devices,
.dev_configure = cdx_configure_device,
};
diff --git a/drivers/cdx/controller/mc_cdx_pcol.h b/drivers/cdx/controller/mc_cdx_pcol.h
index 4ccb7b52951b..2de019406b57 100644
--- a/drivers/cdx/controller/mc_cdx_pcol.h
+++ b/drivers/cdx/controller/mc_cdx_pcol.h
@@ -455,6 +455,60 @@
#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID_OFST 84
#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID_LEN 4
+/***********************************/
+/*
+ * MC_CMD_CDX_BUS_DOWN
+ * Asserting reset on the CDX bus causes all devices on the bus to be quiesced.
+ * DMA bus mastering is disabled and any pending DMA request are flushed. Once
+ * the response is returned, the devices are guaranteed to no longer issue DMA
+ * requests or raise MSI interrupts. Further device MMIO accesses may have
+ * undefined results. While the bus reset is asserted, any of the enumeration
+ * or device configuration MCDIs will fail with EAGAIN. It is only legal to
+ * reload the relevant PL region containing CDX devices if the corresponding CDX
+ * bus is in reset. Depending on the implementation, the firmware may or may
+ * not enforce this restriction and it is up to the caller to make sure this
+ * requirement is satisfied.
+ */
+#define MC_CMD_CDX_BUS_DOWN 0x4
+#define MC_CMD_CDX_BUS_DOWN_MSGSET 0x4
+
+/* MC_CMD_CDX_BUS_DOWN_IN msgrequest */
+#define MC_CMD_CDX_BUS_DOWN_IN_LEN 4
+/* Bus number to put in reset, in range 0 to BUS_COUNT-1 */
+#define MC_CMD_CDX_BUS_DOWN_IN_BUS_OFST 0
+#define MC_CMD_CDX_BUS_DOWN_IN_BUS_LEN 4
+
+/*
+ * MC_CMD_CDX_BUS_DOWN_OUT msgresponse: The bus is quiesced, no further
+ * upstream traffic for devices on this bus.
+ */
+#define MC_CMD_CDX_BUS_DOWN_OUT_LEN 0
+
+/***********************************/
+/*
+ * MC_CMD_CDX_BUS_UP
+ * After bus reset is de-asserted, devices are in a state which is functionally
+ * equivalent to each device having been reset with MC_CMD_CDX_DEVICE_RESET. In
+ * other words, device logic is reset in a hardware-specific way, MMIO accesses
+ * are forwarded to the device, DMA bus mastering is disabled and needs to be
+ * re-enabled with MC_CMD_CDX_DEVICE_DMA_ENABLE once the driver is ready to
+ * start servicing DMA. If the underlying number of devices or device resources
+ * changed (e.g. if PL was reloaded) while the bus was in reset, the bus driver
+ * is expected to re-enumerate the bus. Returns EALREADY if the bus was already
+ * up before the call.
+ */
+#define MC_CMD_CDX_BUS_UP 0x5
+#define MC_CMD_CDX_BUS_UP_MSGSET 0x5
+
+/* MC_CMD_CDX_BUS_UP_IN msgrequest */
+#define MC_CMD_CDX_BUS_UP_IN_LEN 4
+/* Bus number to take out of reset, in range 0 to BUS_COUNT-1 */
+#define MC_CMD_CDX_BUS_UP_IN_BUS_OFST 0
+#define MC_CMD_CDX_BUS_UP_IN_BUS_LEN 4
+
+/* MC_CMD_CDX_BUS_UP_OUT msgresponse: The bus can now be enumerated. */
+#define MC_CMD_CDX_BUS_UP_OUT_LEN 0
+
/***********************************/
/*
* MC_CMD_CDX_DEVICE_RESET
diff --git a/drivers/cdx/controller/mcdi_functions.c b/drivers/cdx/controller/mcdi_functions.c
index 0158f26533dd..400fdc771104 100644
--- a/drivers/cdx/controller/mcdi_functions.c
+++ b/drivers/cdx/controller/mcdi_functions.c
@@ -124,6 +124,30 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx,
return 0;
}
+int cdx_mcdi_bus_enable(struct cdx_mcdi *cdx)
+{
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_BUS_UP_IN_LEN);
+ int ret;
+
+ MCDI_SET_DWORD(inbuf, CDX_BUS_UP_IN_BUS, 0);
+ ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_BUS_UP, inbuf, sizeof(inbuf),
+ NULL, 0, NULL);
+
+ return ret;
+}
+
+int cdx_mcdi_bus_disable(struct cdx_mcdi *cdx)
+{
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_BUS_DOWN_IN_LEN);
+ int ret;
+
+ MCDI_SET_DWORD(inbuf, CDX_BUS_DOWN_IN_BUS, 0);
+ ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_BUS_DOWN, inbuf, sizeof(inbuf),
+ NULL, 0, NULL);
+
+ return ret;
+}
+
int cdx_mcdi_reset_device(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num)
{
MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_DEVICE_RESET_IN_LEN);
diff --git a/drivers/cdx/controller/mcdi_functions.h b/drivers/cdx/controller/mcdi_functions.h
index 7440ace5539a..6d26b7cdc07c 100644
--- a/drivers/cdx/controller/mcdi_functions.h
+++ b/drivers/cdx/controller/mcdi_functions.h
@@ -47,6 +47,22 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx,
u8 bus_num, u8 dev_num,
struct cdx_dev_params *dev_params);
+/**
+ * cdx_mcdi_bus_enable - Enable CDX bus represented by bus_num
+ * @cdx: pointer to MCDI interface.
+ *
+ * Return: 0 on success, <0 on failure
+ */
+int cdx_mcdi_bus_enable(struct cdx_mcdi *cdx);
+
+/**
+ * cdx_mcdi_bus_disable - Disable CDX bus represented by bus_num
+ * @cdx: pointer to MCDI interface.
+ *
+ * Return: 0 on success, <0 on failure
+ */
+int cdx_mcdi_bus_disable(struct cdx_mcdi *cdx);
+
/**
* cdx_mcdi_reset_device - Reset cdx device represented by bus_num:dev_num
* @cdx: pointer to MCDI interface.
diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
index bead71b7bc73..5da0634ae4ee 100644
--- a/include/linux/cdx/cdx_bus.h
+++ b/include/linux/cdx/cdx_bus.h
@@ -28,6 +28,8 @@ struct cdx_device_config {
u8 type;
};
+typedef int (*cdx_bus_enable_cb)(struct cdx_controller *cdx, bool enable);
+
typedef int (*cdx_scan_cb)(struct cdx_controller *cdx);
typedef int (*cdx_dev_configure_cb)(struct cdx_controller *cdx,
@@ -49,11 +51,13 @@ typedef int (*cdx_dev_configure_cb)(struct cdx_controller *cdx,
/**
* struct cdx_ops - Callbacks supported by CDX controller.
+ * @enable: enable or disable bus on the controller
* @scan: scan the devices on the controller
* @dev_configure: configuration like reset, master_enable,
* msi_config etc for a CDX device
*/
struct cdx_ops {
+ cdx_bus_enable_cb enable;
cdx_scan_cb scan;
cdx_dev_configure_cb dev_configure;
};
@@ -63,12 +67,14 @@ struct cdx_ops {
* @dev: Linux device associated with the CDX controller.
* @priv: private data
* @id: Controller ID
+ * @enabled: state enabled/disabled
* @ops: CDX controller ops
*/
struct cdx_controller {
struct device *dev;
void *priv;
u32 id;
+ bool enabled;
struct cdx_ops *ops;
};
--
2.25.1