Re: [RFC PATCH 4/4] PCI: endpoint: function: Add EP function driver to provide virtio net device

From: Michael S. Tsirkin
Date: Fri Feb 03 2023 - 05:23:46 EST

On Fri, Feb 03, 2023 at 07:04:18PM +0900, Shunsuke Mie wrote:
> Add a new endpoint(EP) function driver to provide virtio-net device. This
> function not only shows virtio-net device for PCIe host system, but also
> provides virtio-net device to EP side(local) system. Virtualy those network
> devices are connected, so we can use to communicate over IP like a simple
> NIC.
>
> Architecture overview is following:
>
> to Host | to Endpoint
> network stack | network stack
> | | |
> +-----------+ | +-----------+ +-----------+
> |virtio-net | | |virtio-net | |virtio-net |
> |driver | | |EP function|---|driver |
> +-----------+ | +-----------+ +-----------+
> | | |
> +-----------+ | +-----------+
> |PCIeC | | |PCIeC |
> |Rootcomplex|-|-|Endpoint |
> +-----------+ | +-----------+
> Host side | Endpoint side
>
> This driver uses PCIe EP framework to show virtio-net (pci) device Host
> side, and generate virtual virtio-net device and register to EP side.
> A communication date

data?

> is diractly

directly?

> transported between virtqueue level
> with each other using PCIe embedded DMA controller.
>
> by a limitation of the hardware and Linux EP framework, this function
> follows a virtio legacy specification.

what exactly is the limitation and why does it force legacy?

> This function driver has beed tested on S4 Rcar (r8a779fa-spider) board but
> just use the PCIe EP framework and depends on the PCIe EDMA.
>
> Signed-off-by: Shunsuke Mie <mie@xxxxxxxxxx>
> Signed-off-by: Takanari Hayama <taki@xxxxxxxxxx>
> ---
> drivers/pci/endpoint/functions/Kconfig | 12 +
> drivers/pci/endpoint/functions/Makefile | 1 +
> .../pci/endpoint/functions/pci-epf-vnet-ep.c | 343 ++++++++++
> .../pci/endpoint/functions/pci-epf-vnet-rc.c | 635 ++++++++++++++++++
> drivers/pci/endpoint/functions/pci-epf-vnet.c | 387 +++++++++++
> drivers/pci/endpoint/functions/pci-epf-vnet.h | 62 ++
> 6 files changed, 1440 insertions(+)
> create mode 100644 drivers/pci/endpoint/functions/pci-epf-vnet-ep.c
> create mode 100644 drivers/pci/endpoint/functions/pci-epf-vnet-rc.c
> create mode 100644 drivers/pci/endpoint/functions/pci-epf-vnet.c
> create mode 100644 drivers/pci/endpoint/functions/pci-epf-vnet.h
>
> diff --git a/drivers/pci/endpoint/functions/Kconfig b/drivers/pci/endpoint/functions/Kconfig
> index 9fd560886871..f88d8baaf689 100644
> --- a/drivers/pci/endpoint/functions/Kconfig
> +++ b/drivers/pci/endpoint/functions/Kconfig
> @@ -37,3 +37,15 @@ config PCI_EPF_VNTB
> between PCI Root Port and PCIe Endpoint.
>
> If in doubt, say "N" to disable Endpoint NTB driver.
> +
> +config PCI_EPF_VNET
> + tristate "PCI Endpoint virtio-net driver"
> + depends on PCI_ENDPOINT
> + select PCI_ENDPOINT_VIRTIO
> + select VHOST_RING
> + select VHOST_IOMEM
> + help
> + PCIe Endpoint virtio-net function implementation. This module enables to
> + show the virtio-net as pci device to PCIe Host side, and, another
> + virtio-net device show to local machine. Those devices can communicate
> + each other.
> diff --git a/drivers/pci/endpoint/functions/Makefile b/drivers/pci/endpoint/functions/Makefile
> index 5c13001deaba..74cc4c330c62 100644
> --- a/drivers/pci/endpoint/functions/Makefile
> +++ b/drivers/pci/endpoint/functions/Makefile
> @@ -6,3 +6,4 @@
> obj-$(CONFIG_PCI_EPF_TEST) += pci-epf-test.o
> obj-$(CONFIG_PCI_EPF_NTB) += pci-epf-ntb.o
> obj-$(CONFIG_PCI_EPF_VNTB) += pci-epf-vntb.o
> +obj-$(CONFIG_PCI_EPF_VNET) += pci-epf-vnet.o pci-epf-vnet-rc.o pci-epf-vnet-ep.o
> diff --git a/drivers/pci/endpoint/functions/pci-epf-vnet-ep.c b/drivers/pci/endpoint/functions/pci-epf-vnet-ep.c
> new file mode 100644
> index 000000000000..93b7e00e8d06
> --- /dev/null
> +++ b/drivers/pci/endpoint/functions/pci-epf-vnet-ep.c
> @@ -0,0 +1,343 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Functions work for Endpoint side(local) using EPF framework
> + */
> +#include <linux/pci-epc.h>
> +#include <linux/virtio_pci.h>
> +#include <linux/virtio_net.h>
> +#include <linux/virtio_ring.h>
> +
> +#include "pci-epf-vnet.h"
> +
> +static inline struct epf_vnet *vdev_to_vnet(struct virtio_device *vdev)
> +{
> + return container_of(vdev, struct epf_vnet, ep.vdev);
> +}
> +
> +static void epf_vnet_ep_set_status(struct epf_vnet *vnet, u16 status)
> +{
> + vnet->ep.net_config_status |= status;
> +}
> +
> +static void epf_vnet_ep_clear_status(struct epf_vnet *vnet, u16 status)
> +{
> + vnet->ep.net_config_status &= ~status;
> +}
> +
> +static void epf_vnet_ep_raise_config_irq(struct epf_vnet *vnet)
> +{
> + virtio_config_changed(&vnet->ep.vdev);
> +}
> +
> +void epf_vnet_ep_announce_linkup(struct epf_vnet *vnet)
> +{
> + epf_vnet_ep_set_status(vnet,
> + VIRTIO_NET_S_LINK_UP | VIRTIO_NET_S_ANNOUNCE);
> + epf_vnet_ep_raise_config_irq(vnet);
> +}
> +
> +void epf_vnet_ep_notify(struct epf_vnet *vnet, struct virtqueue *vq)
> +{
> + vring_interrupt(0, vq);
> +}
> +
> +static int epf_vnet_ep_process_ctrlq_entry(struct epf_vnet *vnet)
> +{
> + struct vringh *vrh = &vnet->ep.ctlvrh;
> + struct vringh_kiov *wiov = &vnet->ep.ctl_riov;
> + struct vringh_kiov *riov = &vnet->ep.ctl_wiov;
> + struct virtio_net_ctrl_hdr *hdr;
> + virtio_net_ctrl_ack *ack;
> + int err;
> + u16 head;
> + size_t len;
> +
> + err = vringh_getdesc(vrh, riov, wiov, &head);
> + if (err <= 0)
> + goto done;
> +
> + len = vringh_kiov_length(riov);
> + if (len < sizeof(*hdr)) {
> + pr_debug("Command is too short: %ld\n", len);
> + err = -EIO;
> + goto done;
> + }
> +
> + if (vringh_kiov_length(wiov) < sizeof(*ack)) {
> + pr_debug("Space for ack is not enough\n");
> + err = -EIO;
> + goto done;
> + }
> +
> + hdr = phys_to_virt((unsigned long)riov->iov[riov->i].iov_base);
> + ack = phys_to_virt((unsigned long)wiov->iov[wiov->i].iov_base);
> +
> + switch (hdr->class) {
> + case VIRTIO_NET_CTRL_ANNOUNCE:
> + if (hdr->cmd != VIRTIO_NET_CTRL_ANNOUNCE_ACK) {
> + pr_debug("Invalid command: announce: %d\n", hdr->cmd);
> + goto done;
> + }
> +
> + epf_vnet_ep_clear_status(vnet, VIRTIO_NET_S_ANNOUNCE);
> + *ack = VIRTIO_NET_OK;
> + break;
> + default:
> + pr_debug("Found not supported class: %d\n", hdr->class);
> + err = -EIO;
> + }
> +
> +done:
> + vringh_complete(vrh, head, len);
> + return err;
> +}
> +
> +static u64 epf_vnet_ep_vdev_get_features(struct virtio_device *vdev)
> +{
> + struct epf_vnet *vnet = vdev_to_vnet(vdev);
> +
> + return vnet->virtio_features;
> +}
> +
> +static int epf_vnet_ep_vdev_finalize_features(struct virtio_device *vdev)
> +{
> + struct epf_vnet *vnet = vdev_to_vnet(vdev);
> +
> + if (vdev->features != vnet->virtio_features)
> + return -EINVAL;
> +
> + return 0;
> +}
> +
> +static void epf_vnet_ep_vdev_get_config(struct virtio_device *vdev,
> + unsigned int offset, void *buf,
> + unsigned int len)
> +{
> + struct epf_vnet *vnet = vdev_to_vnet(vdev);
> + const unsigned int mac_len = sizeof(vnet->vnet_cfg.mac);
> + const unsigned int status_len = sizeof(vnet->vnet_cfg.status);
> + unsigned int copy_len;
> +
> + switch (offset) {
> + case offsetof(struct virtio_net_config, mac):
> + /* This PCIe EP function doesn't provide a VIRTIO_NET_F_MAC feature, so just
> + * clear the buffer.
> + */
> + copy_len = len >= mac_len ? mac_len : len;
> + memset(buf, 0x00, copy_len);
> + len -= copy_len;
> + buf += copy_len;
> + fallthrough;
> + case offsetof(struct virtio_net_config, status):
> + copy_len = len >= status_len ? status_len : len;
> + memcpy(buf, &vnet->ep.net_config_status, copy_len);
> + len -= copy_len;
> + buf += copy_len;
> + fallthrough;
> + default:
> + if (offset > sizeof(vnet->vnet_cfg)) {
> + memset(buf, 0x00, len);
> + break;
> + }
> + memcpy(buf, (void *)&vnet->vnet_cfg + offset, len);
> + }
> +}
> +
> +static void epf_vnet_ep_vdev_set_config(struct virtio_device *vdev,
> + unsigned int offset, const void *buf,
> + unsigned int len)
> +{
> + /* Do nothing, because all of virtio net config space is readonly. */
> +}
> +
> +static u8 epf_vnet_ep_vdev_get_status(struct virtio_device *vdev)
> +{
> + return 0;
> +}
> +
> +static void epf_vnet_ep_vdev_set_status(struct virtio_device *vdev, u8 status)
> +{
> + struct epf_vnet *vnet = vdev_to_vnet(vdev);
> +
> + if (status & VIRTIO_CONFIG_S_DRIVER_OK)
> + epf_vnet_init_complete(vnet, EPF_VNET_INIT_COMPLETE_EP);
> +}
> +
> +static void epf_vnet_ep_vdev_reset(struct virtio_device *vdev)
> +{
> + pr_debug("doesn't support yet");
> +}
> +
> +static bool epf_vnet_ep_vdev_vq_notify(struct virtqueue *vq)
> +{
> + struct epf_vnet *vnet = vdev_to_vnet(vq->vdev);
> + struct vringh *tx_vrh = &vnet->ep.txvrh;
> + struct vringh *rx_vrh = &vnet->rc.rxvrh->vrh;
> + struct vringh_kiov *tx_iov = &vnet->ep.tx_iov;
> + struct vringh_kiov *rx_iov = &vnet->rc.rx_iov;
> + int err;
> +
> + /* Support only one queue pair */
> + switch (vq->index) {
> + case 0: // rx queue
> + break;
> + case 1: // tx queue
> + while ((err = epf_vnet_transfer(vnet, tx_vrh, rx_vrh, tx_iov,
> + rx_iov, DMA_MEM_TO_DEV)) > 0)
> + ;
> + if (err < 0)
> + pr_debug("Failed to transmit: EP -> Host: %d\n", err);
> + break;
> + case 2: // control queue
> + epf_vnet_ep_process_ctrlq_entry(vnet);
> + break;
> + default:
> + return false;
> + }
> +
> + return true;
> +}
> +
> +static int epf_vnet_ep_vdev_find_vqs(struct virtio_device *vdev,
> + unsigned int nvqs, struct virtqueue *vqs[],
> + vq_callback_t *callback[],
> + const char *const names[], const bool *ctx,
> + struct irq_affinity *desc)
> +{
> + struct epf_vnet *vnet = vdev_to_vnet(vdev);
> + const size_t vq_size = epf_vnet_get_vq_size();
> + int i;
> + int err;
> + int qidx;
> +
> + for (qidx = 0, i = 0; i < nvqs; i++) {
> + struct virtqueue *vq;
> + struct vring *vring;
> + struct vringh *vrh;
> +
> + if (!names[i]) {
> + vqs[i] = NULL;
> + continue;
> + }
> +
> + vq = vring_create_virtqueue(qidx++, vq_size,
> + VIRTIO_PCI_VRING_ALIGN, vdev, true,
> + false, ctx ? ctx[i] : false,
> + epf_vnet_ep_vdev_vq_notify,
> + callback[i], names[i]);
> + if (!vq) {
> + err = -ENOMEM;
> + goto err_del_vqs;
> + }
> +
> + vqs[i] = vq;
> + vring = virtqueue_get_vring(vq);
> +
> + switch (i) {
> + case 0: // rx
> + vrh = &vnet->ep.rxvrh;
> + vnet->ep.rxvq = vq;
> + break;
> + case 1: // tx
> + vrh = &vnet->ep.txvrh;
> + vnet->ep.txvq = vq;
> + break;
> + case 2: // control
> + vrh = &vnet->ep.ctlvrh;
> + vnet->ep.ctlvq = vq;
> + break;
> + default:
> + err = -EIO;
> + goto err_del_vqs;
> + }
> +
> + err = vringh_init_kern(vrh, vnet->virtio_features, vq_size,
> + true, GFP_KERNEL, vring->desc,
> + vring->avail, vring->used);
> + if (err) {
> + pr_err("failed to init vringh for vring %d\n", i);
> + goto err_del_vqs;
> + }
> + }
> +
> + err = epf_vnet_init_kiov(&vnet->ep.tx_iov, vq_size);
> + if (err)
> + goto err_free_kiov;
> + err = epf_vnet_init_kiov(&vnet->ep.rx_iov, vq_size);
> + if (err)
> + goto err_free_kiov;
> + err = epf_vnet_init_kiov(&vnet->ep.ctl_riov, vq_size);
> + if (err)
> + goto err_free_kiov;
> + err = epf_vnet_init_kiov(&vnet->ep.ctl_wiov, vq_size);
> + if (err)
> + goto err_free_kiov;
> +
> + return 0;
> +
> +err_free_kiov:
> + epf_vnet_deinit_kiov(&vnet->ep.tx_iov);
> + epf_vnet_deinit_kiov(&vnet->ep.rx_iov);
> + epf_vnet_deinit_kiov(&vnet->ep.ctl_riov);
> + epf_vnet_deinit_kiov(&vnet->ep.ctl_wiov);
> +
> +err_del_vqs:
> + for (; i >= 0; i--) {
> + if (!names[i])
> + continue;
> +
> + if (!vqs[i])
> + continue;
> +
> + vring_del_virtqueue(vqs[i]);
> + }
> + return err;
> +}
> +
> +static void epf_vnet_ep_vdev_del_vqs(struct virtio_device *vdev)
> +{
> + struct virtqueue *vq, *n;
> + struct epf_vnet *vnet = vdev_to_vnet(vdev);
> +
> + list_for_each_entry_safe(vq, n, &vdev->vqs, list)
> + vring_del_virtqueue(vq);
> +
> + epf_vnet_deinit_kiov(&vnet->ep.tx_iov);
> + epf_vnet_deinit_kiov(&vnet->ep.rx_iov);
> + epf_vnet_deinit_kiov(&vnet->ep.ctl_riov);
> + epf_vnet_deinit_kiov(&vnet->ep.ctl_wiov);
> +}
> +
> +static const struct virtio_config_ops epf_vnet_ep_vdev_config_ops = {
> + .get_features = epf_vnet_ep_vdev_get_features,
> + .finalize_features = epf_vnet_ep_vdev_finalize_features,
> + .get = epf_vnet_ep_vdev_get_config,
> + .set = epf_vnet_ep_vdev_set_config,
> + .get_status = epf_vnet_ep_vdev_get_status,
> + .set_status = epf_vnet_ep_vdev_set_status,
> + .reset = epf_vnet_ep_vdev_reset,
> + .find_vqs = epf_vnet_ep_vdev_find_vqs,
> + .del_vqs = epf_vnet_ep_vdev_del_vqs,
> +};
> +
> +void epf_vnet_ep_cleanup(struct epf_vnet *vnet)
> +{
> + unregister_virtio_device(&vnet->ep.vdev);
> +}
> +
> +int epf_vnet_ep_setup(struct epf_vnet *vnet)
> +{
> + int err;
> + struct virtio_device *vdev = &vnet->ep.vdev;
> +
> + vdev->dev.parent = vnet->epf->epc->dev.parent;
> + vdev->config = &epf_vnet_ep_vdev_config_ops;
> + vdev->id.vendor = PCI_VENDOR_ID_REDHAT_QUMRANET;
> + vdev->id.device = VIRTIO_ID_NET;
> +
> + err = register_virtio_device(vdev);
> + if (err)
> + return err;
> +
> + return 0;
> +}
> diff --git a/drivers/pci/endpoint/functions/pci-epf-vnet-rc.c b/drivers/pci/endpoint/functions/pci-epf-vnet-rc.c
> new file mode 100644
> index 000000000000..2ca0245a9134
> --- /dev/null
> +++ b/drivers/pci/endpoint/functions/pci-epf-vnet-rc.c
> @@ -0,0 +1,635 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Functions work for PCie Host side(remote) using EPF framework.
> + */
> +#include <linux/pci-epf.h>
> +#include <linux/pci-epc.h>
> +#include <linux/pci_ids.h>
> +#include <linux/sched.h>
> +#include <linux/virtio_pci.h>
> +
> +#include "pci-epf-vnet.h"
> +
> +#define VIRTIO_NET_LEGACY_CFG_BAR BAR_0
> +
> +/* Returns an out side of the valid queue index. */
> +static inline u16 epf_vnet_rc_get_number_of_queues(struct epf_vnet *vnet)
> +
> +{
> + /* number of queue pairs and control queue */
> + return vnet->vnet_cfg.max_virtqueue_pairs * 2 + 1;
> +}
> +
> +static void epf_vnet_rc_memcpy_config(struct epf_vnet *vnet, size_t offset,
> + void *buf, size_t len)
> +{
> + void __iomem *base = vnet->rc.cfg_base + offset;
> +
> + memcpy_toio(base, buf, len);
> +}
> +
> +static void epf_vnet_rc_set_config8(struct epf_vnet *vnet, size_t offset,
> + u8 config)
> +{
> + void __iomem *base = vnet->rc.cfg_base + offset;
> +
> + iowrite8(ioread8(base) | config, base);
> +}
> +
> +static void epf_vnet_rc_set_config16(struct epf_vnet *vnet, size_t offset,
> + u16 config)
> +{
> + void __iomem *base = vnet->rc.cfg_base + offset;
> +
> + iowrite16(ioread16(base) | config, base);
> +}
> +
> +static void epf_vnet_rc_clear_config16(struct epf_vnet *vnet, size_t offset,
> + u16 config)
> +{
> + void __iomem *base = vnet->rc.cfg_base + offset;
> +
> + iowrite16(ioread16(base) & ~config, base);
> +}
> +
> +static void epf_vnet_rc_set_config32(struct epf_vnet *vnet, size_t offset,
> + u32 config)
> +{
> + void __iomem *base = vnet->rc.cfg_base + offset;
> +
> + iowrite32(ioread32(base) | config, base);
> +}
> +
> +static void epf_vnet_rc_raise_config_irq(struct epf_vnet *vnet)
> +{
> + epf_vnet_rc_set_config16(vnet, VIRTIO_PCI_ISR, VIRTIO_PCI_ISR_CONFIG);
> + queue_work(vnet->rc.irq_wq, &vnet->rc.raise_irq_work);
> +}
> +
> +void epf_vnet_rc_announce_linkup(struct epf_vnet *vnet)
> +{
> + epf_vnet_rc_set_config16(vnet,
> + VIRTIO_PCI_CONFIG_OFF(false) +
> + offsetof(struct virtio_net_config,
> + status),
> + VIRTIO_NET_S_LINK_UP | VIRTIO_NET_S_ANNOUNCE);
> + epf_vnet_rc_raise_config_irq(vnet);
> +}
> +
> +/*
> + * For the PCIe host, this driver shows legacy virtio-net device. Because,
> + * virtio structure pci capabilities is mandatory for modern virtio device,
> + * but there is no PCIe EP hardware that can be configured with any pci
> + * capabilities and Linux PCIe EP framework doesn't support it.
> + */
> +static struct pci_epf_header epf_vnet_pci_header = {
> + .vendorid = PCI_VENDOR_ID_REDHAT_QUMRANET,
> + .deviceid = VIRTIO_TRANS_ID_NET,
> + .subsys_vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET,
> + .subsys_id = VIRTIO_ID_NET,
> + .revid = 0,
> + .baseclass_code = PCI_BASE_CLASS_NETWORK,
> + .interrupt_pin = PCI_INTERRUPT_PIN,
> +};
> +
> +static void epf_vnet_rc_setup_configs(struct epf_vnet *vnet,
> + void __iomem *cfg_base)
> +{
> + u16 default_qindex = epf_vnet_rc_get_number_of_queues(vnet);
> +
> + epf_vnet_rc_set_config32(vnet, VIRTIO_PCI_HOST_FEATURES,
> + vnet->virtio_features);
> +
> + epf_vnet_rc_set_config16(vnet, VIRTIO_PCI_ISR, VIRTIO_PCI_ISR_QUEUE);
> + /*
> + * Initialize the queue notify and selector to outside of the appropriate
> + * virtqueue index. It is used to detect change with polling. There is no
> + * other ways to detect host side driver updateing those values
> + */
> + epf_vnet_rc_set_config16(vnet, VIRTIO_PCI_QUEUE_NOTIFY, default_qindex);
> + epf_vnet_rc_set_config16(vnet, VIRTIO_PCI_QUEUE_SEL, default_qindex);
> + /* This pfn is also set to 0 for the polling as well */
> + epf_vnet_rc_set_config16(vnet, VIRTIO_PCI_QUEUE_PFN, 0);
> +
> + epf_vnet_rc_set_config16(vnet, VIRTIO_PCI_QUEUE_NUM,
> + epf_vnet_get_vq_size());
> + epf_vnet_rc_set_config8(vnet, VIRTIO_PCI_STATUS, 0);
> + epf_vnet_rc_memcpy_config(vnet, VIRTIO_PCI_CONFIG_OFF(false),
> + &vnet->vnet_cfg, sizeof(vnet->vnet_cfg));
> +}
> +
> +static void epf_vnet_cleanup_bar(struct epf_vnet *vnet)
> +{
> + struct pci_epf *epf = vnet->epf;
> +
> + pci_epc_clear_bar(epf->epc, epf->func_no, epf->vfunc_no,
> + &epf->bar[VIRTIO_NET_LEGACY_CFG_BAR]);
> + pci_epf_free_space(epf, vnet->rc.cfg_base, VIRTIO_NET_LEGACY_CFG_BAR,
> + PRIMARY_INTERFACE);
> +}
> +
> +static int epf_vnet_setup_bar(struct epf_vnet *vnet)
> +{
> + int err;
> + size_t cfg_bar_size =
> + VIRTIO_PCI_CONFIG_OFF(false) + sizeof(struct virtio_net_config);
> + struct pci_epf *epf = vnet->epf;
> + const struct pci_epc_features *features;
> + struct pci_epf_bar *config_bar = &epf->bar[VIRTIO_NET_LEGACY_CFG_BAR];
> +
> + features = pci_epc_get_features(epf->epc, epf->func_no, epf->vfunc_no);
> + if (!features) {
> + pr_debug("Failed to get PCI EPC features\n");
> + return -EOPNOTSUPP;
> + }
> +
> + if (features->reserved_bar & BIT(VIRTIO_NET_LEGACY_CFG_BAR)) {
> + pr_debug("Cannot use the PCI BAR for legacy virtio pci\n");
> + return -EOPNOTSUPP;
> + }
> +
> + if (features->bar_fixed_size[VIRTIO_NET_LEGACY_CFG_BAR]) {
> + if (cfg_bar_size >
> + features->bar_fixed_size[VIRTIO_NET_LEGACY_CFG_BAR]) {
> + pr_debug("PCI BAR size is not enough\n");
> + return -ENOMEM;
> + }
> + }
> +
> + config_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
> +
> + vnet->rc.cfg_base = pci_epf_alloc_space(epf, cfg_bar_size,
> + VIRTIO_NET_LEGACY_CFG_BAR,
> + features->align,
> + PRIMARY_INTERFACE);
> + if (!vnet->rc.cfg_base) {
> + pr_debug("Failed to allocate virtio-net config memory\n");
> + return -ENOMEM;
> + }
> +
> + epf_vnet_rc_setup_configs(vnet, vnet->rc.cfg_base);
> +
> + err = pci_epc_set_bar(epf->epc, epf->func_no, epf->vfunc_no,
> + config_bar);
> + if (err) {
> + pr_debug("Failed to set PCI BAR");
> + goto err_free_space;
> + }
> +
> + return 0;
> +
> +err_free_space:
> + pci_epf_free_space(epf, vnet->rc.cfg_base, VIRTIO_NET_LEGACY_CFG_BAR,
> + PRIMARY_INTERFACE);
> + return err;
> +}
> +
> +static int epf_vnet_rc_negotiate_configs(struct epf_vnet *vnet, u32 *txpfn,
> + u32 *rxpfn, u32 *ctlpfn)
> +{
> + const u16 nqueues = epf_vnet_rc_get_number_of_queues(vnet);
> + const u16 default_sel = nqueues;
> + u32 __iomem *queue_pfn = vnet->rc.cfg_base + VIRTIO_PCI_QUEUE_PFN;
> + u16 __iomem *queue_sel = vnet->rc.cfg_base + VIRTIO_PCI_QUEUE_SEL;
> + u8 __iomem *pci_status = vnet->rc.cfg_base + VIRTIO_PCI_STATUS;
> + u32 pfn;
> + u16 sel;
> + struct {
> + u32 pfn;
> + u16 sel;
> + } tmp[3] = {};
> + int tmp_index = 0;
> +
> + *rxpfn = *txpfn = *ctlpfn = 0;
> +
> + /* To avoid to miss a getting the pfn and selector for virtqueue wrote by
> + * host driver, we need to implement fast polling with saving.
> + *
> + * This implementation suspects that the host driver writes pfn only once
> + * for each queues
> + */
> + while (tmp_index < nqueues) {
> + pfn = ioread32(queue_pfn);
> + if (pfn == 0)
> + continue;
> +
> + iowrite32(0, queue_pfn);
> +
> + sel = ioread16(queue_sel);
> + if (sel == default_sel)
> + continue;
> +
> + tmp[tmp_index].pfn = pfn;
> + tmp[tmp_index].sel = sel;
> + tmp_index++;
> + }
> +
> + while (!((ioread8(pci_status) & VIRTIO_CONFIG_S_DRIVER_OK)))
> + ;
> +
> + for (int i = 0; i < nqueues; ++i) {
> + switch (tmp[i].sel) {
> + case 0:
> + *rxpfn = tmp[i].pfn;
> + break;
> + case 1:
> + *txpfn = tmp[i].pfn;
> + break;
> + case 2:
> + *ctlpfn = tmp[i].pfn;
> + break;
> + }
> + }
> +
> + if (!*rxpfn || !*txpfn || !*ctlpfn)
> + return -EIO;
> +
> + return 0;
> +}
> +
> +static int epf_vnet_rc_monitor_notify(void *data)
> +{
> + struct epf_vnet *vnet = data;
> + u16 __iomem *queue_notify = vnet->rc.cfg_base + VIRTIO_PCI_QUEUE_NOTIFY;
> + const u16 notify_default = epf_vnet_rc_get_number_of_queues(vnet);
> +
> + epf_vnet_init_complete(vnet, EPF_VNET_INIT_COMPLETE_RC);
> +
> + /* Poll to detect a change of the queue_notify register. Sometimes this
> + * polling misses the change, so try to check each virtqueues
> + * everytime.
> + */
> + while (true) {
> + while (ioread16(queue_notify) == notify_default)
> + ;
> + iowrite16(notify_default, queue_notify);
> +
> + queue_work(vnet->rc.tx_wq, &vnet->rc.tx_work);
> + queue_work(vnet->rc.ctl_wq, &vnet->rc.ctl_work);
> + }
> +
> + return 0;
> +}
> +
> +static int epf_vnet_rc_spawn_notify_monitor(struct epf_vnet *vnet)
> +{
> + vnet->rc.notify_monitor_task =
> + kthread_create(epf_vnet_rc_monitor_notify, vnet,
> + "pci-epf-vnet/cfg_negotiator");
> + if (IS_ERR(vnet->rc.notify_monitor_task))
> + return PTR_ERR(vnet->rc.notify_monitor_task);
> +
> + /* Change the thread priority to high for polling. */
> + sched_set_fifo(vnet->rc.notify_monitor_task);
> + wake_up_process(vnet->rc.notify_monitor_task);
> +
> + return 0;
> +}
> +
> +static int epf_vnet_rc_device_setup(void *data)
> +{
> + struct epf_vnet *vnet = data;
> + struct pci_epf *epf = vnet->epf;
> + u32 txpfn, rxpfn, ctlpfn;
> + const size_t vq_size = epf_vnet_get_vq_size();
> + int err;
> +
> + err = epf_vnet_rc_negotiate_configs(vnet, &txpfn, &rxpfn, &ctlpfn);
> + if (err) {
> + pr_debug("Failed to negatiate configs with driver\n");
> + return err;
> + }
> +
> + /* Polling phase is finished. This thread backs to normal priority. */
> + sched_set_normal(vnet->rc.device_setup_task, 19);
> +
> + vnet->rc.txvrh = pci_epf_virtio_alloc_vringh(epf, vnet->virtio_features,
> + txpfn, vq_size);
> + if (IS_ERR(vnet->rc.txvrh)) {
> + pr_debug("Failed to setup virtqueue for tx\n");
> + return PTR_ERR(vnet->rc.txvrh);
> + }
> +
> + err = epf_vnet_init_kiov(&vnet->rc.tx_iov, vq_size);
> + if (err)
> + goto err_free_epf_tx_vringh;
> +
> + vnet->rc.rxvrh = pci_epf_virtio_alloc_vringh(epf, vnet->virtio_features,
> + rxpfn, vq_size);
> + if (IS_ERR(vnet->rc.rxvrh)) {
> + pr_debug("Failed to setup virtqueue for rx\n");
> + err = PTR_ERR(vnet->rc.rxvrh);
> + goto err_deinit_tx_kiov;
> + }
> +
> + err = epf_vnet_init_kiov(&vnet->rc.rx_iov, vq_size);
> + if (err)
> + goto err_free_epf_rx_vringh;
> +
> + vnet->rc.ctlvrh = pci_epf_virtio_alloc_vringh(
> + epf, vnet->virtio_features, ctlpfn, vq_size);
> + if (IS_ERR(vnet->rc.ctlvrh)) {
> + pr_err("failed to setup virtqueue\n");
> + err = PTR_ERR(vnet->rc.ctlvrh);
> + goto err_deinit_rx_kiov;
> + }
> +
> + err = epf_vnet_init_kiov(&vnet->rc.ctl_riov, vq_size);
> + if (err)
> + goto err_free_epf_ctl_vringh;
> +
> + err = epf_vnet_init_kiov(&vnet->rc.ctl_wiov, vq_size);
> + if (err)
> + goto err_deinit_ctl_riov;
> +
> + err = epf_vnet_rc_spawn_notify_monitor(vnet);
> + if (err) {
> + pr_debug("Failed to create notify monitor thread\n");
> + goto err_deinit_ctl_wiov;
> + }
> +
> + return 0;
> +
> +err_deinit_ctl_wiov:
> + epf_vnet_deinit_kiov(&vnet->rc.ctl_wiov);
> +err_deinit_ctl_riov:
> + epf_vnet_deinit_kiov(&vnet->rc.ctl_riov);
> +err_free_epf_ctl_vringh:
> + pci_epf_virtio_free_vringh(epf, vnet->rc.ctlvrh);
> +err_deinit_rx_kiov:
> + epf_vnet_deinit_kiov(&vnet->rc.rx_iov);
> +err_free_epf_rx_vringh:
> + pci_epf_virtio_free_vringh(epf, vnet->rc.rxvrh);
> +err_deinit_tx_kiov:
> + epf_vnet_deinit_kiov(&vnet->rc.tx_iov);
> +err_free_epf_tx_vringh:
> + pci_epf_virtio_free_vringh(epf, vnet->rc.txvrh);
> +
> + return err;
> +}
> +
> +static int epf_vnet_rc_spawn_device_setup_task(struct epf_vnet *vnet)
> +{
> + vnet->rc.device_setup_task = kthread_create(
> + epf_vnet_rc_device_setup, vnet, "pci-epf-vnet/cfg_negotiator");
> + if (IS_ERR(vnet->rc.device_setup_task))
> + return PTR_ERR(vnet->rc.device_setup_task);
> +
> + /* Change the thread priority to high for the polling. */
> + sched_set_fifo(vnet->rc.device_setup_task);
> + wake_up_process(vnet->rc.device_setup_task);
> +
> + return 0;
> +}
> +
> +static void epf_vnet_rc_tx_handler(struct work_struct *work)
> +{
> + struct epf_vnet *vnet = container_of(work, struct epf_vnet, rc.tx_work);
> + struct vringh *tx_vrh = &vnet->rc.txvrh->vrh;
> + struct vringh *rx_vrh = &vnet->ep.rxvrh;
> + struct vringh_kiov *tx_iov = &vnet->rc.tx_iov;
> + struct vringh_kiov *rx_iov = &vnet->ep.rx_iov;
> +
> + while (epf_vnet_transfer(vnet, tx_vrh, rx_vrh, tx_iov, rx_iov,
> + DMA_DEV_TO_MEM) > 0)
> + ;
> +}
> +
> +static void epf_vnet_rc_raise_irq_handler(struct work_struct *work)
> +{
> + struct epf_vnet *vnet =
> + container_of(work, struct epf_vnet, rc.raise_irq_work);
> + struct pci_epf *epf = vnet->epf;
> +
> + pci_epc_raise_irq(epf->epc, epf->func_no, epf->vfunc_no,
> + PCI_EPC_IRQ_LEGACY, 0);
> +}
> +
> +struct epf_vnet_rc_meminfo {
> + void __iomem *addr, *virt;
> + phys_addr_t phys;
> + size_t len;
> +};
> +
> +/* Util function to access PCIe host side memory from local CPU. */
> +static struct epf_vnet_rc_meminfo *
> +epf_vnet_rc_epc_mmap(struct pci_epf *epf, phys_addr_t pci_addr, size_t len)
> +{
> + int err;
> + phys_addr_t aaddr, phys_addr;
> + size_t asize, offset;
> + void __iomem *virt_addr;
> + struct epf_vnet_rc_meminfo *meminfo;
> +
> + err = pci_epc_mem_align(epf->epc, pci_addr, len, &aaddr, &asize);
> + if (err) {
> + pr_debug("Failed to get EPC align: %d\n", err);
> + return NULL;
> + }
> +
> + offset = pci_addr - aaddr;
> +
> + virt_addr = pci_epc_mem_alloc_addr(epf->epc, &phys_addr, asize);
> + if (!virt_addr) {
> + pr_debug("Failed to allocate epc memory\n");
> + return NULL;
> + }
> +
> + err = pci_epc_map_addr(epf->epc, epf->func_no, epf->vfunc_no, phys_addr,
> + aaddr, asize);
> + if (err) {
> + pr_debug("Failed to map epc memory\n");
> + goto err_epc_free_addr;
> + }
> +
> + meminfo = kmalloc(sizeof(*meminfo), GFP_KERNEL);
> + if (!meminfo)
> + goto err_epc_unmap_addr;
> +
> + meminfo->virt = virt_addr;
> + meminfo->phys = phys_addr;
> + meminfo->len = len;
> + meminfo->addr = virt_addr + offset;
> +
> + return meminfo;
> +
> +err_epc_unmap_addr:
> + pci_epc_unmap_addr(epf->epc, epf->func_no, epf->vfunc_no,
> + meminfo->phys);
> +err_epc_free_addr:
> + pci_epc_mem_free_addr(epf->epc, meminfo->phys, meminfo->virt,
> + meminfo->len);
> +
> + return NULL;
> +}
> +
> +static void epf_vnet_rc_epc_munmap(struct pci_epf *epf,
> + struct epf_vnet_rc_meminfo *meminfo)
> +{
> + pci_epc_unmap_addr(epf->epc, epf->func_no, epf->vfunc_no,
> + meminfo->phys);
> + pci_epc_mem_free_addr(epf->epc, meminfo->phys, meminfo->virt,
> + meminfo->len);
> + kfree(meminfo);
> +}
> +
> +static int epf_vnet_rc_process_ctrlq_entry(struct epf_vnet *vnet)
> +{
> + struct vringh_kiov *riov = &vnet->rc.ctl_riov;
> + struct vringh_kiov *wiov = &vnet->rc.ctl_wiov;
> + struct vringh *vrh = &vnet->rc.ctlvrh->vrh;
> + struct pci_epf *epf = vnet->epf;
> + struct epf_vnet_rc_meminfo *rmem, *wmem;
> + struct virtio_net_ctrl_hdr *hdr;
> + int err;
> + u16 head;
> + size_t total_len;
> + u8 class, cmd;
> +
> + err = vringh_getdesc(vrh, riov, wiov, &head);
> + if (err <= 0)
> + return err;
> +
> + total_len = vringh_kiov_length(riov);
> +
> + rmem = epf_vnet_rc_epc_mmap(epf, (u64)riov->iov[riov->i].iov_base,
> + riov->iov[riov->i].iov_len);
> + if (!rmem) {
> + err = -ENOMEM;
> + goto err_abandon_descs;
> + }
> +
> + wmem = epf_vnet_rc_epc_mmap(epf, (u64)wiov->iov[wiov->i].iov_base,
> + wiov->iov[wiov->i].iov_len);
> + if (!wmem) {
> + err = -ENOMEM;
> + goto err_epc_unmap_rmem;
> + }
> +
> + hdr = rmem->addr;
> + class = ioread8(&hdr->class);
> + cmd = ioread8(&hdr->cmd);
> + switch (ioread8(&hdr->class)) {
> + case VIRTIO_NET_CTRL_ANNOUNCE:
> + if (cmd != VIRTIO_NET_CTRL_ANNOUNCE_ACK) {
> + pr_err("Found invalid command: announce: %d\n", cmd);
> + break;
> + }
> + epf_vnet_rc_clear_config16(
> + vnet,
> + VIRTIO_PCI_CONFIG_OFF(false) +
> + offsetof(struct virtio_net_config, status),
> + VIRTIO_NET_S_ANNOUNCE);
> + epf_vnet_rc_clear_config16(vnet, VIRTIO_PCI_ISR,
> + VIRTIO_PCI_ISR_CONFIG);
> +
> + iowrite8(VIRTIO_NET_OK, wmem->addr);
> + break;
> + default:
> + pr_err("Found unsupported class in control queue: %d\n", class);
> + break;
> + }
> +
> + epf_vnet_rc_epc_munmap(epf, rmem);
> + epf_vnet_rc_epc_munmap(epf, wmem);
> + vringh_complete(vrh, head, total_len);
> +
> + return 1;
> +
> +err_epc_unmap_rmem:
> + epf_vnet_rc_epc_munmap(epf, rmem);
> +err_abandon_descs:
> + vringh_abandon(vrh, head);
> +
> + return err;
> +}
> +
> +static void epf_vnet_rc_process_ctrlq_entries(struct work_struct *work)
> +{
> + struct epf_vnet *vnet =
> + container_of(work, struct epf_vnet, rc.ctl_work);
> +
> + while (epf_vnet_rc_process_ctrlq_entry(vnet) > 0)
> + ;
> +}
> +
> +void epf_vnet_rc_notify(struct epf_vnet *vnet)
> +{
> + queue_work(vnet->rc.irq_wq, &vnet->rc.raise_irq_work);
> +}
> +
> +void epf_vnet_rc_cleanup(struct epf_vnet *vnet)
> +{
> + epf_vnet_cleanup_bar(vnet);
> + destroy_workqueue(vnet->rc.tx_wq);
> + destroy_workqueue(vnet->rc.irq_wq);
> + destroy_workqueue(vnet->rc.ctl_wq);
> +
> + kthread_stop(vnet->rc.device_setup_task);
> +}
> +
> +int epf_vnet_rc_setup(struct epf_vnet *vnet)
> +{
> + int err;
> + struct pci_epf *epf = vnet->epf;
> +
> + err = pci_epc_write_header(epf->epc, epf->func_no, epf->vfunc_no,
> + &epf_vnet_pci_header);
> + if (err)
> + return err;
> +
> + err = epf_vnet_setup_bar(vnet);
> + if (err)
> + return err;
> +
> + vnet->rc.tx_wq =
> + alloc_workqueue("pci-epf-vnet/tx-wq",
> + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_UNBOUND, 0);
> + if (!vnet->rc.tx_wq) {
> + pr_debug(
> + "Failed to allocate workqueue for rc -> ep transmission\n");
> + err = -ENOMEM;
> + goto err_cleanup_bar;
> + }
> +
> + INIT_WORK(&vnet->rc.tx_work, epf_vnet_rc_tx_handler);
> +
> + vnet->rc.irq_wq =
> + alloc_workqueue("pci-epf-vnet/irq-wq",
> + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_UNBOUND, 0);
> + if (!vnet->rc.irq_wq) {
> + pr_debug("Failed to allocate workqueue for irq\n");
> + err = -ENOMEM;
> + goto err_destory_tx_wq;
> + }
> +
> + INIT_WORK(&vnet->rc.raise_irq_work, epf_vnet_rc_raise_irq_handler);
> +
> + vnet->rc.ctl_wq =
> + alloc_workqueue("pci-epf-vnet/ctl-wq",
> + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_UNBOUND, 0);
> + if (!vnet->rc.ctl_wq) {
> + pr_err("Failed to allocate work queue for control queue processing\n");
> + err = -ENOMEM;
> + goto err_destory_irq_wq;
> + }
> +
> + INIT_WORK(&vnet->rc.ctl_work, epf_vnet_rc_process_ctrlq_entries);
> +
> + err = epf_vnet_rc_spawn_device_setup_task(vnet);
> + if (err)
> + goto err_destory_ctl_wq;
> +
> + return 0;
> +
> +err_cleanup_bar:
> + epf_vnet_cleanup_bar(vnet);
> +err_destory_tx_wq:
> + destroy_workqueue(vnet->rc.tx_wq);
> +err_destory_irq_wq:
> + destroy_workqueue(vnet->rc.irq_wq);
> +err_destory_ctl_wq:
> + destroy_workqueue(vnet->rc.ctl_wq);
> +
> + return err;
> +}
> diff --git a/drivers/pci/endpoint/functions/pci-epf-vnet.c b/drivers/pci/endpoint/functions/pci-epf-vnet.c
> new file mode 100644
> index 000000000000..e48ad8067796
> --- /dev/null
> +++ b/drivers/pci/endpoint/functions/pci-epf-vnet.c
> @@ -0,0 +1,387 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * PCI Endpoint function driver to impliment virtio-net device.
> + */
> +#include <linux/module.h>
> +#include <linux/pci-epf.h>
> +#include <linux/pci-epc.h>
> +#include <linux/vringh.h>
> +#include <linux/dmaengine.h>
> +
> +#include "pci-epf-vnet.h"
> +
> +static int virtio_queue_size = 0x100;
> +module_param(virtio_queue_size, int, 0444);
> +MODULE_PARM_DESC(virtio_queue_size, "A length of virtqueue");
> +
> +int epf_vnet_get_vq_size(void)
> +{
> + return virtio_queue_size;
> +}
> +
> +int epf_vnet_init_kiov(struct vringh_kiov *kiov, const size_t vq_size)
> +{
> + struct kvec *kvec;
> +
> + kvec = kmalloc_array(vq_size, sizeof(*kvec), GFP_KERNEL);
> + if (!kvec)
> + return -ENOMEM;
> +
> + vringh_kiov_init(kiov, kvec, vq_size);
> +
> + return 0;
> +}
> +
> +void epf_vnet_deinit_kiov(struct vringh_kiov *kiov)
> +{
> + kfree(kiov->iov);
> +}
> +
> +void epf_vnet_init_complete(struct epf_vnet *vnet, u8 from)
> +{
> + vnet->init_complete |= from;
> +
> + if (!(vnet->init_complete & EPF_VNET_INIT_COMPLETE_EP))
> + return;
> +
> + if (!(vnet->init_complete & EPF_VNET_INIT_COMPLETE_RC))
> + return;
> +
> + epf_vnet_ep_announce_linkup(vnet);
> + epf_vnet_rc_announce_linkup(vnet);
> +}
> +
> +struct epf_dma_filter_param {
> + struct device *dev;
> + u32 dma_mask;
> +};
> +
> +static bool epf_virtnet_dma_filter(struct dma_chan *chan, void *param)
> +{
> + struct epf_dma_filter_param *fparam = param;
> + struct dma_slave_caps caps;
> +
> + memset(&caps, 0, sizeof(caps));
> + dma_get_slave_caps(chan, &caps);
> +
> + return chan->device->dev == fparam->dev &&
> + (fparam->dma_mask & caps.directions);
> +}
> +
> +static int epf_vnet_init_edma(struct epf_vnet *vnet, struct device *dma_dev)
> +{
> + struct epf_dma_filter_param param;
> + dma_cap_mask_t mask;
> + int err;
> +
> + dma_cap_zero(mask);
> + dma_cap_set(DMA_SLAVE, mask);
> +
> + param.dev = dma_dev;
> + param.dma_mask = BIT(DMA_MEM_TO_DEV);
> + vnet->lr_dma_chan =
> + dma_request_channel(mask, epf_virtnet_dma_filter, &param);
> + if (!vnet->lr_dma_chan)
> + return -EOPNOTSUPP;
> +
> + param.dma_mask = BIT(DMA_DEV_TO_MEM);
> + vnet->rl_dma_chan =
> + dma_request_channel(mask, epf_virtnet_dma_filter, &param);
> + if (!vnet->rl_dma_chan) {
> + err = -EOPNOTSUPP;
> + goto err_release_channel;
> + }
> +
> + return 0;
> +
> +err_release_channel:
> + dma_release_channel(vnet->lr_dma_chan);
> +
> + return err;
> +}
> +
> +static void epf_vnet_deinit_edma(struct epf_vnet *vnet)
> +{
> + dma_release_channel(vnet->lr_dma_chan);
> + dma_release_channel(vnet->rl_dma_chan);
> +}
> +
> +static int epf_vnet_dma_single(struct epf_vnet *vnet, phys_addr_t pci,
> + dma_addr_t dma, size_t len,
> + void (*callback)(void *), void *param,
> + enum dma_transfer_direction dir)
> +{
> + struct dma_async_tx_descriptor *desc;
> + int err;
> + struct dma_chan *chan;
> + struct dma_slave_config sconf;
> + dma_cookie_t cookie;
> + unsigned long flags = 0;
> +
> + if (dir == DMA_MEM_TO_DEV) {
> + sconf.dst_addr = pci;
> + chan = vnet->lr_dma_chan;
> + } else {
> + sconf.src_addr = pci;
> + chan = vnet->rl_dma_chan;
> + }
> +
> + err = dmaengine_slave_config(chan, &sconf);
> + if (unlikely(err))
> + return err;
> +
> + if (callback)
> + flags = DMA_PREP_INTERRUPT | DMA_PREP_FENCE;
> +
> + desc = dmaengine_prep_slave_single(chan, dma, len, dir, flags);
> + if (unlikely(!desc))
> + return -EIO;
> +
> + desc->callback = callback;
> + desc->callback_param = param;
> +
> + cookie = dmaengine_submit(desc);
> + err = dma_submit_error(cookie);
> + if (unlikely(err))
> + return err;
> +
> + dma_async_issue_pending(chan);
> +
> + return 0;
> +}
> +
> +struct epf_vnet_dma_callback_param {
> + struct epf_vnet *vnet;
> + struct vringh *tx_vrh, *rx_vrh;
> + struct virtqueue *vq;
> + size_t total_len;
> + u16 tx_head, rx_head;
> +};
> +
> +static void epf_vnet_dma_callback(void *p)
> +{
> + struct epf_vnet_dma_callback_param *param = p;
> + struct epf_vnet *vnet = param->vnet;
> +
> + vringh_complete(param->tx_vrh, param->tx_head, param->total_len);
> + vringh_complete(param->rx_vrh, param->rx_head, param->total_len);
> +
> + epf_vnet_rc_notify(vnet);
> + epf_vnet_ep_notify(vnet, param->vq);
> +
> + kfree(param);
> +}
> +
> +/**
> + * epf_vnet_transfer() - transfer data between tx vring to rx vring using edma
> + * @vnet: epf virtio net device to do dma
> + * @tx_vrh: vringh related to source tx vring
> + * @rx_vrh: vringh related to target rx vring
> + * @tx_iov: buffer to use tx
> + * @rx_iov: buffer to use rx
> + * @dir: a direction of DMA. local to remote or local from remote
> + *
> + * This function returns 0, 1 or error number. The 0 indicates there is not
> + * data to send. The 1 indicates a request to DMA is succeeded. Other error
> + * numbers shows error, however, ENOSPC means there is no buffer on target
> + * vring, so should retry to call later.
> + */
> +int epf_vnet_transfer(struct epf_vnet *vnet, struct vringh *tx_vrh,
> + struct vringh *rx_vrh, struct vringh_kiov *tx_iov,
> + struct vringh_kiov *rx_iov,
> + enum dma_transfer_direction dir)
> +{
> + int err;
> + u16 tx_head, rx_head;
> + size_t total_tx_len;
> + struct epf_vnet_dma_callback_param *cb_param;
> + struct vringh_kiov *liov, *riov;
> +
> + err = vringh_getdesc(tx_vrh, tx_iov, NULL, &tx_head);
> + if (err <= 0)
> + return err;
> +
> + total_tx_len = vringh_kiov_length(tx_iov);
> +
> + err = vringh_getdesc(rx_vrh, NULL, rx_iov, &rx_head);
> + if (err < 0) {
> + goto err_tx_complete;
> + } else if (!err) {
> + /* There is not space on a vring of destination to transmit data, so
> + * rollback tx vringh
> + */
> + vringh_abandon(tx_vrh, tx_head);
> + return -ENOSPC;
> + }
> +
> + cb_param = kmalloc(sizeof(*cb_param), GFP_KERNEL);
> + if (!cb_param) {
> + err = -ENOMEM;
> + goto err_rx_complete;
> + }
> +
> + cb_param->tx_vrh = tx_vrh;
> + cb_param->rx_vrh = rx_vrh;
> + cb_param->tx_head = tx_head;
> + cb_param->rx_head = rx_head;
> + cb_param->total_len = total_tx_len;
> + cb_param->vnet = vnet;
> +
> + switch (dir) {
> + case DMA_MEM_TO_DEV:
> + liov = tx_iov;
> + riov = rx_iov;
> + cb_param->vq = vnet->ep.txvq;
> + break;
> + case DMA_DEV_TO_MEM:
> + liov = rx_iov;
> + riov = tx_iov;
> + cb_param->vq = vnet->ep.rxvq;
> + break;
> + default:
> + err = -EINVAL;
> + goto err_free_param;
> + }
> +
> + for (; tx_iov->i < tx_iov->used; tx_iov->i++, rx_iov->i++) {
> + size_t len;
> + u64 lbase, rbase;
> + void (*callback)(void *) = NULL;
> +
> + lbase = (u64)liov->iov[liov->i].iov_base;
> + rbase = (u64)riov->iov[riov->i].iov_base;
> + len = tx_iov->iov[tx_iov->i].iov_len;
> +
> + if (tx_iov->i + 1 == tx_iov->used)
> + callback = epf_vnet_dma_callback;
> +
> + err = epf_vnet_dma_single(vnet, rbase, lbase, len, callback,
> + cb_param, dir);
> + if (err)
> + goto err_free_param;
> + }
> +
> + return 1;
> +
> +err_free_param:
> + kfree(cb_param);
> +err_rx_complete:
> + vringh_complete(rx_vrh, rx_head, vringh_kiov_length(rx_iov));
> +err_tx_complete:
> + vringh_complete(tx_vrh, tx_head, total_tx_len);
> +
> + return err;
> +}
> +
> +static int epf_vnet_bind(struct pci_epf *epf)
> +{
> + int err;
> + struct epf_vnet *vnet = epf_get_drvdata(epf);
> +
> + err = epf_vnet_init_edma(vnet, epf->epc->dev.parent);
> + if (err)
> + return err;
> +
> + err = epf_vnet_rc_setup(vnet);
> + if (err)
> + goto err_free_edma;
> +
> + err = epf_vnet_ep_setup(vnet);
> + if (err)
> + goto err_cleanup_rc;
> +
> + return 0;
> +
> +err_free_edma:
> + epf_vnet_deinit_edma(vnet);
> +err_cleanup_rc:
> + epf_vnet_rc_cleanup(vnet);
> +
> + return err;
> +}
> +
> +static void epf_vnet_unbind(struct pci_epf *epf)
> +{
> + struct epf_vnet *vnet = epf_get_drvdata(epf);
> +
> + epf_vnet_deinit_edma(vnet);
> + epf_vnet_rc_cleanup(vnet);
> + epf_vnet_ep_cleanup(vnet);
> +}
> +
> +static struct pci_epf_ops epf_vnet_ops = {
> + .bind = epf_vnet_bind,
> + .unbind = epf_vnet_unbind,
> +};
> +
> +static const struct pci_epf_device_id epf_vnet_ids[] = {
> + { .name = "pci_epf_vnet" },
> + {}
> +};
> +
> +static void epf_vnet_virtio_init(struct epf_vnet *vnet)
> +{
> + vnet->virtio_features =
> + BIT(VIRTIO_NET_F_MTU) | BIT(VIRTIO_NET_F_STATUS) |
> + /* Following features are to skip any of checking and offloading, Like a
> + * transmission between virtual machines on same system. Details are on
> + * section 5.1.5 in virtio specification.
> + */
> + BIT(VIRTIO_NET_F_GUEST_CSUM) | BIT(VIRTIO_NET_F_GUEST_TSO4) |
> + BIT(VIRTIO_NET_F_GUEST_TSO6) | BIT(VIRTIO_NET_F_GUEST_ECN) |
> + BIT(VIRTIO_NET_F_GUEST_UFO) |
> + // The control queue is just used for linkup announcement.
> + BIT(VIRTIO_NET_F_CTRL_VQ);
> +
> + vnet->vnet_cfg.max_virtqueue_pairs = 1;
> + vnet->vnet_cfg.status = 0;
> + vnet->vnet_cfg.mtu = PAGE_SIZE;
> +}
> +
> +static int epf_vnet_probe(struct pci_epf *epf)
> +{
> + struct epf_vnet *vnet;
> +
> + vnet = devm_kzalloc(&epf->dev, sizeof(*vnet), GFP_KERNEL);
> + if (!vnet)
> + return -ENOMEM;
> +
> + epf_set_drvdata(epf, vnet);
> + vnet->epf = epf;
> +
> + epf_vnet_virtio_init(vnet);
> +
> + return 0;
> +}
> +
> +static struct pci_epf_driver epf_vnet_drv = {
> + .driver.name = "pci_epf_vnet",
> + .ops = &epf_vnet_ops,
> + .id_table = epf_vnet_ids,
> + .probe = epf_vnet_probe,
> + .owner = THIS_MODULE,
> +};
> +
> +static int __init epf_vnet_init(void)
> +{
> + int err;
> +
> + err = pci_epf_register_driver(&epf_vnet_drv);
> + if (err) {
> + pr_err("Failed to register epf vnet driver\n");
> + return err;
> + }
> +
> + return 0;
> +}
> +module_init(epf_vnet_init);
> +
> +static void epf_vnet_exit(void)
> +{
> + pci_epf_unregister_driver(&epf_vnet_drv);
> +}
> +module_exit(epf_vnet_exit);
> +
> +MODULE_LICENSE("GPL");
> +MODULE_AUTHOR("Shunsuke Mie <mie@xxxxxxxxxx>");
> +MODULE_DESCRIPTION("PCI endpoint function acts as virtio net device");
> diff --git a/drivers/pci/endpoint/functions/pci-epf-vnet.h b/drivers/pci/endpoint/functions/pci-epf-vnet.h
> new file mode 100644
> index 000000000000..1e0f90c95578
> --- /dev/null
> +++ b/drivers/pci/endpoint/functions/pci-epf-vnet.h
> @@ -0,0 +1,62 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef _PCI_EPF_VNET_H
> +#define _PCI_EPF_VNET_H
> +
> +#include <linux/pci-epf.h>
> +#include <linux/pci-epf-virtio.h>
> +#include <linux/virtio_net.h>
> +#include <linux/dmaengine.h>
> +#include <linux/virtio.h>
> +
> +struct epf_vnet {
> + //TODO Should this variable be placed here?
> + struct pci_epf *epf;
> + struct virtio_net_config vnet_cfg;
> + u64 virtio_features;
> +
> + // dma channels for local to remote(lr) and remote to local(rl)
> + struct dma_chan *lr_dma_chan, *rl_dma_chan;
> +
> + struct {
> + void __iomem *cfg_base;
> + struct task_struct *device_setup_task;
> + struct task_struct *notify_monitor_task;
> + struct workqueue_struct *tx_wq, *irq_wq, *ctl_wq;
> + struct work_struct tx_work, raise_irq_work, ctl_work;
> + struct pci_epf_vringh *txvrh, *rxvrh, *ctlvrh;
> + struct vringh_kiov tx_iov, rx_iov, ctl_riov, ctl_wiov;
> + } rc;
> +
> + struct {
> + struct virtqueue *rxvq, *txvq, *ctlvq;
> + struct vringh txvrh, rxvrh, ctlvrh;
> + struct vringh_kiov tx_iov, rx_iov, ctl_riov, ctl_wiov;
> + struct virtio_device vdev;
> + u16 net_config_status;
> + } ep;
> +
> +#define EPF_VNET_INIT_COMPLETE_EP BIT(0)
> +#define EPF_VNET_INIT_COMPLETE_RC BIT(1)
> + u8 init_complete;
> +};
> +
> +int epf_vnet_rc_setup(struct epf_vnet *vnet);
> +void epf_vnet_rc_cleanup(struct epf_vnet *vnet);
> +int epf_vnet_ep_setup(struct epf_vnet *vnet);
> +void epf_vnet_ep_cleanup(struct epf_vnet *vnet);
> +
> +int epf_vnet_get_vq_size(void);
> +int epf_vnet_init_kiov(struct vringh_kiov *kiov, const size_t vq_size);
> +void epf_vnet_deinit_kiov(struct vringh_kiov *kiov);
> +int epf_vnet_transfer(struct epf_vnet *vnet, struct vringh *tx_vrh,
> + struct vringh *rx_vrh, struct vringh_kiov *tx_iov,
> + struct vringh_kiov *rx_iov,
> + enum dma_transfer_direction dir);
> +void epf_vnet_rc_notify(struct epf_vnet *vnet);
> +void epf_vnet_ep_notify(struct epf_vnet *vnet, struct virtqueue *vq);
> +
> +void epf_vnet_init_complete(struct epf_vnet *vnet, u8 from);
> +void epf_vnet_ep_announce_linkup(struct epf_vnet *vnet);
> +void epf_vnet_rc_announce_linkup(struct epf_vnet *vnet);
> +
> +#endif // _PCI_EPF_VNET_H
> --
> 2.25.1