Re: [PATCH v12 19/25] irqchip/riscv-imsic: Add device MSI domain support for platform devices

[Anup Patel]

On Sat, Feb 17, 2024 at 1:42 AM Thomas Gleixner <tglx@xxxxxxxxxxxxx> wrote:
>
> On Sat, Jan 27 2024 at 21:47, Anup Patel wrote:
> > +static int imsic_cpu_page_phys(unsigned int cpu,
> > +                            unsigned int guest_index,
> > +                            phys_addr_t *out_msi_pa)
> > +{
> > +     struct imsic_global_config *global;
> > +     struct imsic_local_config *local;
> > +
> > +     global = &imsic->global;
> > +     local = per_cpu_ptr(global->local, cpu);
> > +
> > +     if (BIT(global->guest_index_bits) <= guest_index)
> > +             return -EINVAL;
>
> As the callsite does not care about the return value, just make this
> function boolean and return true on success.

Okay, I will update.

>
> > +     if (out_msi_pa)
> > +             *out_msi_pa = local->msi_pa +
> > +                           (guest_index * IMSIC_MMIO_PAGE_SZ);
> > +
> > +     return 0;
> > +}
> > +
> > +static void imsic_irq_mask(struct irq_data *d)
> > +{
> > +     imsic_vector_mask(irq_data_get_irq_chip_data(d));
> > +}
> > +
> > +static void imsic_irq_unmask(struct irq_data *d)
> > +{
> > +     imsic_vector_unmask(irq_data_get_irq_chip_data(d));
> > +}
> > +
> > +static int imsic_irq_retrigger(struct irq_data *d)
> > +{
> > +     struct imsic_vector *vec = irq_data_get_irq_chip_data(d);
> > +     struct imsic_local_config *local;
> > +
> > +     if (WARN_ON(vec == NULL))
> > +             return -ENOENT;
> > +
> > +     local = per_cpu_ptr(imsic->global.local, vec->cpu);
> > +     writel(vec->local_id, local->msi_va);
> > +     return 0;
> > +}
> > +
> > +static void imsic_irq_compose_vector_msg(struct imsic_vector *vec,
> > +                                      struct msi_msg *msg)
> > +{
> > +     phys_addr_t msi_addr;
> > +     int err;
> > +
> > +     if (WARN_ON(vec == NULL))
> > +             return;
> > +
> > +     err = imsic_cpu_page_phys(vec->cpu, 0, &msi_addr);
> > +     if (WARN_ON(err))
> > +             return;
>
>         if (WARN_ON(!imsic_cpu_page_phys(...)))
>                 return
> Hmm?

Okay, I will update like you suggested.

>
> > +
> > +     msg->address_hi = upper_32_bits(msi_addr);
> > +     msg->address_lo = lower_32_bits(msi_addr);
> > +     msg->data = vec->local_id;
> > +}
> > +
> > +static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg)
> > +{
> > +     imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg);
> > +}
> > +
> > +#ifdef CONFIG_SMP
> > +static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec)
> > +{
> > +     struct msi_msg msg[2] = { [1] = { }, };
> > +
> > +     imsic_irq_compose_vector_msg(vec, msg);
> > +     irq_data_get_irq_chip(d)->irq_write_msi_msg(d, msg);
> > +}
> > +
> > +static int imsic_irq_set_affinity(struct irq_data *d,
> > +                               const struct cpumask *mask_val,
> > +                               bool force)
> > +{
> > +     struct imsic_vector *old_vec, *new_vec;
> > +     struct irq_data *pd = d->parent_data;
> > +
> > +     old_vec = irq_data_get_irq_chip_data(pd);
> > +     if (WARN_ON(old_vec == NULL))
> > +             return -ENOENT;
> > +
> > +     /* Get a new vector on the desired set of CPUs */
> > +     new_vec = imsic_vector_alloc(old_vec->hwirq, mask_val);
> > +     if (!new_vec)
> > +             return -ENOSPC;
> > +
> > +     /* If old vector belongs to the desired CPU then do nothing */
> > +     if (old_vec->cpu == new_vec->cpu) {
> > +             imsic_vector_free(new_vec);
> > +             return IRQ_SET_MASK_OK_DONE;
> > +     }
>
> You can spare that exercise by checking it before the allocation:
>
>         if (cpumask_test_cpu(old_vec->cpu, mask_val))
>                 return IRQ_SET_MASK_OK_DONE;

Okay, I will update.

>
> > +
> > +     /* Point device to the new vector */
> > +     imsic_msi_update_msg(d, new_vec);
>
> > +static int imsic_irq_domain_alloc(struct irq_domain *domain,
> > +                               unsigned int virq, unsigned int nr_irqs,
> > +                               void *args)
> > +{
> > +     struct imsic_vector *vec;
> > +     int hwirq;
> > +
> > +     /* Legacy-MSI or multi-MSI not supported yet. */
>
> What's legacy MSI in that context?

The legacy-MSI is the MSI support in PCI v2.2 where
number of MSIs allocated by device were either 1, 2, 4,
8, 16, or 32 and the data written is <data_word> + <irqnum>.

>
> > +     if (nr_irqs > 1)
> > +             return -ENOTSUPP;
> > +
> > +     hwirq = imsic_hwirq_alloc();
> > +     if (hwirq < 0)
> > +             return hwirq;
> > +
> > +     vec = imsic_vector_alloc(hwirq, cpu_online_mask);
> > +     if (!vec) {
> > +             imsic_hwirq_free(hwirq);
> > +             return -ENOSPC;
> > +     }
> > +
> > +     irq_domain_set_info(domain, virq, hwirq,
> > +                         &imsic_irq_base_chip, vec,
> > +                         handle_simple_irq, NULL, NULL);
> > +     irq_set_noprobe(virq);
> > +     irq_set_affinity(virq, cpu_online_mask);
> > +
> > +     /*
> > +      * IMSIC does not implement irq_disable() so Linux interrupt
> > +      * subsystem will take a lazy approach for disabling an IMSIC
> > +      * interrupt. This means IMSIC interrupts are left unmasked
> > +      * upon system suspend and interrupts are not processed
> > +      * immediately upon system wake up. To tackle this, we disable
> > +      * the lazy approach for all IMSIC interrupts.
>
> Why? Lazy works perfectly fine even w/o an irq_disable() callback.

This was suggested by SiFive folks. I am also not sure why we
need this. For now, I will drop this and bring it back as a separate
patch if required.

>
> > +      */
> > +     irq_set_status_flags(virq, IRQ_DISABLE_UNLAZY);
>
> > +
> > +#define MATCH_PLATFORM_MSI           BIT(DOMAIN_BUS_PLATFORM_MSI)
>
> You really love macro indirections :)

This is to be consistent with MATCH_PCI_MSI introduced by the
subsequent patch.

Also, this is inspired from your ARM GIC patches.
https://lore.kernel.org/linux-arm-kernel/20221121140049.038269899@xxxxxxxxxxxxx/
https://lore.kernel.org/linux-arm-kernel/20221121140049.112451419@xxxxxxxxxxxxx/
https://lore.kernel.org/linux-arm-kernel/20221121140049.237988384@xxxxxxxxxxxxx/
https://lore.kernel.org/linux-arm-kernel/20221121140049.941784867@xxxxxxxxxxxxx/

>
> > +static const struct msi_parent_ops imsic_msi_parent_ops = {
> > +     .supported_flags        = MSI_GENERIC_FLAGS_MASK,
> > +     .required_flags         = MSI_FLAG_USE_DEF_DOM_OPS |
> > +                               MSI_FLAG_USE_DEF_CHIP_OPS,
> > +     .bus_select_token       = DOMAIN_BUS_NEXUS,
> > +     .bus_select_mask        = MATCH_PLATFORM_MSI,
> > +     .init_dev_msi_info      = imsic_init_dev_msi_info,
> > +};
> > +
> > +int imsic_irqdomain_init(void)
> > +{
> > +     struct imsic_global_config *global;
> > +
> > +     if (!imsic || !imsic->fwnode) {
> > +             pr_err("early driver not probed\n");
> > +             return -ENODEV;
> > +     }
> > +
> > +     if (imsic->base_domain) {
> > +             pr_err("%pfwP: irq domain already created\n", imsic->fwnode);
> > +             return -ENODEV;
> > +     }
> > +
> > +     global = &imsic->global;
>
> Please move that assignment down to the usage site. Here it's just a
> distraction.

Okay, I will update.

>
> > +     /* Create Base IRQ domain */
> > +     imsic->base_domain = irq_domain_create_tree(imsic->fwnode,
> > +                                     &imsic_base_domain_ops, imsic);
> > +     if (!imsic->base_domain) {
> > +             pr_err("%pfwP: failed to create IMSIC base domain\n",
> > +                     imsic->fwnode);
> > +             return -ENOMEM;
> > +     }
> > +     imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;
> > +     imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops;
>

Regards,
Anup