| Message ID | e9dadd96aa5b64b9232e10a083ce393af620adde.1639472078.git.oleksii_moisieiev@epam.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Introduce SCI-mediator feature | expand |
On 14.12.21 11:34, Oleksii Moisieiev wrote: Hi Oleksii > This is the implementation of SCI interface, called SCMI-SMC driver, > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > This allows devices from the Domains to work with clocks, resets and > power-domains without access to CPG. > > The following features are implemented: > - request SCMI channels from ATF and pass channels to Domains; > - set device permissions for Domains based on the Domain partial > device-tree. Devices with permissions are able to work with clocks, > resets and power-domains via SCMI; > - redirect scmi messages from Domains to ATF. > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > --- > xen/arch/arm/Kconfig | 2 + > xen/arch/arm/sci/Kconfig | 10 + > xen/arch/arm/sci/Makefile | 1 + > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > xen/include/public/arch-arm.h | 1 + > 5 files changed, 809 insertions(+) > create mode 100644 xen/arch/arm/sci/Kconfig > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > index 186e1db389..02d96c6cfc 100644 > --- a/xen/arch/arm/Kconfig > +++ b/xen/arch/arm/Kconfig > @@ -114,6 +114,8 @@ config SCI > support. It allows guests to control system resourcess via one of > SCI mediators implemented in XEN. > > +source "arch/arm/sci/Kconfig" > + > endmenu > > menu "ARM errata workaround via the alternative framework" > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > new file mode 100644 > index 0000000000..9563067ddc > --- /dev/null > +++ b/xen/arch/arm/sci/Kconfig > @@ -0,0 +1,10 @@ > +config SCMI_SMC > + bool "Enable SCMI-SMC mediator driver" > + default n > + depends on SCI > + ---help--- > + > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > + This feature allows drivers from Domains to work with System > + Controllers (such as power,resets,clock etc.). SCP is used as transport > + for communication. > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > index 837dc7492b..67f2611872 100644 > --- a/xen/arch/arm/sci/Makefile > +++ b/xen/arch/arm/sci/Makefile > @@ -1 +1,2 @@ > obj-y += sci.o > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > new file mode 100644 > index 0000000000..2eb01ea82d > --- /dev/null > +++ b/xen/arch/arm/sci/scmi_smc.c > @@ -0,0 +1,795 @@ > +/* > + * xen/arch/arm/sci/scmi_smc.c > + * > + * SCMI mediator driver, using SCP as transport. > + * > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > + * Copyright (C) 2021, EPAM Systems. > + * > + * This program is free software; you can redistribute it and/or modify > + * it under the terms of the GNU General Public License as published by > + * the Free Software Foundation; either version 2 of the License, or > + * (at your option) any later version. > + * > + * This program is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + */ > + > +#include <asm/sci/sci.h> > +#include <asm/smccc.h> > +#include <asm/io.h> > +#include <xen/bitops.h> > +#include <xen/config.h> > +#include <xen/sched.h> > +#include <xen/device_tree.h> > +#include <xen/iocap.h> > +#include <xen/init.h> > +#include <xen/err.h> > +#include <xen/lib.h> > +#include <xen/list.h> > +#include <xen/mm.h> > +#include <xen/string.h> > +#include <xen/time.h> > +#include <xen/vmap.h> > + > +#define SCMI_BASE_PROTOCOL 0x10 > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > + > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > +#define SCMI_SUCCESS 0 > +#define SCMI_NOT_SUPPORTED (-1) > +#define SCMI_INVALID_PARAMETERS (-2) > +#define SCMI_DENIED (-3) > +#define SCMI_NOT_FOUND (-4) > +#define SCMI_OUT_OF_RANGE (-5) > +#define SCMI_BUSY (-6) > +#define SCMI_COMMS_ERROR (-7) > +#define SCMI_GENERIC_ERROR (-8) > +#define SCMI_HARDWARE_ERROR (-9) > +#define SCMI_PROTOCOL_ERROR (-10) > + > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > + > +#define SCMI_SMC_ID "arm,smc-id" > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > +#define SCMI_SHMEM "shmem" > + > +#define HYP_CHANNEL 0x0 > + > +#define HDR_ID GENMASK(7,0) > +#define HDR_TYPE GENMASK(9, 8) > +#define HDR_PROTO GENMASK(17, 10) > + > +/* SCMI protocol, refer to section 4.2.2.2 (DEN0056C) */ > +#define MSG_N_AGENTS_MASK GENMASK(15, 8) > + > +#define FIELD_GET(_mask, _reg)\ > + ((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1))) > +#define FIELD_PREP(_mask, _val)\ > + (((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask)) > + > +typedef struct scmi_msg_header { > + uint8_t id; > + uint8_t type; > + uint8_t protocol; > +} scmi_msg_header_t; > + > +typedef struct scmi_perms_tx { > + uint32_t agent_id; > + uint32_t device_id; > + uint32_t flags; > +} scmi_perms_tx_t; > + > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE BIT(0, UL) > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_ERROR BIT(1, UL) > + > +#define SCMI_ALLOW_ACCESS BIT(0, UL) > + > +struct scmi_shared_mem { > + uint32_t reserved; > + uint32_t channel_status; > + uint32_t reserved1[2]; > + uint32_t flags; > + uint32_t length; > + uint32_t msg_header; > + uint8_t msg_payload[]; > +}; > + > +struct scmi_channel { > + int chan_id; > + int agent_id; > + uint32_t func_id; > + int domain_id; > + uint64_t paddr; > + struct scmi_shared_mem *shmem; > + spinlock_t lock; > + struct list_head list; > +}; > + > +struct scmi_data { > + struct list_head channel_list; > + spinlock_t channel_list_lock; > + bool initialized; > + u64 shmem_addr, shmem_size; > +}; > + > +static struct scmi_data scmi_data; > + > +/* > + * pack_scmi_header() - packs and returns 32-bit header > + * > + * @hdr: pointer to header containing all the information on message id, > + * protocol id and type id. > + * > + * Return: 32-bit packed message header to be sent to the platform. > + */ > +static inline uint32_t pack_scmi_header(scmi_msg_header_t *hdr) > +{ > + return FIELD_PREP(HDR_ID, hdr->id) | > + FIELD_PREP(HDR_TYPE, hdr->type) | > + FIELD_PREP(HDR_PROTO, hdr->protocol); > +} > + > +/* > + * unpack_scmi_header() - unpacks and records message and protocol id > + * > + * @msg_hdr: 32-bit packed message header sent from the platform > + * @hdr: pointer to header to fetch message and protocol id. > + */ > +static inline void unpack_scmi_header(uint32_t msg_hdr, scmi_msg_header_t *hdr) > +{ > + hdr->id = FIELD_GET(HDR_ID, msg_hdr); > + hdr->type = FIELD_GET(HDR_TYPE, msg_hdr); > + hdr->protocol = FIELD_GET(HDR_PROTO, msg_hdr); > +} > + > +static inline int channel_is_free(struct scmi_channel *chan_info) > +{ > + return ( chan_info->shmem->channel_status > + & SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE ) ? 0 : -EBUSY; > +} > + > +static int send_smc_message(struct scmi_channel *chan_info, > + scmi_msg_header_t *hdr, void *data, int len) > +{ > + struct arm_smccc_res resp; > + int ret; > + > + printk(XENLOG_DEBUG "scmi: status =%d len=%d\n", > + chan_info->shmem->channel_status, len); > + printk(XENLOG_DEBUG "scmi: header id = %d type = %d, proto = %d\n", > + hdr->id, hdr->type, hdr->protocol); > + > + ret = channel_is_free(chan_info); > + if ( IS_ERR_VALUE(ret) ) > + return ret; > + > + chan_info->shmem->channel_status = 0x0; > + /* Writing 0x0 right now, but SCMI_SHMEM_FLAG_INTR_ENABLED can be set */ > + chan_info->shmem->flags = 0x0; > + chan_info->shmem->length = sizeof(chan_info->shmem->msg_header) + len; > + chan_info->shmem->msg_header = pack_scmi_header(hdr); > + > + printk(XENLOG_DEBUG "scmi: Writing to shmem address %p\n", > + chan_info->shmem); > + if ( len > 0 && data ) > + memcpy((void *)(chan_info->shmem->msg_payload), data, len); > + > + arm_smccc_smc(chan_info->func_id, 0, 0, 0, 0, 0, 0, chan_info->chan_id, > + &resp); > + > + printk(XENLOG_DEBUG "scmi: scmccc_smc response %d\n", (int)(resp.a0)); > + > + if ( resp.a0 ) > + return -EOPNOTSUPP; > + > + return 0; > +} > + > +static int check_scmi_status(int scmi_status) > +{ > + if ( scmi_status == SCMI_SUCCESS ) > + return 0; > + > + printk(XENLOG_DEBUG "scmi: Error received: %d\n", scmi_status); > + > + switch ( scmi_status ) > + { > + case SCMI_NOT_SUPPORTED: > + return -EOPNOTSUPP; > + case SCMI_INVALID_PARAMETERS: > + return -EINVAL; > + case SCMI_DENIED: > + return -EACCES; > + case SCMI_NOT_FOUND: > + return -ENOENT; > + case SCMI_OUT_OF_RANGE: > + return -ERANGE; > + case SCMI_BUSY: > + return -EBUSY; > + case SCMI_COMMS_ERROR: > + return -ENOTCONN; > + case SCMI_GENERIC_ERROR: > + return -EIO; > + case SCMI_HARDWARE_ERROR: > + return -ENXIO; > + case SCMI_PROTOCOL_ERROR: > + return -EBADMSG; Probably we should add a default: here? > + } > + > + return -EINVAL; > +} > + > +static int get_smc_response(struct scmi_channel *chan_info, > + scmi_msg_header_t *hdr, void *data, int len) > +{ > + int recv_len; > + int ret; > + > + printk(XENLOG_DEBUG "scmi: get smc responce msgid %d\n", hdr->id); > + > + ret = channel_is_free(chan_info); > + if ( IS_ERR_VALUE(ret) ) > + return ret; > + > + recv_len = chan_info->shmem->length - sizeof(chan_info->shmem->msg_header); > + > + if ( recv_len < 0 ) > + { > + printk(XENLOG_ERR > + "scmi: Wrong size of smc message. Data may be invalid\n"); > + return -EINVAL; > + } > + > + if ( recv_len > len ) > + { > + printk(XENLOG_ERR > + "scmi: Not enough buffer for message %d, expecting %d\n", > + recv_len, len); > + return -EINVAL; > + } > + > + unpack_scmi_header(chan_info->shmem->msg_header, hdr); > + > + if ( recv_len > 0 ) > + { > + memcpy(data, chan_info->shmem->msg_payload, recv_len); > + } > + > + return 0; > +} > + > +static int do_smc_xfer(struct scmi_channel *channel, scmi_msg_header_t *hdr, void *tx_data, int tx_size, > + void *rx_data, int rx_size) > +{ > + int ret = 0; > + > + if ( !hdr ) > + return -EINVAL; > + > + spin_lock(&channel->lock); > + > + ret = send_smc_message(channel, hdr, tx_data, tx_size); > + if ( ret ) > + goto clean; > + > + ret = get_smc_response(channel, hdr, rx_data, rx_size); > +clean: > + spin_unlock(&channel->lock); > + > + return ret; > +} > + > +static struct scmi_channel *get_channel_by_id(uint8_t chan_id) > +{ > + struct scmi_channel *curr; > + bool found = false; > + > + spin_lock(&scmi_data.channel_list_lock); > + list_for_each_entry(curr, &scmi_data.channel_list, list) > + if ( curr->chan_id == chan_id ) > + { > + found = true; > + break; > + } > + > + spin_unlock(&scmi_data.channel_list_lock); > + if ( found ) > + return curr; > + > + return NULL; > +} > + > +static struct scmi_channel *get_channel_by_domain(uint8_t domain_id) > +{ > + struct scmi_channel *curr; > + bool found = false; > + > + spin_lock(&scmi_data.channel_list_lock); > + list_for_each_entry(curr, &scmi_data.channel_list, list) > + if ( curr->domain_id == domain_id ) > + { > + found = true; > + break; > + } > + > + spin_unlock(&scmi_data.channel_list_lock); > + if ( found ) > + return curr; > + > + return NULL; > +} > + > +static struct scmi_channel *aquire_scmi_channel(int domain_id) > +{ > + struct scmi_channel *curr; > + bool found = false; > + > + ASSERT(domain_id != DOMID_INVALID && domain_id >= 0); > + > + spin_lock(&scmi_data.channel_list_lock); > + list_for_each_entry(curr, &scmi_data.channel_list, list) > + if ( (curr->domain_id == DOMID_INVALID) > + && (curr->chan_id != HYP_CHANNEL) ) > + { > + curr->domain_id = domain_id; > + found = true; > + break; > + } > + > + spin_unlock(&scmi_data.channel_list_lock); > + if ( found ) > + return curr; > + > + return NULL; > +} > + > +static void relinquish_scmi_channel(struct scmi_channel *channel) > +{ > + spin_lock(&scmi_data.channel_list_lock); > + ASSERT(channel != NULL); > + channel->domain_id = DOMID_INVALID; > + spin_unlock(&scmi_data.channel_list_lock); > +} > + > +static struct scmi_channel *smc_create_channel(uint8_t chan_id, > + uint32_t func_id, uint64_t addr) > +{ > + struct scmi_channel *channel; > + mfn_t mfn; > + > + channel = get_channel_by_id(chan_id); > + if ( channel ) > + return ERR_PTR(EEXIST); > + > + channel = xmalloc(struct scmi_channel); > + if ( !channel ) > + return ERR_PTR(ENOMEM); > + > + channel->chan_id = chan_id; > + channel->func_id = func_id; > + channel->domain_id = DOMID_INVALID; > + mfn = maddr_to_mfn(addr); > + channel->shmem = vmap(&mfn, 1); > + if ( !channel->shmem ) > + { > + xfree(channel); > + return ERR_PTR(ENOMEM); > + } > + > + printk(XENLOG_DEBUG "scmi: Got shmem after vmap %p\n", channel->shmem); > + channel->paddr = addr; > + channel->shmem->channel_status = SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE; > + spin_lock_init(&channel->lock); > + spin_lock(&scmi_data.channel_list_lock); > + list_add(&channel->list, &scmi_data.channel_list); > + spin_unlock(&scmi_data.channel_list_lock); > + return channel; > +} > + > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > +{ > + return iomem_permit_access(d, paddr_to_pfn(addr), > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > +} > + > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > + uint64_t len) > +{ > + return iomem_deny_access(d, paddr_to_pfn(addr), > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > +} I wonder, why we need an extra level of indirection here. And if this is really needed, I wonder why map(unmap)_memory* name was chosen, as there is no memory mapping/unmapping really happens here. > + > +static int dt_update_domain_range(struct domain *d, uint64_t addr, > + uint64_t size) Looks like *d is not used in this function. > +{ > + struct dt_device_node *shmem_node; > + __be32 *hw_reg; > + const struct dt_property *pp; > + uint32_t len; > + > + shmem_node = dt_find_compatible_node(NULL, NULL, "arm,scmi-shmem"); > + > + if ( !shmem_node ) > + { > + printk(XENLOG_ERR "scmi: Unable to find %s node in DT\n", SCMI_SHMEM); > + return -EINVAL; > + } > + > + pp = dt_find_property(shmem_node, "reg", &len); > + if ( !pp ) > + { > + printk(XENLOG_ERR "scmi: Unable to find regs entry in shmem node\n"); > + return -ENOENT; > + } > + > + hw_reg = pp->value; > + dt_set_range(&hw_reg, shmem_node, addr, size); > + > + return 0; > +} > + > +static void free_channel_list(void) > +{ > + struct scmi_channel *curr, *_curr; > + > + spin_lock(&scmi_data.channel_list_lock); > + list_for_each_entry_safe (curr, _curr, &scmi_data.channel_list, list) > + { > + vunmap(curr->shmem); > + list_del(&curr->list); > + xfree(curr); > + } > + > + spin_unlock(&scmi_data.channel_list_lock); > +} > + > +static __init bool scmi_probe(struct dt_device_node *scmi_node) Generic question to consider ... If probe fails for some reason (so we cannot use mediator in Xen) what should we do with SCMI nodes in domain's device-tree (leave as is, drop, etc)? > +{ > + struct dt_device_node *shmem_node; > + int ret, i; > + struct scmi_channel *channel, *agent_channel; > + int n_agents; > + scmi_msg_header_t hdr; > + struct rx_t { > + int32_t status; > + uint32_t attributes; > + } rx; > + > + uint32_t func_id; > + > + ASSERT(scmi_node != NULL); > + > + INIT_LIST_HEAD(&scmi_data.channel_list); > + spin_lock_init(&scmi_data.channel_list_lock); > + > + if ( !dt_property_read_u32(scmi_node, SCMI_SMC_ID, &func_id) ) > + { > + printk(XENLOG_ERR "scmi: Unable to read smc-id from DT\n"); > + return false; > + } > + > + shmem_node = dt_find_node_by_name(NULL, SCMI_SHARED_MEMORY); > + if ( IS_ERR_OR_NULL(shmem_node) ) > + { > + printk(XENLOG_ERR > + "scmi: Device tree error, can't parse shmem phandle %ld\n", > + PTR_ERR(shmem_node)); > + return false; > + } > + > + ret = dt_device_get_address(shmem_node, 0, &scmi_data.shmem_addr, > + &scmi_data.shmem_size); > + if ( IS_ERR_VALUE(ret) ) > + return false; > + > + channel = smc_create_channel(HYP_CHANNEL, func_id, scmi_data.shmem_addr); > + if ( IS_ERR(channel) ) > + return false; > + > + hdr.id = SCMI_BASE_PROTOCOL_ATTIBUTES; > + hdr.type = 0; > + hdr.protocol = SCMI_BASE_PROTOCOL; > + > + ret = do_smc_xfer(channel, &hdr, NULL, 0, &rx, sizeof(rx)); > + if ( ret ) > + goto clean; > + > + ret = check_scmi_status(rx.status); > + if ( ret ) > + goto clean; > + > + n_agents = FIELD_GET(MSG_N_AGENTS_MASK, rx.attributes); > + printk(XENLOG_DEBUG "scmi: Got agent count %d\n", n_agents); > + > + n_agents = (n_agents > scmi_data.shmem_size / PAGE_SIZE) ? > + scmi_data.shmem_size / PAGE_SIZE : n_agents; > + > + for ( i = 1; i < n_agents; i++ ) > + { > + uint32_t tx_agent_id = 0xFFFFFFFF; > + struct { > + int32_t status; > + uint32_t agent_id; > + char name[16]; > + } da_rx; > + > + agent_channel = smc_create_channel(i, func_id, scmi_data.shmem_addr + > + i * PAGE_SIZE); > + if ( IS_ERR(agent_channel) ) > + { > + ret = PTR_ERR(agent_channel); > + goto clean; > + } > + > + hdr.id = SCMI_BASE_DISCOVER_AGENT; > + hdr.type = 0; > + hdr.protocol = SCMI_BASE_PROTOCOL; > + > + ret = do_smc_xfer(agent_channel, &hdr, &tx_agent_id, > + sizeof(tx_agent_id), &da_rx, sizeof(da_rx)); > + if ( ret ) > + goto clean; > + > + ret = check_scmi_status(da_rx.status); > + if ( ret ) > + goto clean; > + > + printk(XENLOG_DEBUG "scmi: status=0x%x id=0x%x name=%s\n", > + da_rx.status, da_rx.agent_id, da_rx.name); > + > + agent_channel->agent_id = da_rx.agent_id; > + } > + > + scmi_data.initialized = true; > + return true; > + > +clean: > + free_channel_list(); > + return ret == 0; > +} > + > +static int scmi_domain_init(struct domain *d) > +{ > + struct scmi_channel *channel; > + int ret; > + > + if ( !scmi_data.initialized ) > + return 0; > + > + channel = aquire_scmi_channel(d->domain_id); > + if ( IS_ERR_OR_NULL(channel) ) > + return -ENOENT; > + > + printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = %d" > + "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, > + channel->paddr); It seems this breaks build on Arm32: scmi_smc.c: In function ‘scmi_domain_init’: /home/otyshchenko/xen/xen/include/xen/config.h:53:24: error: format ‘%lx’ expects argument of type ‘long unsigned int’, but argument 4 has type ‘uint64_t {aka long long unsigned int}’ [-Werror=format=] #define XENLOG_INFO "<2>" ^ scmi_smc.c:569:12: note: in expansion of macro ‘XENLOG_INFO’ printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = %d" ^~~~~~~~~~~ scmi_smc.c:570:25: note: format string is defined here "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, ~~^ %llx > + > + if ( is_hardware_domain(d) ) > + { > + ret = map_memory_to_domain(d, scmi_data.shmem_addr, > + scmi_data.shmem_size); > + if ( IS_ERR_VALUE(ret) ) > + goto error; > + > + ret = dt_update_domain_range(d, channel->paddr, PAGE_SIZE); > + if ( IS_ERR_VALUE(ret) ) > + { > + int rc = unmap_memory_from_domain(d, scmi_data.shmem_addr, > + scmi_data.shmem_size); > + if ( rc ) > + printk(XENLOG_ERR "Unable to unmap_memory_from_domain\n"); > + > + goto error; > + } > + } > + > + d->arch.sci = channel; > + > + return 0; > +error: > + relinquish_scmi_channel(channel); > + > + return ret; > +} > + > +static int scmi_add_device_by_devid(struct domain *d, uint32_t scmi_devid) > +{ > + struct scmi_channel *channel, *agent_channel; > + scmi_msg_header_t hdr; > + scmi_perms_tx_t tx; > + struct rx_t { > + int32_t status; > + uint32_t attributes; > + } rx; > + int ret; > + > + if ( !scmi_data.initialized ) > + return 0; > + > + printk(XENLOG_DEBUG "scmi: scmi_devid = %d\n", scmi_devid); > + > + agent_channel = get_channel_by_domain(d->domain_id); > + if ( IS_ERR_OR_NULL(agent_channel) ) > + return PTR_ERR(agent_channel); > + > + channel = get_channel_by_id(HYP_CHANNEL); > + if ( IS_ERR_OR_NULL(channel) ) > + return PTR_ERR(channel); > + > + hdr.id = SCMI_BASE_SET_DEVICE_PERMISSIONS; > + hdr.type = 0; > + hdr.protocol = SCMI_BASE_PROTOCOL; > + > + tx.agent_id = agent_channel->agent_id; > + tx.device_id = scmi_devid; > + tx.flags = SCMI_ALLOW_ACCESS; > + > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(&rx)); > + if ( IS_ERR_VALUE(ret) ) > + return ret; > + > + ret = check_scmi_status(rx.status); > + if ( IS_ERR_VALUE(ret) ) > + return ret; > + > + return 0; > +} > + > +static int scmi_add_dt_device(struct domain *d, struct dt_device_node *dev) > +{ > + uint32_t scmi_devid; > + > + if ( (!scmi_data.initialized) || (!d->arch.sci) ) > + return 0; > + > + if ( !dt_property_read_u32(dev, "scmi_devid", &scmi_devid) ) > + return 0; > + > + printk(XENLOG_INFO "scmi: dt_node = %s\n", dt_node_full_name(dev)); > + > + return scmi_add_device_by_devid(d, scmi_devid); > +} > + > +static int scmi_relinquish_resources(struct domain *d) > +{ > + int ret; > + struct scmi_channel *channel, *agent_channel; > + scmi_msg_header_t hdr; > + struct reset_agent_tx { > + uint32_t agent_id; > + uint32_t flags; > + } tx; > + uint32_t rx; > + > + if ( !d->arch.sci ) > + return 0; > + > + agent_channel = d->arch.sci; > + > + spin_lock(&agent_channel->lock); > + tx.agent_id = agent_channel->agent_id; > + spin_unlock(&agent_channel->lock); > + > + channel = get_channel_by_id(HYP_CHANNEL); > + if ( !channel ) > + { > + printk(XENLOG_ERR > + "scmi: Unable to get Hypervisor scmi channel for domain %d\n", > + d->domain_id); > + return -EINVAL; > + } > + > + hdr.id = SCMI_BASE_RESET_AGENT_CONFIGURATION; > + hdr.type = 0; > + hdr.protocol = SCMI_BASE_PROTOCOL; > + > + tx.flags = 0; > + > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(rx)); > + if ( ret ) > + return ret; > + > + ret = check_scmi_status(rx); > + > + return ret; > +} > + > +static void scmi_domain_destroy(struct domain *d) > +{ > + struct scmi_channel *channel; > + > + if ( !d->arch.sci ) > + return; > + > + channel = d->arch.sci; > + spin_lock(&channel->lock); > + > + relinquish_scmi_channel(channel); > + printk(XENLOG_DEBUG "scmi: Free domain %d\n", d->domain_id); > + > + d->arch.sci = NULL; > + > + unmap_memory_from_domain(d, channel->paddr, PAGE_SIZE); I didn't manage to find where corresponding map_memory_from_domain() is called for a non-hardware domain (it seems that scmi_domain_init() only directly handles hardware domain case). Or perhaps, it is called indirectly at do_domctl(): case XEN_DOMCTL_iomem_permission:? I wonder, do we really need to call this here? Taking into the account that unmap_memory_from_domain() doesn't actually unmap anything, but only removes a range from the iomem_caps rangeset for the domain to be destroyed and all involved rangesets (including iomem_caps) will be removed soon at rangeset_domain_destroy() anyway. Or I missed something? > + spin_unlock(&channel->lock); > + return; empty return could be dropped, I think. > +} > + > +static bool scmi_handle_call(struct domain *d, void *args) > +{ > + bool res = false; > + struct scmi_channel *agent_channel; > + struct arm_smccc_res resp; > + struct cpu_user_regs *regs = args; > + > + if ( !d->arch.sci ) > + return false; > + > + agent_channel = d->arch.sci; > + spin_lock(&agent_channel->lock); > + > + if ( agent_channel->func_id != regs->x0 ) This also breaks build on Arm32: scmi_smc.c: In function ‘scmi_handle_call’: scmi_smc.c:736:42: error: ‘struct cpu_user_regs’ has no member named ‘x0’; did you mean ‘r0’? if ( agent_channel->func_id != regs->x0 ) ^~ r0 cc1: all warnings being treated as errors *** BTW, I noticed that xen/arch/arm/traps.c contains the following construct, probably we might want something similar here? #ifdef CONFIG_ARM_64 #define HYPERCALL_RESULT_REG(r) (r)->x0 [snip] #else #define HYPERCALL_RESULT_REG(r) (r)->r0 [snip] #endif This RFC patch series, so I think, there is no serious issues at the moment, this is rather to let you know for the future (when you drop RFC tag). I have to admit, I often forget to build-test on Arm32 also)) > + { > + printk(XENLOG_ERR "scmi: func_id mismatch, exiting\n"); > + goto unlock; > + } > + > + arm_smccc_smc(agent_channel->func_id, 0, 0, 0, 0, 0, 0, > + agent_channel->chan_id, &resp); > + > + set_user_reg(regs, 0, resp.a0); > + set_user_reg(regs, 1, resp.a1); > + set_user_reg(regs, 2, resp.a2); > + set_user_reg(regs, 3, resp.a3); > + res = true; > +unlock: > + spin_unlock(&agent_channel->lock); > + > + return res; > +} > + > +static int scmi_get_channel_paddr(void *scmi_ops, > + struct xen_arch_domainconfig *config) > +{ > + struct scmi_channel *agent_channel = scmi_ops; > + > + if ( !agent_channel ) > + return -EINVAL; > + > + config->sci_agent_paddr = agent_channel->paddr; > + return 0; > +} > + > +static const struct dt_device_match scmi_smc_match[] __initconst = > +{ > + DT_MATCH_SCMI_SMC, > + { /* sentinel */ }, > +}; > + > +static const struct sci_mediator_ops scmi_ops = > +{ > + .probe = scmi_probe, > + .domain_init = scmi_domain_init, > + .domain_destroy = scmi_domain_destroy, > + .add_dt_device = scmi_add_dt_device, > + .relinquish_resources = scmi_relinquish_resources, > + .handle_call = scmi_handle_call, > + .get_channel_info = scmi_get_channel_paddr > +}; > + > +REGISTER_SCI_MEDIATOR(scmi_smc, "SCMI-SMC", XEN_DOMCTL_CONFIG_SCI_SCMI_SMC, > + scmi_smc_match, &scmi_ops); > + > +/* > + * Local variables: > + * mode: C > + * c-file-style: "BSD" > + * c-basic-offset: 4 > + * indent-tabs-mode: nil > + * End: > + */ > diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h > index 9180be5e86..a67237942d 100644 > --- a/xen/include/public/arch-arm.h > +++ b/xen/include/public/arch-arm.h > @@ -315,6 +315,7 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t); > #define XEN_DOMCTL_CONFIG_TEE_OPTEE 1 > > #define XEN_DOMCTL_CONFIG_SCI_NONE 0 > +#define XEN_DOMCTL_CONFIG_SCI_SCMI_SMC 1 > > struct xen_arch_domainconfig { > /* IN/OUT */
Hi Oleksandr, On Fri, Dec 17, 2021 at 01:35:35PM +0200, Oleksandr wrote: > > On 14.12.21 11:34, Oleksii Moisieiev wrote: > > > Hi Oleksii > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > This allows devices from the Domains to work with clocks, resets and > > power-domains without access to CPG. > > > > The following features are implemented: > > - request SCMI channels from ATF and pass channels to Domains; > > - set device permissions for Domains based on the Domain partial > > device-tree. Devices with permissions are able to work with clocks, > > resets and power-domains via SCMI; > > - redirect scmi messages from Domains to ATF. > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > --- > > xen/arch/arm/Kconfig | 2 + > > xen/arch/arm/sci/Kconfig | 10 + > > xen/arch/arm/sci/Makefile | 1 + > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > xen/include/public/arch-arm.h | 1 + > > 5 files changed, 809 insertions(+) > > create mode 100644 xen/arch/arm/sci/Kconfig > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > index 186e1db389..02d96c6cfc 100644 > > --- a/xen/arch/arm/Kconfig > > +++ b/xen/arch/arm/Kconfig > > @@ -114,6 +114,8 @@ config SCI > > support. It allows guests to control system resourcess via one of > > SCI mediators implemented in XEN. > > +source "arch/arm/sci/Kconfig" > > + > > endmenu > > menu "ARM errata workaround via the alternative framework" > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > new file mode 100644 > > index 0000000000..9563067ddc > > --- /dev/null > > +++ b/xen/arch/arm/sci/Kconfig > > @@ -0,0 +1,10 @@ > > +config SCMI_SMC > > + bool "Enable SCMI-SMC mediator driver" > > + default n > > + depends on SCI > > + ---help--- > > + > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > + This feature allows drivers from Domains to work with System > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > + for communication. > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > index 837dc7492b..67f2611872 100644 > > --- a/xen/arch/arm/sci/Makefile > > +++ b/xen/arch/arm/sci/Makefile > > @@ -1 +1,2 @@ > > obj-y += sci.o > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > new file mode 100644 > > index 0000000000..2eb01ea82d > > --- /dev/null > > +++ b/xen/arch/arm/sci/scmi_smc.c > > @@ -0,0 +1,795 @@ > > +/* > > + * xen/arch/arm/sci/scmi_smc.c > > + * > > + * SCMI mediator driver, using SCP as transport. > > + * > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > + * Copyright (C) 2021, EPAM Systems. > > + * > > + * This program is free software; you can redistribute it and/or modify > > + * it under the terms of the GNU General Public License as published by > > + * the Free Software Foundation; either version 2 of the License, or > > + * (at your option) any later version. > > + * > > + * This program is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > + * GNU General Public License for more details. > > + */ > > + > > +#include <asm/sci/sci.h> > > +#include <asm/smccc.h> > > +#include <asm/io.h> > > +#include <xen/bitops.h> > > +#include <xen/config.h> > > +#include <xen/sched.h> > > +#include <xen/device_tree.h> > > +#include <xen/iocap.h> > > +#include <xen/init.h> > > +#include <xen/err.h> > > +#include <xen/lib.h> > > +#include <xen/list.h> > > +#include <xen/mm.h> > > +#include <xen/string.h> > > +#include <xen/time.h> > > +#include <xen/vmap.h> > > + > > +#define SCMI_BASE_PROTOCOL 0x10 > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > + > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > +#define SCMI_SUCCESS 0 > > +#define SCMI_NOT_SUPPORTED (-1) > > +#define SCMI_INVALID_PARAMETERS (-2) > > +#define SCMI_DENIED (-3) > > +#define SCMI_NOT_FOUND (-4) > > +#define SCMI_OUT_OF_RANGE (-5) > > +#define SCMI_BUSY (-6) > > +#define SCMI_COMMS_ERROR (-7) > > +#define SCMI_GENERIC_ERROR (-8) > > +#define SCMI_HARDWARE_ERROR (-9) > > +#define SCMI_PROTOCOL_ERROR (-10) > > + > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > + > > +#define SCMI_SMC_ID "arm,smc-id" > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > +#define SCMI_SHMEM "shmem" > > + > > +#define HYP_CHANNEL 0x0 > > + > > +#define HDR_ID GENMASK(7,0) > > +#define HDR_TYPE GENMASK(9, 8) > > +#define HDR_PROTO GENMASK(17, 10) > > + > > +/* SCMI protocol, refer to section 4.2.2.2 (DEN0056C) */ > > +#define MSG_N_AGENTS_MASK GENMASK(15, 8) > > + > > +#define FIELD_GET(_mask, _reg)\ > > + ((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1))) > > +#define FIELD_PREP(_mask, _val)\ > > + (((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask)) > > + > > +typedef struct scmi_msg_header { > > + uint8_t id; > > + uint8_t type; > > + uint8_t protocol; > > +} scmi_msg_header_t; > > + > > +typedef struct scmi_perms_tx { > > + uint32_t agent_id; > > + uint32_t device_id; > > + uint32_t flags; > > +} scmi_perms_tx_t; > > + > > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE BIT(0, UL) > > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_ERROR BIT(1, UL) > > + > > +#define SCMI_ALLOW_ACCESS BIT(0, UL) > > + > > +struct scmi_shared_mem { > > + uint32_t reserved; > > + uint32_t channel_status; > > + uint32_t reserved1[2]; > > + uint32_t flags; > > + uint32_t length; > > + uint32_t msg_header; > > + uint8_t msg_payload[]; > > +}; > > + > > +struct scmi_channel { > > + int chan_id; > > + int agent_id; > > + uint32_t func_id; > > + int domain_id; > > + uint64_t paddr; > > + struct scmi_shared_mem *shmem; > > + spinlock_t lock; > > + struct list_head list; > > +}; > > + > > +struct scmi_data { > > + struct list_head channel_list; > > + spinlock_t channel_list_lock; > > + bool initialized; > > + u64 shmem_addr, shmem_size; > > +}; > > + > > +static struct scmi_data scmi_data; > > + > > +/* > > + * pack_scmi_header() - packs and returns 32-bit header > > + * > > + * @hdr: pointer to header containing all the information on message id, > > + * protocol id and type id. > > + * > > + * Return: 32-bit packed message header to be sent to the platform. > > + */ > > +static inline uint32_t pack_scmi_header(scmi_msg_header_t *hdr) > > +{ > > + return FIELD_PREP(HDR_ID, hdr->id) | > > + FIELD_PREP(HDR_TYPE, hdr->type) | > > + FIELD_PREP(HDR_PROTO, hdr->protocol); > > +} > > + > > +/* > > + * unpack_scmi_header() - unpacks and records message and protocol id > > + * > > + * @msg_hdr: 32-bit packed message header sent from the platform > > + * @hdr: pointer to header to fetch message and protocol id. > > + */ > > +static inline void unpack_scmi_header(uint32_t msg_hdr, scmi_msg_header_t *hdr) > > +{ > > + hdr->id = FIELD_GET(HDR_ID, msg_hdr); > > + hdr->type = FIELD_GET(HDR_TYPE, msg_hdr); > > + hdr->protocol = FIELD_GET(HDR_PROTO, msg_hdr); > > +} > > + > > +static inline int channel_is_free(struct scmi_channel *chan_info) > > +{ > > + return ( chan_info->shmem->channel_status > > + & SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE ) ? 0 : -EBUSY; > > +} > > + > > +static int send_smc_message(struct scmi_channel *chan_info, > > + scmi_msg_header_t *hdr, void *data, int len) > > +{ > > + struct arm_smccc_res resp; > > + int ret; > > + > > + printk(XENLOG_DEBUG "scmi: status =%d len=%d\n", > > + chan_info->shmem->channel_status, len); > > + printk(XENLOG_DEBUG "scmi: header id = %d type = %d, proto = %d\n", > > + hdr->id, hdr->type, hdr->protocol); > > + > > + ret = channel_is_free(chan_info); > > + if ( IS_ERR_VALUE(ret) ) > > + return ret; > > + > > + chan_info->shmem->channel_status = 0x0; > > + /* Writing 0x0 right now, but SCMI_SHMEM_FLAG_INTR_ENABLED can be set */ > > + chan_info->shmem->flags = 0x0; > > + chan_info->shmem->length = sizeof(chan_info->shmem->msg_header) + len; > > + chan_info->shmem->msg_header = pack_scmi_header(hdr); > > + > > + printk(XENLOG_DEBUG "scmi: Writing to shmem address %p\n", > > + chan_info->shmem); > > + if ( len > 0 && data ) > > + memcpy((void *)(chan_info->shmem->msg_payload), data, len); > > + > > + arm_smccc_smc(chan_info->func_id, 0, 0, 0, 0, 0, 0, chan_info->chan_id, > > + &resp); > > + > > + printk(XENLOG_DEBUG "scmi: scmccc_smc response %d\n", (int)(resp.a0)); > > + > > + if ( resp.a0 ) > > + return -EOPNOTSUPP; > > + > > + return 0; > > +} > > + > > +static int check_scmi_status(int scmi_status) > > +{ > > + if ( scmi_status == SCMI_SUCCESS ) > > + return 0; > > + > > + printk(XENLOG_DEBUG "scmi: Error received: %d\n", scmi_status); > > + > > + switch ( scmi_status ) > > + { > > + case SCMI_NOT_SUPPORTED: > > + return -EOPNOTSUPP; > > + case SCMI_INVALID_PARAMETERS: > > + return -EINVAL; > > + case SCMI_DENIED: > > + return -EACCES; > > + case SCMI_NOT_FOUND: > > + return -ENOENT; > > + case SCMI_OUT_OF_RANGE: > > + return -ERANGE; > > + case SCMI_BUSY: > > + return -EBUSY; > > + case SCMI_COMMS_ERROR: > > + return -ENOTCONN; > > + case SCMI_GENERIC_ERROR: > > + return -EIO; > > + case SCMI_HARDWARE_ERROR: > > + return -ENXIO; > > + case SCMI_PROTOCOL_ERROR: > > + return -EBADMSG; > > Probably we should add a default: here? > Thanks, I will add the default, so fucntion will look like this: static int check_scmi_status(int scmi_status) { ... switch ( scmi_status ) { ... default: return -EINVAL; } } > > > + } > > + > > + return -EINVAL; > > +} > > + > > +static int get_smc_response(struct scmi_channel *chan_info, > > + scmi_msg_header_t *hdr, void *data, int len) > > +{ > > + int recv_len; > > + int ret; > > + > > + printk(XENLOG_DEBUG "scmi: get smc responce msgid %d\n", hdr->id); > > + > > + ret = channel_is_free(chan_info); > > + if ( IS_ERR_VALUE(ret) ) > > + return ret; > > + > > + recv_len = chan_info->shmem->length - sizeof(chan_info->shmem->msg_header); > > + > > + if ( recv_len < 0 ) > > + { > > + printk(XENLOG_ERR > > + "scmi: Wrong size of smc message. Data may be invalid\n"); > > + return -EINVAL; > > + } > > + > > + if ( recv_len > len ) > > + { > > + printk(XENLOG_ERR > > + "scmi: Not enough buffer for message %d, expecting %d\n", > > + recv_len, len); > > + return -EINVAL; > > + } > > + > > + unpack_scmi_header(chan_info->shmem->msg_header, hdr); > > + > > + if ( recv_len > 0 ) > > + { > > + memcpy(data, chan_info->shmem->msg_payload, recv_len); > > + } > > + > > + return 0; > > +} > > + > > +static int do_smc_xfer(struct scmi_channel *channel, scmi_msg_header_t *hdr, void *tx_data, int tx_size, > > + void *rx_data, int rx_size) > > +{ > > + int ret = 0; > > + > > + if ( !hdr ) > > + return -EINVAL; > > + > > + spin_lock(&channel->lock); > > + > > + ret = send_smc_message(channel, hdr, tx_data, tx_size); > > + if ( ret ) > > + goto clean; > > + > > + ret = get_smc_response(channel, hdr, rx_data, rx_size); > > +clean: > > + spin_unlock(&channel->lock); > > + > > + return ret; > > +} > > + > > +static struct scmi_channel *get_channel_by_id(uint8_t chan_id) > > +{ > > + struct scmi_channel *curr; > > + bool found = false; > > + > > + spin_lock(&scmi_data.channel_list_lock); > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > + if ( curr->chan_id == chan_id ) > > + { > > + found = true; > > + break; > > + } > > + > > + spin_unlock(&scmi_data.channel_list_lock); > > + if ( found ) > > + return curr; > > + > > + return NULL; > > +} > > + > > +static struct scmi_channel *get_channel_by_domain(uint8_t domain_id) > > +{ > > + struct scmi_channel *curr; > > + bool found = false; > > + > > + spin_lock(&scmi_data.channel_list_lock); > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > + if ( curr->domain_id == domain_id ) > > + { > > + found = true; > > + break; > > + } > > + > > + spin_unlock(&scmi_data.channel_list_lock); > > + if ( found ) > > + return curr; > > + > > + return NULL; > > +} > > + > > +static struct scmi_channel *aquire_scmi_channel(int domain_id) > > +{ > > + struct scmi_channel *curr; > > + bool found = false; > > + > > + ASSERT(domain_id != DOMID_INVALID && domain_id >= 0); > > + > > + spin_lock(&scmi_data.channel_list_lock); > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > + if ( (curr->domain_id == DOMID_INVALID) > > + && (curr->chan_id != HYP_CHANNEL) ) > > + { > > + curr->domain_id = domain_id; > > + found = true; > > + break; > > + } > > + > > + spin_unlock(&scmi_data.channel_list_lock); > > + if ( found ) > > + return curr; > > + > > + return NULL; > > +} > > + > > +static void relinquish_scmi_channel(struct scmi_channel *channel) > > +{ > > + spin_lock(&scmi_data.channel_list_lock); > > + ASSERT(channel != NULL); > > + channel->domain_id = DOMID_INVALID; > > + spin_unlock(&scmi_data.channel_list_lock); > > +} > > + > > +static struct scmi_channel *smc_create_channel(uint8_t chan_id, > > + uint32_t func_id, uint64_t addr) > > +{ > > + struct scmi_channel *channel; > > + mfn_t mfn; > > + > > + channel = get_channel_by_id(chan_id); > > + if ( channel ) > > + return ERR_PTR(EEXIST); > > + > > + channel = xmalloc(struct scmi_channel); > > + if ( !channel ) > > + return ERR_PTR(ENOMEM); > > + > > + channel->chan_id = chan_id; > > + channel->func_id = func_id; > > + channel->domain_id = DOMID_INVALID; > > + mfn = maddr_to_mfn(addr); > > + channel->shmem = vmap(&mfn, 1); > > + if ( !channel->shmem ) > > + { > > + xfree(channel); > > + return ERR_PTR(ENOMEM); > > + } > > + > > + printk(XENLOG_DEBUG "scmi: Got shmem after vmap %p\n", channel->shmem); > > + channel->paddr = addr; > > + channel->shmem->channel_status = SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE; > > + spin_lock_init(&channel->lock); > > + spin_lock(&scmi_data.channel_list_lock); > > + list_add(&channel->list, &scmi_data.channel_list); > > + spin_unlock(&scmi_data.channel_list_lock); > > + return channel; > > +} > > + > > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > > +{ > > + return iomem_permit_access(d, paddr_to_pfn(addr), > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > +} > > + > > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > > + uint64_t len) > > +{ > > + return iomem_deny_access(d, paddr_to_pfn(addr), > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > +} > > I wonder, why we need an extra level of indirection here. And if this is > really needed, I wonder why map(unmap)_memory* name was chosen, as there is > no memory mapping/unmapping really happens here. > I've added extra indirection to hide math like paddr_to_pfn(PAGE_ALIGN(addr + len -1) so you don't have to math in each call. unmap_memory_from_domain called from 2 places, so I moved both calls to separate function. Although, I agree that map/unmap is not perfect name. I consider renaming it to mem_permit_acces and mam_deny_access. > > > + > > +static int dt_update_domain_range(struct domain *d, uint64_t addr, > > + uint64_t size) > > Looks like *d is not used in this function. > Yep I will remove it. Thanks. > > > +{ > > + struct dt_device_node *shmem_node; > > + __be32 *hw_reg; > > + const struct dt_property *pp; > > + uint32_t len; > > + > > + shmem_node = dt_find_compatible_node(NULL, NULL, "arm,scmi-shmem"); > > + > > + if ( !shmem_node ) > > + { > > + printk(XENLOG_ERR "scmi: Unable to find %s node in DT\n", SCMI_SHMEM); > > + return -EINVAL; > > + } > > + > > + pp = dt_find_property(shmem_node, "reg", &len); > > + if ( !pp ) > > + { > > + printk(XENLOG_ERR "scmi: Unable to find regs entry in shmem node\n"); > > + return -ENOENT; > > + } > > + > > + hw_reg = pp->value; > > + dt_set_range(&hw_reg, shmem_node, addr, size); > > + > > + return 0; > > +} > > + > > +static void free_channel_list(void) > > +{ > > + struct scmi_channel *curr, *_curr; > > + > > + spin_lock(&scmi_data.channel_list_lock); > > + list_for_each_entry_safe (curr, _curr, &scmi_data.channel_list, list) > > + { > > + vunmap(curr->shmem); > > + list_del(&curr->list); > > + xfree(curr); > > + } > > + > > + spin_unlock(&scmi_data.channel_list_lock); > > +} > > + > > +static __init bool scmi_probe(struct dt_device_node *scmi_node) > > > Generic question to consider ... > If probe fails for some reason (so we cannot use mediator in Xen) what > should we do with SCMI nodes in domain's device-tree (leave as is, drop, > etc)? > If probe fails - then the devices, which were configured to use scmi clocks\resets\power-domains will not work properly. If Domain configuration do not use SCMI - then it will not have the scmi node. So from my standpoint, no additional check is needed before copying scmi nodes to Domain device-tree. > > > +{ > > + struct dt_device_node *shmem_node; > > + int ret, i; > > + struct scmi_channel *channel, *agent_channel; > > + int n_agents; > > + scmi_msg_header_t hdr; > > + struct rx_t { > > + int32_t status; > > + uint32_t attributes; > > + } rx; > > + > > + uint32_t func_id; > > + > > + ASSERT(scmi_node != NULL); > > + > > + INIT_LIST_HEAD(&scmi_data.channel_list); > > + spin_lock_init(&scmi_data.channel_list_lock); > > + > > + if ( !dt_property_read_u32(scmi_node, SCMI_SMC_ID, &func_id) ) > > + { > > + printk(XENLOG_ERR "scmi: Unable to read smc-id from DT\n"); > > + return false; > > + } > > + > > + shmem_node = dt_find_node_by_name(NULL, SCMI_SHARED_MEMORY); > > + if ( IS_ERR_OR_NULL(shmem_node) ) > > + { > > + printk(XENLOG_ERR > > + "scmi: Device tree error, can't parse shmem phandle %ld\n", > > + PTR_ERR(shmem_node)); > > + return false; > > + } > > + > > + ret = dt_device_get_address(shmem_node, 0, &scmi_data.shmem_addr, > > + &scmi_data.shmem_size); > > + if ( IS_ERR_VALUE(ret) ) > > + return false; > > + > > + channel = smc_create_channel(HYP_CHANNEL, func_id, scmi_data.shmem_addr); > > + if ( IS_ERR(channel) ) > > + return false; > > + > > + hdr.id = SCMI_BASE_PROTOCOL_ATTIBUTES; > > + hdr.type = 0; > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > + > > + ret = do_smc_xfer(channel, &hdr, NULL, 0, &rx, sizeof(rx)); > > + if ( ret ) > > + goto clean; > > + > > + ret = check_scmi_status(rx.status); > > + if ( ret ) > > + goto clean; > > + > > + n_agents = FIELD_GET(MSG_N_AGENTS_MASK, rx.attributes); > > + printk(XENLOG_DEBUG "scmi: Got agent count %d\n", n_agents); > > + > > + n_agents = (n_agents > scmi_data.shmem_size / PAGE_SIZE) ? > > + scmi_data.shmem_size / PAGE_SIZE : n_agents; > > + > > + for ( i = 1; i < n_agents; i++ ) > > + { > > + uint32_t tx_agent_id = 0xFFFFFFFF; > > + struct { > > + int32_t status; > > + uint32_t agent_id; > > + char name[16]; > > + } da_rx; > > + > > + agent_channel = smc_create_channel(i, func_id, scmi_data.shmem_addr + > > + i * PAGE_SIZE); > > + if ( IS_ERR(agent_channel) ) > > + { > > + ret = PTR_ERR(agent_channel); > > + goto clean; > > + } > > + > > + hdr.id = SCMI_BASE_DISCOVER_AGENT; > > + hdr.type = 0; > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > + > > + ret = do_smc_xfer(agent_channel, &hdr, &tx_agent_id, > > + sizeof(tx_agent_id), &da_rx, sizeof(da_rx)); > > + if ( ret ) > > + goto clean; > > + > > + ret = check_scmi_status(da_rx.status); > > + if ( ret ) > > + goto clean; > > + > > + printk(XENLOG_DEBUG "scmi: status=0x%x id=0x%x name=%s\n", > > + da_rx.status, da_rx.agent_id, da_rx.name); > > + > > + agent_channel->agent_id = da_rx.agent_id; > > + } > > + > > + scmi_data.initialized = true; > > + return true; > > + > > +clean: > > + free_channel_list(); > > + return ret == 0; > > +} > > + > > +static int scmi_domain_init(struct domain *d) > > +{ > > + struct scmi_channel *channel; > > + int ret; > > + > > + if ( !scmi_data.initialized ) > > + return 0; > > + > > + channel = aquire_scmi_channel(d->domain_id); > > + if ( IS_ERR_OR_NULL(channel) ) > > + return -ENOENT; > > + > > + printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = %d" > > + "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, > > + channel->paddr); > > > It seems this breaks build on Arm32: > > scmi_smc.c: In function ‘scmi_domain_init’: > /home/otyshchenko/xen/xen/include/xen/config.h:53:24: error: format ‘%lx’ > expects argument of type ‘long unsigned int’, but argument 4 has type > ‘uint64_t {aka long long unsigned int}’ [-Werror=format=] > #define XENLOG_INFO "<2>" > ^ > scmi_smc.c:569:12: note: in expansion of macro ‘XENLOG_INFO’ > printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = > %d" > ^~~~~~~~~~~ > scmi_smc.c:570:25: note: format string is defined here > "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, > ~~^ > %llx > Thanks a lot. I will work through all buids in v2. > > > > + > > + if ( is_hardware_domain(d) ) > > + { > > + ret = map_memory_to_domain(d, scmi_data.shmem_addr, > > + scmi_data.shmem_size); > > + if ( IS_ERR_VALUE(ret) ) > > + goto error; > > + > > + ret = dt_update_domain_range(d, channel->paddr, PAGE_SIZE); > > + if ( IS_ERR_VALUE(ret) ) > > + { > > + int rc = unmap_memory_from_domain(d, scmi_data.shmem_addr, > > + scmi_data.shmem_size); > > + if ( rc ) > > + printk(XENLOG_ERR "Unable to unmap_memory_from_domain\n"); > > + > > + goto error; > > + } > > + } > > + > > + d->arch.sci = channel; > > + > > + return 0; > > +error: > > + relinquish_scmi_channel(channel); > > + > > + return ret; > > +} > > + > > +static int scmi_add_device_by_devid(struct domain *d, uint32_t scmi_devid) > > +{ > > + struct scmi_channel *channel, *agent_channel; > > + scmi_msg_header_t hdr; > > + scmi_perms_tx_t tx; > > + struct rx_t { > > + int32_t status; > > + uint32_t attributes; > > + } rx; > > + int ret; > > + > > + if ( !scmi_data.initialized ) > > + return 0; > > + > > + printk(XENLOG_DEBUG "scmi: scmi_devid = %d\n", scmi_devid); > > + > > + agent_channel = get_channel_by_domain(d->domain_id); > > + if ( IS_ERR_OR_NULL(agent_channel) ) > > + return PTR_ERR(agent_channel); > > + > > + channel = get_channel_by_id(HYP_CHANNEL); > > + if ( IS_ERR_OR_NULL(channel) ) > > + return PTR_ERR(channel); > > + > > + hdr.id = SCMI_BASE_SET_DEVICE_PERMISSIONS; > > + hdr.type = 0; > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > + > > + tx.agent_id = agent_channel->agent_id; > > + tx.device_id = scmi_devid; > > + tx.flags = SCMI_ALLOW_ACCESS; > > + > > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(&rx)); > > + if ( IS_ERR_VALUE(ret) ) > > + return ret; > > + > > + ret = check_scmi_status(rx.status); > > + if ( IS_ERR_VALUE(ret) ) > > + return ret; > > + > > + return 0; > > +} > > + > > +static int scmi_add_dt_device(struct domain *d, struct dt_device_node *dev) > > +{ > > + uint32_t scmi_devid; > > + > > + if ( (!scmi_data.initialized) || (!d->arch.sci) ) > > + return 0; > > + > > + if ( !dt_property_read_u32(dev, "scmi_devid", &scmi_devid) ) > > + return 0; > > + > > + printk(XENLOG_INFO "scmi: dt_node = %s\n", dt_node_full_name(dev)); > > + > > + return scmi_add_device_by_devid(d, scmi_devid); > > +} > > + > > +static int scmi_relinquish_resources(struct domain *d) > > +{ > > + int ret; > > + struct scmi_channel *channel, *agent_channel; > > + scmi_msg_header_t hdr; > > + struct reset_agent_tx { > > + uint32_t agent_id; > > + uint32_t flags; > > + } tx; > > + uint32_t rx; > > + > > + if ( !d->arch.sci ) > > + return 0; > > + > > + agent_channel = d->arch.sci; > > + > > + spin_lock(&agent_channel->lock); > > + tx.agent_id = agent_channel->agent_id; > > + spin_unlock(&agent_channel->lock); > > + > > + channel = get_channel_by_id(HYP_CHANNEL); > > + if ( !channel ) > > + { > > + printk(XENLOG_ERR > > + "scmi: Unable to get Hypervisor scmi channel for domain %d\n", > > + d->domain_id); > > + return -EINVAL; > > + } > > + > > + hdr.id = SCMI_BASE_RESET_AGENT_CONFIGURATION; > > + hdr.type = 0; > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > + > > + tx.flags = 0; > > + > > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(rx)); > > + if ( ret ) > > + return ret; > > + > > + ret = check_scmi_status(rx); > > + > > + return ret; > > +} > > + > > +static void scmi_domain_destroy(struct domain *d) > > +{ > > + struct scmi_channel *channel; > > + > > + if ( !d->arch.sci ) > > + return; > > + > > + channel = d->arch.sci; > > + spin_lock(&channel->lock); > > + > > + relinquish_scmi_channel(channel); > > + printk(XENLOG_DEBUG "scmi: Free domain %d\n", d->domain_id); > > + > > + d->arch.sci = NULL; > > + > > + unmap_memory_from_domain(d, channel->paddr, PAGE_SIZE); > I didn't manage to find where corresponding map_memory_from_domain() is > called for a non-hardware domain (it seems that scmi_domain_init() only > directly handles hardware domain case). > Or perhaps, it is called indirectly at do_domctl(): case > XEN_DOMCTL_iomem_permission:? > You were right. We need to permit access only for the Hardware domain. > > I wonder, do we really need to call this here? Taking into the account that > unmap_memory_from_domain() doesn't actually unmap anything, but only removes > a range from the iomem_caps rangeset > for the domain to be destroyed and all involved rangesets (including > iomem_caps) will be removed soon at rangeset_domain_destroy() anyway. Or I > missed something? > unmap_memory_from_domain is also used in scmi_domain_init if error occured after successfull map_memory_to_domain call. But this is a good point. I think it will be a good idea to make unmap_memory_from_domain in scmi_domain_destroy only for hardware domain. And make sure that for Software domains range is unmapped using XEN_DOMCTL_iomem_permission call. I'll make those changes in v2. > > > + spin_unlock(&channel->lock); > > + return; > > empty return could be dropped, I think. > Agree. Will be fixed. > > > +} > > + > > +static bool scmi_handle_call(struct domain *d, void *args) > > +{ > > + bool res = false; > > + struct scmi_channel *agent_channel; > > + struct arm_smccc_res resp; > > + struct cpu_user_regs *regs = args; > > + > > + if ( !d->arch.sci ) > > + return false; > > + > > + agent_channel = d->arch.sci; > > + spin_lock(&agent_channel->lock); > > + > > + if ( agent_channel->func_id != regs->x0 ) > > This also breaks build on Arm32: > > scmi_smc.c: In function ‘scmi_handle_call’: > scmi_smc.c:736:42: error: ‘struct cpu_user_regs’ has no member named ‘x0’; > did you mean ‘r0’? > if ( agent_channel->func_id != regs->x0 ) > ^~ > r0 > cc1: all warnings being treated as errors > Thanks, I will fix it in v2. > *** > > BTW, I noticed that xen/arch/arm/traps.c contains the following construct, > probably we might want something similar here? > > #ifdef CONFIG_ARM_64 > #define HYPERCALL_RESULT_REG(r) (r)->x0 > [snip] > #else > #define HYPERCALL_RESULT_REG(r) (r)->r0 > [snip] > #endif > > > This RFC patch series, so I think, there is no serious issues at the moment, > this is rather to let you know for the future (when you drop RFC tag). > I have to admit, I often forget to build-test on Arm32 also)) > Thank you for understanding. But will try to cover all architectures. > > > + { > > + printk(XENLOG_ERR "scmi: func_id mismatch, exiting\n"); > > + goto unlock; > > + } > > + > > + arm_smccc_smc(agent_channel->func_id, 0, 0, 0, 0, 0, 0, > > + agent_channel->chan_id, &resp); > > + > > + set_user_reg(regs, 0, resp.a0); > > + set_user_reg(regs, 1, resp.a1); > > + set_user_reg(regs, 2, resp.a2); > > + set_user_reg(regs, 3, resp.a3); > > + res = true; > > +unlock: > > + spin_unlock(&agent_channel->lock); > > + > > + return res; > > +} > > + > > +static int scmi_get_channel_paddr(void *scmi_ops, > > + struct xen_arch_domainconfig *config) > > +{ > > + struct scmi_channel *agent_channel = scmi_ops; > > + > > + if ( !agent_channel ) > > + return -EINVAL; > > + > > + config->sci_agent_paddr = agent_channel->paddr; > > + return 0; > > +} > > + > > +static const struct dt_device_match scmi_smc_match[] __initconst = > > +{ > > + DT_MATCH_SCMI_SMC, > > + { /* sentinel */ }, > > +}; > > + > > +static const struct sci_mediator_ops scmi_ops = > > +{ > > + .probe = scmi_probe, > > + .domain_init = scmi_domain_init, > > + .domain_destroy = scmi_domain_destroy, > > + .add_dt_device = scmi_add_dt_device, > > + .relinquish_resources = scmi_relinquish_resources, > > + .handle_call = scmi_handle_call, > > + .get_channel_info = scmi_get_channel_paddr > > +}; > > + > > +REGISTER_SCI_MEDIATOR(scmi_smc, "SCMI-SMC", XEN_DOMCTL_CONFIG_SCI_SCMI_SMC, > > + scmi_smc_match, &scmi_ops); > > + > > +/* > > + * Local variables: > > + * mode: C > > + * c-file-style: "BSD" > > + * c-basic-offset: 4 > > + * indent-tabs-mode: nil > > + * End: > > + */ > > diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h > > index 9180be5e86..a67237942d 100644 > > --- a/xen/include/public/arch-arm.h > > +++ b/xen/include/public/arch-arm.h > > @@ -315,6 +315,7 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t); > > #define XEN_DOMCTL_CONFIG_TEE_OPTEE 1 > > #define XEN_DOMCTL_CONFIG_SCI_NONE 0 > > +#define XEN_DOMCTL_CONFIG_SCI_SCMI_SMC 1 > > struct xen_arch_domainconfig { > > /* IN/OUT */ > > -- > Regards, > > Oleksandr Tyshchenko >
Hi, On 17/12/2021 13:23, Oleksii Moisieiev wrote: >>> +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) >>> +{ >>> + return iomem_permit_access(d, paddr_to_pfn(addr), >>> + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); >>> +} >>> + >>> +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, >>> + uint64_t len) >>> +{ >>> + return iomem_deny_access(d, paddr_to_pfn(addr), >>> + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); >>> +} >> >> I wonder, why we need an extra level of indirection here. And if this is >> really needed, I wonder why map(unmap)_memory* name was chosen, as there is >> no memory mapping/unmapping really happens here. >> > > I've added extra indirection to hide math like > paddr_to_pfn(PAGE_ALIGN(addr + len -1) > so you don't have to math in each call. unmap_memory_from_domain called > from 2 places, so I moved both calls to separate function. > Although, I agree that map/unmap is not perfect name. I consider > renaming it to mem_permit_acces and mam_deny_access. I haven't looked at the rest of the series. But this discussion caught my eye. This code implies that the address is page-aligned but the length not. Is that intended? That said, if you give permission to the domain on a full page then it means it may be able to access address it should not. Can you explain why this is fine? Cheers,
Hi Julien, On Fri, Dec 17, 2021 at 01:37:35PM +0000, Julien Grall wrote: > Hi, > > On 17/12/2021 13:23, Oleksii Moisieiev wrote: > > > > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > > > > +{ > > > > + return iomem_permit_access(d, paddr_to_pfn(addr), > > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > > +} > > > > + > > > > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > > > > + uint64_t len) > > > > +{ > > > > + return iomem_deny_access(d, paddr_to_pfn(addr), > > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > > +} > > > > > > I wonder, why we need an extra level of indirection here. And if this is > > > really needed, I wonder why map(unmap)_memory* name was chosen, as there is > > > no memory mapping/unmapping really happens here. > > > > > > > I've added extra indirection to hide math like > > paddr_to_pfn(PAGE_ALIGN(addr + len -1) > > so you don't have to math in each call. unmap_memory_from_domain called > > from 2 places, so I moved both calls to separate function. > > Although, I agree that map/unmap is not perfect name. I consider > > renaming it to mem_permit_acces and mam_deny_access. > > I haven't looked at the rest of the series. But this discussion caught my > eye. This code implies that the address is page-aligned but the length not. > Is that intended? > > That said, if you give permission to the domain on a full page then it means > it may be able to access address it should not. Can you explain why this is > fine? > The idea was that xen receives some memory from the dt_node linux,scmi_mem, then we split memory between the agents, so each agent get 1 page (we allocate 0x10 pages right now). But this is a good point, I think we should check length to be aligned before making any permission changes. I will add it in v2. Thank you for remark. Oleksii.
On 17/12/2021 13:58, Oleksii Moisieiev wrote: > Hi Julien, Hi, > On Fri, Dec 17, 2021 at 01:37:35PM +0000, Julien Grall wrote: >> Hi, >> >> On 17/12/2021 13:23, Oleksii Moisieiev wrote: >>>>> +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) >>>>> +{ >>>>> + return iomem_permit_access(d, paddr_to_pfn(addr), >>>>> + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); >>>>> +} >>>>> + >>>>> +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, >>>>> + uint64_t len) >>>>> +{ >>>>> + return iomem_deny_access(d, paddr_to_pfn(addr), >>>>> + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); >>>>> +} >>>> >>>> I wonder, why we need an extra level of indirection here. And if this is >>>> really needed, I wonder why map(unmap)_memory* name was chosen, as there is >>>> no memory mapping/unmapping really happens here. >>>> >>> >>> I've added extra indirection to hide math like >>> paddr_to_pfn(PAGE_ALIGN(addr + len -1) >>> so you don't have to math in each call. unmap_memory_from_domain called >>> from 2 places, so I moved both calls to separate function. >>> Although, I agree that map/unmap is not perfect name. I consider >>> renaming it to mem_permit_acces and mam_deny_access. >> >> I haven't looked at the rest of the series. But this discussion caught my >> eye. This code implies that the address is page-aligned but the length not. >> Is that intended? >> >> That said, if you give permission to the domain on a full page then it means >> it may be able to access address it should not. Can you explain why this is >> fine? >> > > The idea was that xen receives some memory from the dt_node linux,scmi_mem, > then we split memory between the agents, so each agent get 1 page (we > allocate 0x10 pages right now). Thanks for the clarification. Does this imply the guest will be able to write message directly to the firmware? If so, this brings a few more questions: 1) What will the guest write in it? Can it contains addresses? 2) What are the expected memory attribute for the regions? 3) What's the threat model for the firmware? Will it verify every request? Cheers,
On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > This is the implementation of SCI interface, called SCMI-SMC driver, > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > This allows devices from the Domains to work with clocks, resets and > power-domains without access to CPG. > > The following features are implemented: > - request SCMI channels from ATF and pass channels to Domains; > - set device permissions for Domains based on the Domain partial > device-tree. Devices with permissions are able to work with clocks, > resets and power-domains via SCMI; > - redirect scmi messages from Domains to ATF. > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > --- > xen/arch/arm/Kconfig | 2 + > xen/arch/arm/sci/Kconfig | 10 + > xen/arch/arm/sci/Makefile | 1 + > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > xen/include/public/arch-arm.h | 1 + > 5 files changed, 809 insertions(+) > create mode 100644 xen/arch/arm/sci/Kconfig > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > index 186e1db389..02d96c6cfc 100644 > --- a/xen/arch/arm/Kconfig > +++ b/xen/arch/arm/Kconfig > @@ -114,6 +114,8 @@ config SCI > support. It allows guests to control system resourcess via one of > SCI mediators implemented in XEN. > > +source "arch/arm/sci/Kconfig" > + > endmenu > > menu "ARM errata workaround via the alternative framework" > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > new file mode 100644 > index 0000000000..9563067ddc > --- /dev/null > +++ b/xen/arch/arm/sci/Kconfig > @@ -0,0 +1,10 @@ > +config SCMI_SMC > + bool "Enable SCMI-SMC mediator driver" > + default n > + depends on SCI > + ---help--- > + > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > + This feature allows drivers from Domains to work with System > + Controllers (such as power,resets,clock etc.). SCP is used as transport > + for communication. > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > index 837dc7492b..67f2611872 100644 > --- a/xen/arch/arm/sci/Makefile > +++ b/xen/arch/arm/sci/Makefile > @@ -1 +1,2 @@ > obj-y += sci.o > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > new file mode 100644 > index 0000000000..2eb01ea82d > --- /dev/null > +++ b/xen/arch/arm/sci/scmi_smc.c > @@ -0,0 +1,795 @@ > +/* > + * xen/arch/arm/sci/scmi_smc.c > + * > + * SCMI mediator driver, using SCP as transport. > + * > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > + * Copyright (C) 2021, EPAM Systems. > + * > + * This program is free software; you can redistribute it and/or modify > + * it under the terms of the GNU General Public License as published by > + * the Free Software Foundation; either version 2 of the License, or > + * (at your option) any later version. > + * > + * This program is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + */ > + > +#include <asm/sci/sci.h> > +#include <asm/smccc.h> > +#include <asm/io.h> > +#include <xen/bitops.h> > +#include <xen/config.h> > +#include <xen/sched.h> > +#include <xen/device_tree.h> > +#include <xen/iocap.h> > +#include <xen/init.h> > +#include <xen/err.h> > +#include <xen/lib.h> > +#include <xen/list.h> > +#include <xen/mm.h> > +#include <xen/string.h> > +#include <xen/time.h> > +#include <xen/vmap.h> > + > +#define SCMI_BASE_PROTOCOL 0x10 > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > + > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > +#define SCMI_SUCCESS 0 > +#define SCMI_NOT_SUPPORTED (-1) > +#define SCMI_INVALID_PARAMETERS (-2) > +#define SCMI_DENIED (-3) > +#define SCMI_NOT_FOUND (-4) > +#define SCMI_OUT_OF_RANGE (-5) > +#define SCMI_BUSY (-6) > +#define SCMI_COMMS_ERROR (-7) > +#define SCMI_GENERIC_ERROR (-8) > +#define SCMI_HARDWARE_ERROR (-9) > +#define SCMI_PROTOCOL_ERROR (-10) > + > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > + > +#define SCMI_SMC_ID "arm,smc-id" > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" I could find the following SCMI binding in Linux, which describes the arm,scmi-smc compatible and the arm,smc-id property: Documentation/devicetree/bindings/firmware/arm,scmi.yaml However, linux,scmi_mem is not described. Aren't you supposed to read the "shmem" property instead? And the compatible string used for this seems to be "arm,scmi-shmem". > +#define SCMI_SHMEM "shmem" > + > +#define HYP_CHANNEL 0x0 > + > +#define HDR_ID GENMASK(7,0) > +#define HDR_TYPE GENMASK(9, 8) > +#define HDR_PROTO GENMASK(17, 10) > + > +/* SCMI protocol, refer to section 4.2.2.2 (DEN0056C) */ > +#define MSG_N_AGENTS_MASK GENMASK(15, 8) > + > +#define FIELD_GET(_mask, _reg)\ > + ((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1))) > +#define FIELD_PREP(_mask, _val)\ > + (((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask)) > + > +typedef struct scmi_msg_header { > + uint8_t id; > + uint8_t type; > + uint8_t protocol; > +} scmi_msg_header_t; > + > +typedef struct scmi_perms_tx { > + uint32_t agent_id; > + uint32_t device_id; > + uint32_t flags; > +} scmi_perms_tx_t; > + > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE BIT(0, UL) > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_ERROR BIT(1, UL) > + > +#define SCMI_ALLOW_ACCESS BIT(0, UL) > + > +struct scmi_shared_mem { > + uint32_t reserved; > + uint32_t channel_status; > + uint32_t reserved1[2]; > + uint32_t flags; > + uint32_t length; > + uint32_t msg_header; > + uint8_t msg_payload[]; > +}; > + > +struct scmi_channel { > + int chan_id; > + int agent_id; > + uint32_t func_id; > + int domain_id; > + uint64_t paddr; > + struct scmi_shared_mem *shmem; > + spinlock_t lock; > + struct list_head list; > +}; > + > +struct scmi_data { > + struct list_head channel_list; > + spinlock_t channel_list_lock; > + bool initialized; > + u64 shmem_addr, shmem_size; > +}; > + > +static struct scmi_data scmi_data; > + > +/* > + * pack_scmi_header() - packs and returns 32-bit header > + * > + * @hdr: pointer to header containing all the information on message id, > + * protocol id and type id. > + * > + * Return: 32-bit packed message header to be sent to the platform. > + */ > +static inline uint32_t pack_scmi_header(scmi_msg_header_t *hdr) > +{ > + return FIELD_PREP(HDR_ID, hdr->id) | > + FIELD_PREP(HDR_TYPE, hdr->type) | > + FIELD_PREP(HDR_PROTO, hdr->protocol); > +} > + > +/* > + * unpack_scmi_header() - unpacks and records message and protocol id > + * > + * @msg_hdr: 32-bit packed message header sent from the platform > + * @hdr: pointer to header to fetch message and protocol id. > + */ > +static inline void unpack_scmi_header(uint32_t msg_hdr, scmi_msg_header_t *hdr) > +{ > + hdr->id = FIELD_GET(HDR_ID, msg_hdr); > + hdr->type = FIELD_GET(HDR_TYPE, msg_hdr); > + hdr->protocol = FIELD_GET(HDR_PROTO, msg_hdr); > +} > + > +static inline int channel_is_free(struct scmi_channel *chan_info) > +{ > + return ( chan_info->shmem->channel_status > + & SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE ) ? 0 : -EBUSY; Does this need a memory barrier? Or not, because the other end always runs on the same CPU at a different execution level so the channel_status would be always guaranteed to be read as updated? > +} > + > +static int send_smc_message(struct scmi_channel *chan_info, > + scmi_msg_header_t *hdr, void *data, int len) > +{ > + struct arm_smccc_res resp; > + int ret; > + > + printk(XENLOG_DEBUG "scmi: status =%d len=%d\n", > + chan_info->shmem->channel_status, len); > + printk(XENLOG_DEBUG "scmi: header id = %d type = %d, proto = %d\n", > + hdr->id, hdr->type, hdr->protocol); > + > + ret = channel_is_free(chan_info); > + if ( IS_ERR_VALUE(ret) ) > + return ret; > + > + chan_info->shmem->channel_status = 0x0; > + /* Writing 0x0 right now, but SCMI_SHMEM_FLAG_INTR_ENABLED can be set */ > + chan_info->shmem->flags = 0x0; > + chan_info->shmem->length = sizeof(chan_info->shmem->msg_header) + len; > + chan_info->shmem->msg_header = pack_scmi_header(hdr); > + > + printk(XENLOG_DEBUG "scmi: Writing to shmem address %p\n", > + chan_info->shmem); > + if ( len > 0 && data ) > + memcpy((void *)(chan_info->shmem->msg_payload), data, len); Again, here we don't need a barrier because it is implicit in the SMC? Don't we need to check that "len" fits in the shared memory? > + arm_smccc_smc(chan_info->func_id, 0, 0, 0, 0, 0, 0, chan_info->chan_id, > + &resp); > + > + printk(XENLOG_DEBUG "scmi: scmccc_smc response %d\n", (int)(resp.a0)); > + > + if ( resp.a0 ) > + return -EOPNOTSUPP; Why is that? > + return 0; > +} > + > +static int check_scmi_status(int scmi_status) > +{ > + if ( scmi_status == SCMI_SUCCESS ) > + return 0; > + > + printk(XENLOG_DEBUG "scmi: Error received: %d\n", scmi_status); > + > + switch ( scmi_status ) > + { > + case SCMI_NOT_SUPPORTED: > + return -EOPNOTSUPP; > + case SCMI_INVALID_PARAMETERS: > + return -EINVAL; > + case SCMI_DENIED: > + return -EACCES; > + case SCMI_NOT_FOUND: > + return -ENOENT; > + case SCMI_OUT_OF_RANGE: > + return -ERANGE; > + case SCMI_BUSY: > + return -EBUSY; > + case SCMI_COMMS_ERROR: > + return -ENOTCONN; > + case SCMI_GENERIC_ERROR: > + return -EIO; > + case SCMI_HARDWARE_ERROR: > + return -ENXIO; > + case SCMI_PROTOCOL_ERROR: > + return -EBADMSG; > + } > + > + return -EINVAL; > +} > + > +static int get_smc_response(struct scmi_channel *chan_info, > + scmi_msg_header_t *hdr, void *data, int len) > +{ > + int recv_len; > + int ret; > + > + printk(XENLOG_DEBUG "scmi: get smc responce msgid %d\n", hdr->id); > + > + ret = channel_is_free(chan_info); > + if ( IS_ERR_VALUE(ret) ) > + return ret; I am not familiar with the spec (do you have a link?) but is it expected that the channel is "free" when actually we want to read a message on the channel? > + recv_len = chan_info->shmem->length - sizeof(chan_info->shmem->msg_header); > + > + if ( recv_len < 0 ) > + { > + printk(XENLOG_ERR > + "scmi: Wrong size of smc message. Data may be invalid\n"); > + return -EINVAL; > + } > + > + if ( recv_len > len ) > + { > + printk(XENLOG_ERR > + "scmi: Not enough buffer for message %d, expecting %d\n", > + recv_len, len); > + return -EINVAL; > + } > + > + unpack_scmi_header(chan_info->shmem->msg_header, hdr); > + > + if ( recv_len > 0 ) > + { > + memcpy(data, chan_info->shmem->msg_payload, recv_len); > + } > + > + return 0; > +} > + > +static int do_smc_xfer(struct scmi_channel *channel, scmi_msg_header_t *hdr, void *tx_data, int tx_size, > + void *rx_data, int rx_size) > +{ > + int ret = 0; > + > + if ( !hdr ) > + return -EINVAL; > + > + spin_lock(&channel->lock); > + > + ret = send_smc_message(channel, hdr, tx_data, tx_size); > + if ( ret ) > + goto clean; > + > + ret = get_smc_response(channel, hdr, rx_data, rx_size); > +clean: > + spin_unlock(&channel->lock); > + > + return ret; > +} > + > +static struct scmi_channel *get_channel_by_id(uint8_t chan_id) > +{ > + struct scmi_channel *curr; > + bool found = false; > + > + spin_lock(&scmi_data.channel_list_lock); > + list_for_each_entry(curr, &scmi_data.channel_list, list) please use parenthesis around the inner if (also in other places) > + if ( curr->chan_id == chan_id ) > + { > + found = true; > + break; > + } > + > + spin_unlock(&scmi_data.channel_list_lock); > + if ( found ) > + return curr; > + > + return NULL; > +} > + > +static struct scmi_channel *get_channel_by_domain(uint8_t domain_id) Use domid_t for domain ids. Also, wouldn't it be better to implement it as: static inline struct scmi_channel *get_channel_by_domain(struct domain *d) { return d->arch.sci } > +{ > + struct scmi_channel *curr; > + bool found = false; > + > + spin_lock(&scmi_data.channel_list_lock); > + list_for_each_entry(curr, &scmi_data.channel_list, list) > + if ( curr->domain_id == domain_id ) > + { > + found = true; > + break; > + } > + > + spin_unlock(&scmi_data.channel_list_lock); > + if ( found ) > + return curr; > + > + return NULL; > +} > + > +static struct scmi_channel *aquire_scmi_channel(int domain_id) > +{ > + struct scmi_channel *curr; > + bool found = false; > + > + ASSERT(domain_id != DOMID_INVALID && domain_id >= 0); > + > + spin_lock(&scmi_data.channel_list_lock); > + list_for_each_entry(curr, &scmi_data.channel_list, list) > + if ( (curr->domain_id == DOMID_INVALID) > + && (curr->chan_id != HYP_CHANNEL) ) If you use DOMID_XEN for HYP_CHANNEL, then this check becomes more intuitive > + { > + curr->domain_id = domain_id; > + found = true; > + break; > + } > + > + spin_unlock(&scmi_data.channel_list_lock); > + if ( found ) > + return curr; > + > + return NULL; > +} > + > +static void relinquish_scmi_channel(struct scmi_channel *channel) > +{ > + spin_lock(&scmi_data.channel_list_lock); > + ASSERT(channel != NULL); the ASSERT could be before the spin_lock > + channel->domain_id = DOMID_INVALID; > + spin_unlock(&scmi_data.channel_list_lock); > +} > + > +static struct scmi_channel *smc_create_channel(uint8_t chan_id, > + uint32_t func_id, uint64_t addr) > +{ > + struct scmi_channel *channel; > + mfn_t mfn; > + > + channel = get_channel_by_id(chan_id); > + if ( channel ) > + return ERR_PTR(EEXIST); > + > + channel = xmalloc(struct scmi_channel); > + if ( !channel ) > + return ERR_PTR(ENOMEM); > + > + channel->chan_id = chan_id; > + channel->func_id = func_id; > + channel->domain_id = DOMID_INVALID; I take you are using DOMID_INVALID to mark a channel used by Xen itself? If so, then DOMID_XEN would be more appropriate. > + mfn = maddr_to_mfn(addr); > + channel->shmem = vmap(&mfn, 1); One thing to be careful is the mapping attributes, for a couple of reasons. As you might be aware, the ARM architecture forbids mismatching attributes for mapping memory in different places in the system. So the attributes that we use here must be the same used by the firmware (and/or the guest.) The second reason to be careful is that in the bindings example Documentation/devicetree/bindings/firmware/arm,scmi.yaml the shared memory is "mmio-sram", which is special. It is not supposed to be normal memory, but it is OK to map it cacheable. Still, it might be more appropriate to use ioremap_cache. > + if ( !channel->shmem ) > + { > + xfree(channel); > + return ERR_PTR(ENOMEM); > + } > + > + printk(XENLOG_DEBUG "scmi: Got shmem after vmap %p\n", channel->shmem); > + channel->paddr = addr; > + channel->shmem->channel_status = SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE; > + spin_lock_init(&channel->lock); > + spin_lock(&scmi_data.channel_list_lock); > + list_add(&channel->list, &scmi_data.channel_list); > + spin_unlock(&scmi_data.channel_list_lock); > + return channel; > +} > + > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > +{ > + return iomem_permit_access(d, paddr_to_pfn(addr), > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > +} > + > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > + uint64_t len) > +{ > + return iomem_deny_access(d, paddr_to_pfn(addr), > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > +} > + > +static int dt_update_domain_range(struct domain *d, uint64_t addr, > + uint64_t size) > +{ > + struct dt_device_node *shmem_node; > + __be32 *hw_reg; > + const struct dt_property *pp; > + uint32_t len; > + > + shmem_node = dt_find_compatible_node(NULL, NULL, "arm,scmi-shmem"); Here we are using "arm,scmi-shmem" while below we are checking for "linux,scmi_mem". What's the difference? Also, this function is looking for "arm,scmi-shmem" in dt_host and replaces its value. For dom0less domUs we'll probably need a make_scmi_node function to create the node from scratch like for instance xen/arch/arm/domain_build.c:make_gic_domU_node. I wonder if we had such a function whether it wouldn't be better to also use it for dom0 (and blacklist the physical "arm,scmi-shmem" in handle_node so that dom0 doesn't get the real shared memory information by accident). > + > + if ( !shmem_node ) > + { > + printk(XENLOG_ERR "scmi: Unable to find %s node in DT\n", SCMI_SHMEM); > + return -EINVAL; > + } > + > + pp = dt_find_property(shmem_node, "reg", &len); > + if ( !pp ) > + { > + printk(XENLOG_ERR "scmi: Unable to find regs entry in shmem node\n"); > + return -ENOENT; > + } > + > + hw_reg = pp->value; > + dt_set_range(&hw_reg, shmem_node, addr, size); > + > + return 0; > +} > + > +static void free_channel_list(void) > +{ > + struct scmi_channel *curr, *_curr; > + > + spin_lock(&scmi_data.channel_list_lock); > + list_for_each_entry_safe (curr, _curr, &scmi_data.channel_list, list) > + { > + vunmap(curr->shmem); > + list_del(&curr->list); > + xfree(curr); > + } > + > + spin_unlock(&scmi_data.channel_list_lock); > +} > + > +static __init bool scmi_probe(struct dt_device_node *scmi_node) > +{ > + struct dt_device_node *shmem_node; > + int ret, i; > + struct scmi_channel *channel, *agent_channel; > + int n_agents; > + scmi_msg_header_t hdr; > + struct rx_t { > + int32_t status; > + uint32_t attributes; > + } rx; Should rx be defined at the top together with scmi_perms_tx_t and others? > + uint32_t func_id; > + > + ASSERT(scmi_node != NULL); > + > + INIT_LIST_HEAD(&scmi_data.channel_list); > + spin_lock_init(&scmi_data.channel_list_lock); > + > + if ( !dt_property_read_u32(scmi_node, SCMI_SMC_ID, &func_id) ) > + { > + printk(XENLOG_ERR "scmi: Unable to read smc-id from DT\n"); > + return false; > + } > + > + shmem_node = dt_find_node_by_name(NULL, SCMI_SHARED_MEMORY); From the spec, it looks like you should be getting the shared memory area from the phandle list "shmem". > + if ( IS_ERR_OR_NULL(shmem_node) ) > + { > + printk(XENLOG_ERR > + "scmi: Device tree error, can't parse shmem phandle %ld\n", > + PTR_ERR(shmem_node)); > + return false; > + } > + > + ret = dt_device_get_address(shmem_node, 0, &scmi_data.shmem_addr, > + &scmi_data.shmem_size); > + if ( IS_ERR_VALUE(ret) ) > + return false; > + > + channel = smc_create_channel(HYP_CHANNEL, func_id, scmi_data.shmem_addr); > + if ( IS_ERR(channel) ) > + return false; > + > + hdr.id = SCMI_BASE_PROTOCOL_ATTIBUTES; > + hdr.type = 0; > + hdr.protocol = SCMI_BASE_PROTOCOL; > + > + ret = do_smc_xfer(channel, &hdr, NULL, 0, &rx, sizeof(rx)); > + if ( ret ) > + goto clean; > + > + ret = check_scmi_status(rx.status); > + if ( ret ) > + goto clean; > + > + n_agents = FIELD_GET(MSG_N_AGENTS_MASK, rx.attributes); > + printk(XENLOG_DEBUG "scmi: Got agent count %d\n", n_agents); > + > + n_agents = (n_agents > scmi_data.shmem_size / PAGE_SIZE) ? > + scmi_data.shmem_size / PAGE_SIZE : n_agents; > + > + for ( i = 1; i < n_agents; i++ ) > + { Given that HYP_CHANNEL is actually zero, it looks like we could do everything here in this loop but starting from i=0? > + uint32_t tx_agent_id = 0xFFFFFFFF; > + struct { > + int32_t status; > + uint32_t agent_id; > + char name[16]; > + } da_rx; > + > + agent_channel = smc_create_channel(i, func_id, scmi_data.shmem_addr + > + i * PAGE_SIZE); > + if ( IS_ERR(agent_channel) ) > + { > + ret = PTR_ERR(agent_channel); > + goto clean; > + } > + > + hdr.id = SCMI_BASE_DISCOVER_AGENT; > + hdr.type = 0; > + hdr.protocol = SCMI_BASE_PROTOCOL; > + > + ret = do_smc_xfer(agent_channel, &hdr, &tx_agent_id, > + sizeof(tx_agent_id), &da_rx, sizeof(da_rx)); > + if ( ret ) > + goto clean; > + > + ret = check_scmi_status(da_rx.status); > + if ( ret ) > + goto clean; > + > + printk(XENLOG_DEBUG "scmi: status=0x%x id=0x%x name=%s\n", > + da_rx.status, da_rx.agent_id, da_rx.name); > + > + agent_channel->agent_id = da_rx.agent_id; > + } > + > + scmi_data.initialized = true; > + return true; > + > +clean: > + free_channel_list(); > + return ret == 0; > +} > + > +static int scmi_domain_init(struct domain *d) > +{ > + struct scmi_channel *channel; > + int ret; > + > + if ( !scmi_data.initialized ) > + return 0; > + > + channel = aquire_scmi_channel(d->domain_id); > + if ( IS_ERR_OR_NULL(channel) ) > + return -ENOENT; > + > + printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = %d" > + "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, > + channel->paddr); > + > + if ( is_hardware_domain(d) ) > + { > + ret = map_memory_to_domain(d, scmi_data.shmem_addr, > + scmi_data.shmem_size); > + if ( IS_ERR_VALUE(ret) ) > + goto error; > + > + ret = dt_update_domain_range(d, channel->paddr, PAGE_SIZE); > + if ( IS_ERR_VALUE(ret) ) > + { > + int rc = unmap_memory_from_domain(d, scmi_data.shmem_addr, > + scmi_data.shmem_size); > + if ( rc ) > + printk(XENLOG_ERR "Unable to unmap_memory_from_domain\n"); > + > + goto error; > + } > + } Is dom0 the only domain to get direct access to the shared memory region? If so, I don't think it is a good idea to make Dom0 "special" in this case. Let me make an example: if we assign a device to a domU since boot, and dom0 wants to change the frequency of a clock that affects the assigned device (likely because it doesn't know it is assigned), then dom0 shouldn't be able to. We might have to perform checks in Xen to make sure dom0 cannot stop the clock for the assigned device. So I think it would be better if all domains are treated the same way in the mediator unless really necessary. On the other hand, if all domains get access to the shared memory region, then I don't think this is likely the right place to create the dom0 mapping. We probably want to do it in domain_build.c in a way that can be reused for dom0less domUs. In regards to shared memory: it looks like the only two functions to access the real shared memory are send_smc_message and get_smc_response. If that is the case, then we actually don't need to expose the real shared memory to any of the domains. We could simply: - expose a regular normal memory region as dom0/domU channel memory - on SMC trap, read from the "fake" shared memory and set the corresponding real shared memory on the appropriate channel - issue the SMC call - on return from SMC, copy over data from the real shared memory to the "fake" channel reagion This is useful if we need to "filter" any of the SCMI commands and options from the domains to the firmware, and also it is useful if the channel memory is not page aligned. But if the permissions are fine-grained enough and also the channel memory is page aligned (and multiple of 4K in size) then we could map the memory. > + > + d->arch.sci = channel; > + > + return 0; > +error: > + relinquish_scmi_channel(channel); > + > + return ret; > +} > + > +static int scmi_add_device_by_devid(struct domain *d, uint32_t scmi_devid) > +{ > + struct scmi_channel *channel, *agent_channel; > + scmi_msg_header_t hdr; > + scmi_perms_tx_t tx; > + struct rx_t { > + int32_t status; > + uint32_t attributes; > + } rx; > + int ret; > + > + if ( !scmi_data.initialized ) > + return 0; > + > + printk(XENLOG_DEBUG "scmi: scmi_devid = %d\n", scmi_devid); > + > + agent_channel = get_channel_by_domain(d->domain_id); > + if ( IS_ERR_OR_NULL(agent_channel) ) > + return PTR_ERR(agent_channel); > + > + channel = get_channel_by_id(HYP_CHANNEL); > + if ( IS_ERR_OR_NULL(channel) ) > + return PTR_ERR(channel); > + > + hdr.id = SCMI_BASE_SET_DEVICE_PERMISSIONS; > + hdr.type = 0; > + hdr.protocol = SCMI_BASE_PROTOCOL; > + > + tx.agent_id = agent_channel->agent_id; > + tx.device_id = scmi_devid; > + tx.flags = SCMI_ALLOW_ACCESS; > + > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(&rx)); > + if ( IS_ERR_VALUE(ret) ) > + return ret; > + > + ret = check_scmi_status(rx.status); > + if ( IS_ERR_VALUE(ret) ) > + return ret; > + > + return 0; > +} > + > +static int scmi_add_dt_device(struct domain *d, struct dt_device_node *dev) > +{ > + uint32_t scmi_devid; > + > + if ( (!scmi_data.initialized) || (!d->arch.sci) ) > + return 0; > + > + if ( !dt_property_read_u32(dev, "scmi_devid", &scmi_devid) ) > + return 0; scmi_devid is another property that is not documented in the binding. > + printk(XENLOG_INFO "scmi: dt_node = %s\n", dt_node_full_name(dev)); > + > + return scmi_add_device_by_devid(d, scmi_devid); > +} > + > +static int scmi_relinquish_resources(struct domain *d) > +{ > + int ret; > + struct scmi_channel *channel, *agent_channel; > + scmi_msg_header_t hdr; > + struct reset_agent_tx { > + uint32_t agent_id; > + uint32_t flags; > + } tx; > + uint32_t rx; > + > + if ( !d->arch.sci ) > + return 0; > + > + agent_channel = d->arch.sci; > + > + spin_lock(&agent_channel->lock); > + tx.agent_id = agent_channel->agent_id; > + spin_unlock(&agent_channel->lock); > + > + channel = get_channel_by_id(HYP_CHANNEL); > + if ( !channel ) > + { > + printk(XENLOG_ERR > + "scmi: Unable to get Hypervisor scmi channel for domain %d\n", > + d->domain_id); > + return -EINVAL; > + } > + > + hdr.id = SCMI_BASE_RESET_AGENT_CONFIGURATION; > + hdr.type = 0; > + hdr.protocol = SCMI_BASE_PROTOCOL; > + > + tx.flags = 0; > + > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(rx)); > + if ( ret ) > + return ret; > + > + ret = check_scmi_status(rx); > + > + return ret; > +} > + > +static void scmi_domain_destroy(struct domain *d) > +{ > + struct scmi_channel *channel; > + > + if ( !d->arch.sci ) > + return; > + > + channel = d->arch.sci; > + spin_lock(&channel->lock); > + > + relinquish_scmi_channel(channel); > + printk(XENLOG_DEBUG "scmi: Free domain %d\n", d->domain_id); > + > + d->arch.sci = NULL; > + > + unmap_memory_from_domain(d, channel->paddr, PAGE_SIZE); > + spin_unlock(&channel->lock); > + return; > +} > + > +static bool scmi_handle_call(struct domain *d, void *args) > +{ > + bool res = false; > + struct scmi_channel *agent_channel; > + struct arm_smccc_res resp; > + struct cpu_user_regs *regs = args; > + > + if ( !d->arch.sci ) > + return false; > + > + agent_channel = d->arch.sci; > + spin_lock(&agent_channel->lock); > + > + if ( agent_channel->func_id != regs->x0 ) > + { > + printk(XENLOG_ERR "scmi: func_id mismatch, exiting\n"); > + goto unlock; > + } > + > + arm_smccc_smc(agent_channel->func_id, 0, 0, 0, 0, 0, 0, > + agent_channel->chan_id, &resp); > + > + set_user_reg(regs, 0, resp.a0); > + set_user_reg(regs, 1, resp.a1); > + set_user_reg(regs, 2, resp.a2); > + set_user_reg(regs, 3, resp.a3); > + res = true; > +unlock: > + spin_unlock(&agent_channel->lock); > + > + return res; > +} > + > +static int scmi_get_channel_paddr(void *scmi_ops, > + struct xen_arch_domainconfig *config) > +{ > + struct scmi_channel *agent_channel = scmi_ops; > + > + if ( !agent_channel ) > + return -EINVAL; > + > + config->sci_agent_paddr = agent_channel->paddr; > + return 0; > +} I am still not sure why it couldn't be done by scmi_domain_init. > +static const struct dt_device_match scmi_smc_match[] __initconst = > +{ > + DT_MATCH_SCMI_SMC, > + { /* sentinel */ }, > +}; > + > +static const struct sci_mediator_ops scmi_ops = > +{ > + .probe = scmi_probe, > + .domain_init = scmi_domain_init, > + .domain_destroy = scmi_domain_destroy, > + .add_dt_device = scmi_add_dt_device, > + .relinquish_resources = scmi_relinquish_resources, > + .handle_call = scmi_handle_call, > + .get_channel_info = scmi_get_channel_paddr > +}; > + > +REGISTER_SCI_MEDIATOR(scmi_smc, "SCMI-SMC", XEN_DOMCTL_CONFIG_SCI_SCMI_SMC, > + scmi_smc_match, &scmi_ops); > + > +/* > + * Local variables: > + * mode: C > + * c-file-style: "BSD" > + * c-basic-offset: 4 > + * indent-tabs-mode: nil > + * End: > + */ > diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h > index 9180be5e86..a67237942d 100644 > --- a/xen/include/public/arch-arm.h > +++ b/xen/include/public/arch-arm.h > @@ -315,6 +315,7 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t); > #define XEN_DOMCTL_CONFIG_TEE_OPTEE 1 > > #define XEN_DOMCTL_CONFIG_SCI_NONE 0 > +#define XEN_DOMCTL_CONFIG_SCI_SCMI_SMC 1 > > struct xen_arch_domainconfig { > /* IN/OUT */ > -- > 2.27.0 >
Hi Julien, On Fri, Dec 17, 2021 at 04:38:31PM +0000, Julien Grall wrote: > > > On 17/12/2021 13:58, Oleksii Moisieiev wrote: > > Hi Julien, > > Hi, > > > On Fri, Dec 17, 2021 at 01:37:35PM +0000, Julien Grall wrote: > > > Hi, > > > > > > On 17/12/2021 13:23, Oleksii Moisieiev wrote: > > > > > > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > > > > > > +{ > > > > > > + return iomem_permit_access(d, paddr_to_pfn(addr), > > > > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > > > > +} > > > > > > + > > > > > > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > > > > > > + uint64_t len) > > > > > > +{ > > > > > > + return iomem_deny_access(d, paddr_to_pfn(addr), > > > > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > > > > +} > > > > > > > > > > I wonder, why we need an extra level of indirection here. And if this is > > > > > really needed, I wonder why map(unmap)_memory* name was chosen, as there is > > > > > no memory mapping/unmapping really happens here. > > > > > > > > > > > > > I've added extra indirection to hide math like > > > > paddr_to_pfn(PAGE_ALIGN(addr + len -1) > > > > so you don't have to math in each call. unmap_memory_from_domain called > > > > from 2 places, so I moved both calls to separate function. > > > > Although, I agree that map/unmap is not perfect name. I consider > > > > renaming it to mem_permit_acces and mam_deny_access. > > > > > > I haven't looked at the rest of the series. But this discussion caught my > > > eye. This code implies that the address is page-aligned but the length not. > > > Is that intended? > > > > > > That said, if you give permission to the domain on a full page then it means > > > it may be able to access address it should not. Can you explain why this is > > > fine? > > > > > > > The idea was that xen receives some memory from the dt_node linux,scmi_mem, > > then we split memory between the agents, so each agent get 1 page (we > > allocate 0x10 pages right now). > > Thanks for the clarification. Does this imply the guest will be able to > write message directly to the firmware? We used DEN0056C Specification as base. Available on: https://developer.arm.com/documentation/den0056/latest. SCMI transport is described in Section 5.1. We implemented Shared Memory transport. Firmware has N pages of the shared memory, used to communicate with Agents. It allocates N agents and assign a page for each agent, such as: ------------------------------------- | Agent H | Agent 1 | Agent 2 | ... | ------------------------------------- Agent H is the privilleged Hypervisor agent, which is used to do the base commands, such as getting Agent list, set\unset permissions etc. Hypervisor assign agent to the guest and maps page, related to the agent to the Guest. So the Guest, which is Agent 1 will get an access to Agent 1 page. Guest places SCMI message to Agent 1 memory, then sends SMC message. Hypervisor process SMC request, add agent id to the message parameters and redirects it to the Firmware. Based on the agent_id Firmware knows which page it should read. Then after permission check ( if the resetId/clockID/powerID etc from message is assigned to agent_id ) it does changes to the HW and places response to Agent shared memory and marks channel as FREE ( by setting free bit in shared memory ). Once channel is marked as free - Guest read response from the shared memory. Non-virtualized systems will work as well. OS should send SMC directly to the Firmware. > > If so, this brings a few more questions: > 1) What will the guest write in it? Can it contains addresses? Guest can write scmi request to the shared memory, which include the following data: 1) protocol_id - which protocol is requested, such as clock, power, reset etc 2) message_id - action that should be done to HW, such as do_reset or get_clock 3) message data - which includes reset_id/clock_id/power_id etc. that should be changed. Reset IDs and Clock IDs are assigned in Firmware. Guest receives ID, corresponding to the device from the device-tree. dt_node as an example: &avb { scmi_devid = <0>; clocks = <&scmi_clock 0>; power-domains = <&scmi_power 0>; resets = <&scmi_reset 0>; }; > 2) What are the expected memory attribute for the regions? xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. > 3) What's the threat model for the firmware? Will it verify every request? Firmware reads data from agent page, then makes check if clockid/resetid/powerid etc is assigned to agent. During building guest, Xen sends permission request to the firmware for each device, which is passed-through to the guest. So for avb from previous example the device_id 0 permission request will be sent. Based on the device_id firmware will set permission for clockid 0, resetid 0 and powerid 0. Best regards, Oleksii.
Hi Stefano, On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > This is the implementation of SCI interface, called SCMI-SMC driver, > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > This allows devices from the Domains to work with clocks, resets and > > power-domains without access to CPG. > > > > The following features are implemented: > > - request SCMI channels from ATF and pass channels to Domains; > > - set device permissions for Domains based on the Domain partial > > device-tree. Devices with permissions are able to work with clocks, > > resets and power-domains via SCMI; > > - redirect scmi messages from Domains to ATF. > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > --- > > xen/arch/arm/Kconfig | 2 + > > xen/arch/arm/sci/Kconfig | 10 + > > xen/arch/arm/sci/Makefile | 1 + > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > xen/include/public/arch-arm.h | 1 + > > 5 files changed, 809 insertions(+) > > create mode 100644 xen/arch/arm/sci/Kconfig > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > index 186e1db389..02d96c6cfc 100644 > > --- a/xen/arch/arm/Kconfig > > +++ b/xen/arch/arm/Kconfig > > @@ -114,6 +114,8 @@ config SCI > > support. It allows guests to control system resourcess via one of > > SCI mediators implemented in XEN. > > > > +source "arch/arm/sci/Kconfig" > > + > > endmenu > > > > menu "ARM errata workaround via the alternative framework" > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > new file mode 100644 > > index 0000000000..9563067ddc > > --- /dev/null > > +++ b/xen/arch/arm/sci/Kconfig > > @@ -0,0 +1,10 @@ > > +config SCMI_SMC > > + bool "Enable SCMI-SMC mediator driver" > > + default n > > + depends on SCI > > + ---help--- > > + > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > + This feature allows drivers from Domains to work with System > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > + for communication. > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > index 837dc7492b..67f2611872 100644 > > --- a/xen/arch/arm/sci/Makefile > > +++ b/xen/arch/arm/sci/Makefile > > @@ -1 +1,2 @@ > > obj-y += sci.o > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > new file mode 100644 > > index 0000000000..2eb01ea82d > > --- /dev/null > > +++ b/xen/arch/arm/sci/scmi_smc.c > > @@ -0,0 +1,795 @@ > > +/* > > + * xen/arch/arm/sci/scmi_smc.c > > + * > > + * SCMI mediator driver, using SCP as transport. > > + * > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > + * Copyright (C) 2021, EPAM Systems. > > + * > > + * This program is free software; you can redistribute it and/or modify > > + * it under the terms of the GNU General Public License as published by > > + * the Free Software Foundation; either version 2 of the License, or > > + * (at your option) any later version. > > + * > > + * This program is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > + * GNU General Public License for more details. > > + */ > > + > > +#include <asm/sci/sci.h> > > +#include <asm/smccc.h> > > +#include <asm/io.h> > > +#include <xen/bitops.h> > > +#include <xen/config.h> > > +#include <xen/sched.h> > > +#include <xen/device_tree.h> > > +#include <xen/iocap.h> > > +#include <xen/init.h> > > +#include <xen/err.h> > > +#include <xen/lib.h> > > +#include <xen/list.h> > > +#include <xen/mm.h> > > +#include <xen/string.h> > > +#include <xen/time.h> > > +#include <xen/vmap.h> > > + > > +#define SCMI_BASE_PROTOCOL 0x10 > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > + > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > +#define SCMI_SUCCESS 0 > > +#define SCMI_NOT_SUPPORTED (-1) > > +#define SCMI_INVALID_PARAMETERS (-2) > > +#define SCMI_DENIED (-3) > > +#define SCMI_NOT_FOUND (-4) > > +#define SCMI_OUT_OF_RANGE (-5) > > +#define SCMI_BUSY (-6) > > +#define SCMI_COMMS_ERROR (-7) > > +#define SCMI_GENERIC_ERROR (-8) > > +#define SCMI_HARDWARE_ERROR (-9) > > +#define SCMI_PROTOCOL_ERROR (-10) > > + > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > + > > +#define SCMI_SMC_ID "arm,smc-id" > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > I could find the following SCMI binding in Linux, which describes > the arm,scmi-smc compatible and the arm,smc-id property: > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > However, linux,scmi_mem is not described. Aren't you supposed to read > the "shmem" property instead? And the compatible string used for this > seems to be "arm,scmi-shmem". > We use linux,scmi_mem node to reserve memory, needed for all channels: reserved-memory { /* reserved region for scmi channels*/ scmi_memory: linux,scmi_mem@53FF0000 { no-map; reg = <0x0 0x53FF0000 0x0 0x10000>; }; }; arm,scmi-shmem node used in shmem property defines only 1 page needed to the current scmi channel: cpu_scp_shm: scp-shmem@0x53FF0000 { compatible = "arm,scmi-shmem"; reg = <0x0 0x53FF0000 0x0 0x1000>; }; For each Domain reg points to unigue page from linux,scmi_mem region, assigned to this agent. > > > +#define SCMI_SHMEM "shmem" > > + > > +#define HYP_CHANNEL 0x0 > > + > > +#define HDR_ID GENMASK(7,0) > > +#define HDR_TYPE GENMASK(9, 8) > > +#define HDR_PROTO GENMASK(17, 10) > > + > > +/* SCMI protocol, refer to section 4.2.2.2 (DEN0056C) */ > > +#define MSG_N_AGENTS_MASK GENMASK(15, 8) > > + > > +#define FIELD_GET(_mask, _reg)\ > > + ((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1))) > > +#define FIELD_PREP(_mask, _val)\ > > + (((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask)) > > + > > +typedef struct scmi_msg_header { > > + uint8_t id; > > + uint8_t type; > > + uint8_t protocol; > > +} scmi_msg_header_t; > > + > > +typedef struct scmi_perms_tx { > > + uint32_t agent_id; > > + uint32_t device_id; > > + uint32_t flags; > > +} scmi_perms_tx_t; > > + > > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE BIT(0, UL) > > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_ERROR BIT(1, UL) > > + > > +#define SCMI_ALLOW_ACCESS BIT(0, UL) > > + > > +struct scmi_shared_mem { > > + uint32_t reserved; > > + uint32_t channel_status; > > + uint32_t reserved1[2]; > > + uint32_t flags; > > + uint32_t length; > > + uint32_t msg_header; > > + uint8_t msg_payload[]; > > +}; > > + > > +struct scmi_channel { > > + int chan_id; > > + int agent_id; > > + uint32_t func_id; > > + int domain_id; > > + uint64_t paddr; > > + struct scmi_shared_mem *shmem; > > + spinlock_t lock; > > + struct list_head list; > > +}; > > + > > +struct scmi_data { > > + struct list_head channel_list; > > + spinlock_t channel_list_lock; > > + bool initialized; > > + u64 shmem_addr, shmem_size; > > +}; > > + > > +static struct scmi_data scmi_data; > > + > > +/* > > + * pack_scmi_header() - packs and returns 32-bit header > > + * > > + * @hdr: pointer to header containing all the information on message id, > > + * protocol id and type id. > > + * > > + * Return: 32-bit packed message header to be sent to the platform. > > + */ > > +static inline uint32_t pack_scmi_header(scmi_msg_header_t *hdr) > > +{ > > + return FIELD_PREP(HDR_ID, hdr->id) | > > + FIELD_PREP(HDR_TYPE, hdr->type) | > > + FIELD_PREP(HDR_PROTO, hdr->protocol); > > +} > > + > > +/* > > + * unpack_scmi_header() - unpacks and records message and protocol id > > + * > > + * @msg_hdr: 32-bit packed message header sent from the platform > > + * @hdr: pointer to header to fetch message and protocol id. > > + */ > > +static inline void unpack_scmi_header(uint32_t msg_hdr, scmi_msg_header_t *hdr) > > +{ > > + hdr->id = FIELD_GET(HDR_ID, msg_hdr); > > + hdr->type = FIELD_GET(HDR_TYPE, msg_hdr); > > + hdr->protocol = FIELD_GET(HDR_PROTO, msg_hdr); > > +} > > + > > +static inline int channel_is_free(struct scmi_channel *chan_info) > > +{ > > + return ( chan_info->shmem->channel_status > > + & SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE ) ? 0 : -EBUSY; > > Does this need a memory barrier? Or not, because the other end always > runs on the same CPU at a different execution level so the > channel_status would be always guaranteed to be read as updated? > It don't because the other end runs on the same CPU. Other mediator implemetaions, which uses different areas may need memory barrier. > > > +} > > + > > +static int send_smc_message(struct scmi_channel *chan_info, > > + scmi_msg_header_t *hdr, void *data, int len) > > +{ > > + struct arm_smccc_res resp; > > + int ret; > > + > > + printk(XENLOG_DEBUG "scmi: status =%d len=%d\n", > > + chan_info->shmem->channel_status, len); > > + printk(XENLOG_DEBUG "scmi: header id = %d type = %d, proto = %d\n", > > + hdr->id, hdr->type, hdr->protocol); > > + > > + ret = channel_is_free(chan_info); > > + if ( IS_ERR_VALUE(ret) ) > > + return ret; > > + > > + chan_info->shmem->channel_status = 0x0; > > + /* Writing 0x0 right now, but SCMI_SHMEM_FLAG_INTR_ENABLED can be set */ > > + chan_info->shmem->flags = 0x0; > > + chan_info->shmem->length = sizeof(chan_info->shmem->msg_header) + len; > > + chan_info->shmem->msg_header = pack_scmi_header(hdr); > > + > > + printk(XENLOG_DEBUG "scmi: Writing to shmem address %p\n", > > + chan_info->shmem); > > + if ( len > 0 && data ) > > + memcpy((void *)(chan_info->shmem->msg_payload), data, len); > > Again, here we don't need a barrier because it is implicit in the SMC? > As I mentioned before, the other end runs on the same CPU. > Don't we need to check that "len" fits in the shared memory? > I think it's a good point. I'll add len check in v2. > > > + arm_smccc_smc(chan_info->func_id, 0, 0, 0, 0, 0, 0, chan_info->chan_id, > > + &resp); > > + > > + printk(XENLOG_DEBUG "scmi: scmccc_smc response %d\n", (int)(resp.a0)); > > + > > + if ( resp.a0 ) > > + return -EOPNOTSUPP; > > Why is that? > This change was presented in kernel by Sudeep Holla in commit: f7199cf489027ae38a9a82312d13025f7aefa0b8 However, link posted in the commit: https://lore.kernel.org/r/20200417103232.6896-1-sudeep.holla@arm.com Leads to slightly different patch: + if (res.a0 == SMCCC_RET_NOT_SUPPORTED) + return -EOPNOTSUPP; + else if (res.a0) + return -EINVAL; + return 0; I don't know why it differs from the original commit, but I'll check and place the correct implementation in v2. > > > + return 0; > > +} > > + > > +static int check_scmi_status(int scmi_status) > > +{ > > + if ( scmi_status == SCMI_SUCCESS ) > > + return 0; > > + > > + printk(XENLOG_DEBUG "scmi: Error received: %d\n", scmi_status); > > + > > + switch ( scmi_status ) > > + { > > + case SCMI_NOT_SUPPORTED: > > + return -EOPNOTSUPP; > > + case SCMI_INVALID_PARAMETERS: > > + return -EINVAL; > > + case SCMI_DENIED: > > + return -EACCES; > > + case SCMI_NOT_FOUND: > > + return -ENOENT; > > + case SCMI_OUT_OF_RANGE: > > + return -ERANGE; > > + case SCMI_BUSY: > > + return -EBUSY; > > + case SCMI_COMMS_ERROR: > > + return -ENOTCONN; > > + case SCMI_GENERIC_ERROR: > > + return -EIO; > > + case SCMI_HARDWARE_ERROR: > > + return -ENXIO; > > + case SCMI_PROTOCOL_ERROR: > > + return -EBADMSG; > > + } > > + > > + return -EINVAL; > > +} > > + > > +static int get_smc_response(struct scmi_channel *chan_info, > > + scmi_msg_header_t *hdr, void *data, int len) > > +{ > > + int recv_len; > > + int ret; > > + > > + printk(XENLOG_DEBUG "scmi: get smc responce msgid %d\n", hdr->id); > > + > > + ret = channel_is_free(chan_info); > > + if ( IS_ERR_VALUE(ret) ) > > + return ret; > > I am not familiar with the spec (do you have a link?) but is it expected > that the channel is "free" when actually we want to read a message on > the channel? > Here is the link https://developer.arm.com/documentation/den0056/latest Figure 6 in Section 5.1.1. Caller marks channel as busy, then callee process message and marks channel as free. We are implementing polling based communication flow. > > > > + recv_len = chan_info->shmem->length - sizeof(chan_info->shmem->msg_header); > > + > > + if ( recv_len < 0 ) > > + { > > + printk(XENLOG_ERR > > + "scmi: Wrong size of smc message. Data may be invalid\n"); > > + return -EINVAL; > > + } > > + > > + if ( recv_len > len ) > > + { > > + printk(XENLOG_ERR > > + "scmi: Not enough buffer for message %d, expecting %d\n", > > + recv_len, len); > > + return -EINVAL; > > + } > > + > > + unpack_scmi_header(chan_info->shmem->msg_header, hdr); > > + > > + if ( recv_len > 0 ) > > + { > > + memcpy(data, chan_info->shmem->msg_payload, recv_len); > > + } > > + > > + return 0; > > +} > > + > > +static int do_smc_xfer(struct scmi_channel *channel, scmi_msg_header_t *hdr, void *tx_data, int tx_size, > > + void *rx_data, int rx_size) > > +{ > > + int ret = 0; > > + > > + if ( !hdr ) > > + return -EINVAL; > > + > > + spin_lock(&channel->lock); > > + > > + ret = send_smc_message(channel, hdr, tx_data, tx_size); > > + if ( ret ) > > + goto clean; > > + > > + ret = get_smc_response(channel, hdr, rx_data, rx_size); > > +clean: > > + spin_unlock(&channel->lock); > > + > > + return ret; > > +} > > + > > +static struct scmi_channel *get_channel_by_id(uint8_t chan_id) > > +{ > > + struct scmi_channel *curr; > > + bool found = false; > > + > > + spin_lock(&scmi_data.channel_list_lock); > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > please use parenthesis around the inner if (also in other places) > Thank you for the remark. I will fix it in v2. > > > + if ( curr->chan_id == chan_id ) > > + { > > + found = true; > > + break; > > + } > > + > > + spin_unlock(&scmi_data.channel_list_lock); > > + if ( found ) > > + return curr; > > + > > + return NULL; > > +} > > + > > +static struct scmi_channel *get_channel_by_domain(uint8_t domain_id) > > Use domid_t for domain ids. Thanks, I will fix it in v2. > > Also, wouldn't it be better to implement it as: > > static inline struct scmi_channel *get_channel_by_domain(struct domain *d) { > return d->arch.sci > } > That's a good point. I will take a look on it and fix in v2. > > > +{ > > + struct scmi_channel *curr; > > + bool found = false; > > + > > + spin_lock(&scmi_data.channel_list_lock); > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > + if ( curr->domain_id == domain_id ) > > + { > > + found = true; > > + break; > > + } > > + > > + spin_unlock(&scmi_data.channel_list_lock); > > + if ( found ) > > + return curr; > > + > > + return NULL; > > +} > > + > > +static struct scmi_channel *aquire_scmi_channel(int domain_id) > > +{ > > + struct scmi_channel *curr; > > + bool found = false; > > + > > + ASSERT(domain_id != DOMID_INVALID && domain_id >= 0); > > + > > + spin_lock(&scmi_data.channel_list_lock); > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > + if ( (curr->domain_id == DOMID_INVALID) > > + && (curr->chan_id != HYP_CHANNEL) ) > > If you use DOMID_XEN for HYP_CHANNEL, then this check becomes more > intuitive > We do not have direct relation between channel id and domain id. One channel id can be reused by different domain_ids. So from my standpoint, DOMID_XEN doesn't fit here. > > > + { > > + curr->domain_id = domain_id; > > + found = true; > > + break; > > + } > > + > > + spin_unlock(&scmi_data.channel_list_lock); > > + if ( found ) > > + return curr; > > + > > + return NULL; > > +} > > + > > +static void relinquish_scmi_channel(struct scmi_channel *channel) > > +{ > > + spin_lock(&scmi_data.channel_list_lock); > > + ASSERT(channel != NULL); > > the ASSERT could be before the spin_lock > Thank you. I will fix it in v2. > > > + channel->domain_id = DOMID_INVALID; > > + spin_unlock(&scmi_data.channel_list_lock); > > +} > > + > > +static struct scmi_channel *smc_create_channel(uint8_t chan_id, > > + uint32_t func_id, uint64_t addr) > > +{ > > + struct scmi_channel *channel; > > + mfn_t mfn; > > + > > + channel = get_channel_by_id(chan_id); > > + if ( channel ) > > + return ERR_PTR(EEXIST); > > + > > + channel = xmalloc(struct scmi_channel); > > + if ( !channel ) > > + return ERR_PTR(ENOMEM); > > + > > + channel->chan_id = chan_id; > > + channel->func_id = func_id; > > + channel->domain_id = DOMID_INVALID; > > I take you are using DOMID_INVALID to mark a channel used by Xen itself? > If so, then DOMID_XEN would be more appropriate. > I use DOMID_INVALID to mark channel as free. > > > + mfn = maddr_to_mfn(addr); > > + channel->shmem = vmap(&mfn, 1); > > One thing to be careful is the mapping attributes, for a couple of > reasons. As you might be aware, the ARM architecture forbids mismatching > attributes for mapping memory in different places in the system. So the > attributes that we use here must be the same used by the firmware > (and/or the guest.) > > The second reason to be careful is that in the bindings example > Documentation/devicetree/bindings/firmware/arm,scmi.yaml the shared > memory is "mmio-sram", which is special. It is not supposed to be normal > memory, but it is OK to map it cacheable. Still, it might be more > appropriate to use ioremap_cache. > Originally, I used vmap here to have memcpy and it works fine in our setup. But I will do some research and email you with the results. > > > + if ( !channel->shmem ) > > + { > > + xfree(channel); > > + return ERR_PTR(ENOMEM); > > + } > > + > > + printk(XENLOG_DEBUG "scmi: Got shmem after vmap %p\n", channel->shmem); > > + channel->paddr = addr; > > + channel->shmem->channel_status = SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE; > > + spin_lock_init(&channel->lock); > > + spin_lock(&scmi_data.channel_list_lock); > > + list_add(&channel->list, &scmi_data.channel_list); > > + spin_unlock(&scmi_data.channel_list_lock); > > + return channel; > > +} > > + > > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > > +{ > > + return iomem_permit_access(d, paddr_to_pfn(addr), > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > +} > > + > > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > > + uint64_t len) > > +{ > > + return iomem_deny_access(d, paddr_to_pfn(addr), > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > +} > > + > > +static int dt_update_domain_range(struct domain *d, uint64_t addr, > > + uint64_t size) > > +{ > > + struct dt_device_node *shmem_node; > > + __be32 *hw_reg; > > + const struct dt_property *pp; > > + uint32_t len; > > + > > + shmem_node = dt_find_compatible_node(NULL, NULL, "arm,scmi-shmem"); > > Here we are using "arm,scmi-shmem" while below we are checking for > "linux,scmi_mem". What's the difference? linux,scmi_mem (I posted nodes examples above) describes memory region, allocated for all channels, while arm,scmi-shmem points to the exact channel (page in linux,scmi_mem region). > > Also, this function is looking for "arm,scmi-shmem" in dt_host and > replaces its value. For dom0less domUs we'll probably need a > make_scmi_node function to create the node from scratch like for > instance xen/arch/arm/domain_build.c:make_gic_domU_node. > > I wonder if we had such a function whether it wouldn't be better to also > use it for dom0 (and blacklist the physical "arm,scmi-shmem" in > handle_node so that dom0 doesn't get the real shared memory information > by accident). > Thank you for the remark. I will rework this in v2. > > > + > > + if ( !shmem_node ) > > + { > > + printk(XENLOG_ERR "scmi: Unable to find %s node in DT\n", SCMI_SHMEM); > > + return -EINVAL; > > + } > > + > > + pp = dt_find_property(shmem_node, "reg", &len); > > + if ( !pp ) > > + { > > + printk(XENLOG_ERR "scmi: Unable to find regs entry in shmem node\n"); > > + return -ENOENT; > > + } > > + > > + hw_reg = pp->value; > > + dt_set_range(&hw_reg, shmem_node, addr, size); > > + > > + return 0; > > +} > > + > > +static void free_channel_list(void) > > +{ > > + struct scmi_channel *curr, *_curr; > > + > > + spin_lock(&scmi_data.channel_list_lock); > > + list_for_each_entry_safe (curr, _curr, &scmi_data.channel_list, list) > > + { > > + vunmap(curr->shmem); > > + list_del(&curr->list); > > + xfree(curr); > > + } > > + > > + spin_unlock(&scmi_data.channel_list_lock); > > +} > > + > > +static __init bool scmi_probe(struct dt_device_node *scmi_node) > > +{ > > + struct dt_device_node *shmem_node; > > + int ret, i; > > + struct scmi_channel *channel, *agent_channel; > > + int n_agents; > > + scmi_msg_header_t hdr; > > + struct rx_t { > > + int32_t status; > > + uint32_t attributes; > > + } rx; > > Should rx be defined at the top together with scmi_perms_tx_t and > others? > I'd rather move scmi_perms_tx_t to scmi_add_device_by_devid because it's used only in 1 place. So we will have rx and tx in scmi_add_device_by_devid and rx ( which differs from rx in scmi_add_device_by_devid ) in scmi_probe. I think it will be more understandable and no need to make unique names. What do you think about that? > > > + uint32_t func_id; > > + > > + ASSERT(scmi_node != NULL); > > + > > + INIT_LIST_HEAD(&scmi_data.channel_list); > > + spin_lock_init(&scmi_data.channel_list_lock); > > + > > + if ( !dt_property_read_u32(scmi_node, SCMI_SMC_ID, &func_id) ) > > + { > > + printk(XENLOG_ERR "scmi: Unable to read smc-id from DT\n"); > > + return false; > > + } > > + > > + shmem_node = dt_find_node_by_name(NULL, SCMI_SHARED_MEMORY); > > From the spec, it looks like you should be getting the shared memory > area from the phandle list "shmem". > We use SCMI_SHARED_MEMORY to get whole memory region (0x10 pages in my case), we can use for the agents. As you can see below - Hypervisor received number of agents from Firmware and split this region between agents. > > > + if ( IS_ERR_OR_NULL(shmem_node) ) > > + { > > + printk(XENLOG_ERR > > + "scmi: Device tree error, can't parse shmem phandle %ld\n", > > + PTR_ERR(shmem_node)); > > + return false; > > + } > > + > > + ret = dt_device_get_address(shmem_node, 0, &scmi_data.shmem_addr, > > + &scmi_data.shmem_size); > > + if ( IS_ERR_VALUE(ret) ) > > + return false; > > + > > + channel = smc_create_channel(HYP_CHANNEL, func_id, scmi_data.shmem_addr); > > + if ( IS_ERR(channel) ) > > + return false; > > + > > + hdr.id = SCMI_BASE_PROTOCOL_ATTIBUTES; > > + hdr.type = 0; > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > + > > + ret = do_smc_xfer(channel, &hdr, NULL, 0, &rx, sizeof(rx)); > > + if ( ret ) > > + goto clean; > > + > > + ret = check_scmi_status(rx.status); > > + if ( ret ) > > + goto clean; > > + > > + n_agents = FIELD_GET(MSG_N_AGENTS_MASK, rx.attributes); > > + printk(XENLOG_DEBUG "scmi: Got agent count %d\n", n_agents); > > + > > + n_agents = (n_agents > scmi_data.shmem_size / PAGE_SIZE) ? > > + scmi_data.shmem_size / PAGE_SIZE : n_agents; > > + > > + for ( i = 1; i < n_agents; i++ ) > > + { > > Given that HYP_CHANNEL is actually zero, it looks like we could do > everything here in this loop but starting from i=0? > We allocate HYP_CHANNEL before loop because we need it to request number of agents. And we don't need to send SCMI_BASE_DISCOVER_AGENT to HYP_CHANNEL. > > > + uint32_t tx_agent_id = 0xFFFFFFFF; > > + struct { > > + int32_t status; > > + uint32_t agent_id; > > + char name[16]; > > + } da_rx; > > + > > + agent_channel = smc_create_channel(i, func_id, scmi_data.shmem_addr + > > + i * PAGE_SIZE); > > + if ( IS_ERR(agent_channel) ) > > + { > > + ret = PTR_ERR(agent_channel); > > + goto clean; > > + } > > + > > + hdr.id = SCMI_BASE_DISCOVER_AGENT; > > + hdr.type = 0; > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > + > > + ret = do_smc_xfer(agent_channel, &hdr, &tx_agent_id, > > + sizeof(tx_agent_id), &da_rx, sizeof(da_rx)); > > + if ( ret ) > > + goto clean; > > + > > + ret = check_scmi_status(da_rx.status); > > + if ( ret ) > > + goto clean; > > + > > + printk(XENLOG_DEBUG "scmi: status=0x%x id=0x%x name=%s\n", > > + da_rx.status, da_rx.agent_id, da_rx.name); > > + > > + agent_channel->agent_id = da_rx.agent_id; > > + } > > + > > + scmi_data.initialized = true; > > + return true; > > + > > +clean: > > + free_channel_list(); > > + return ret == 0; > > +} > > + > > +static int scmi_domain_init(struct domain *d) > > +{ > > + struct scmi_channel *channel; > > + int ret; > > + > > + if ( !scmi_data.initialized ) > > + return 0; > > + > > + channel = aquire_scmi_channel(d->domain_id); > > + if ( IS_ERR_OR_NULL(channel) ) > > + return -ENOENT; > > + > > + printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = %d" > > + "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, > > + channel->paddr); > > + > > + if ( is_hardware_domain(d) ) > > + { > > + ret = map_memory_to_domain(d, scmi_data.shmem_addr, > > + scmi_data.shmem_size); > > + if ( IS_ERR_VALUE(ret) ) > > + goto error; > > + > > + ret = dt_update_domain_range(d, channel->paddr, PAGE_SIZE); > > + if ( IS_ERR_VALUE(ret) ) > > + { > > + int rc = unmap_memory_from_domain(d, scmi_data.shmem_addr, > > + scmi_data.shmem_size); > > + if ( rc ) > > + printk(XENLOG_ERR "Unable to unmap_memory_from_domain\n"); > > + > > + goto error; > > + } > > + } > > Is dom0 the only domain to get direct access to the shared memory > region? If so, I don't think it is a good idea to make Dom0 "special" in > this case. > > Let me make an example: if we assign a device to a domU since boot, and > dom0 wants to change the frequency of a clock that affects the assigned > device (likely because it doesn't know it is assigned), then dom0 > shouldn't be able to. We might have to perform checks in Xen to make > sure dom0 cannot stop the clock for the assigned device. > > So I think it would be better if all domains are treated the same way in > the mediator unless really necessary. > > On the other hand, if all domains get access to the shared memory > region, then I don't think this is likely the right place to create the > dom0 mapping. We probably want to do it in domain_build.c in a way that > can be reused for dom0less domUs. > The idea is that all domains have their own page in shared memory region and unigue agent_id. Agent_id is used to set permissions for clocks\resets\power-domains etc. So during creation of domain (domUs or dom0) device-tree is processed using scmi_add_dt_device and clocks\resets\power-domains which are related to dom0 devices will be requested by using SCMI_BASE_SET_DEVICE_PERMISSIONS message. All passed-through devices will be requested during DomU creation. Which means dom0 do not have an access to clocks\resets\power-domains, which are related to DomU. > > In regards to shared memory: it looks like the only two functions to > access the real shared memory are send_smc_message and get_smc_response. > If that is the case, then we actually don't need to expose the real > shared memory to any of the domains. > > We could simply: > > - expose a regular normal memory region as dom0/domU channel memory > - on SMC trap, read from the "fake" shared memory and set the > corresponding real shared memory on the appropriate channel > - issue the SMC call > - on return from SMC, copy over data from the real shared memory to the > "fake" channel reagion Hypervisor redirects only SMC calls from guests and set agent_id to SMC parameters as a7. The idea was to give page for each agent, so we don't need to make additional read/write each time we receive SMC call. All we povide from hypervisor is agent_id. Firmware is responsible for reading memory from the correct address and place the response. > > This is useful if we need to "filter" any of the SCMI commands and > options from the domains to the firmware, and also it is useful if the > channel memory is not page aligned. But if the permissions are > fine-grained enough and also the channel memory is page aligned (and > multiple of 4K in size) then we could map the memory. > In current implementation we suppose that channel memory is page aligned. I think that Firmware should be responsible for permissions handling and "filtering", that's why permission calls were added to SCMI spec. I tried to make mediator as thin as possible. > > > + > > + d->arch.sci = channel; > > + > > + return 0; > > +error: > > + relinquish_scmi_channel(channel); > > + > > + return ret; > > +} > > + > > +static int scmi_add_device_by_devid(struct domain *d, uint32_t scmi_devid) > > +{ > > + struct scmi_channel *channel, *agent_channel; > > + scmi_msg_header_t hdr; > > + scmi_perms_tx_t tx; > > + struct rx_t { > > + int32_t status; > > + uint32_t attributes; > > + } rx; > > + int ret; > > + > > + if ( !scmi_data.initialized ) > > + return 0; > > + > > + printk(XENLOG_DEBUG "scmi: scmi_devid = %d\n", scmi_devid); > > + > > + agent_channel = get_channel_by_domain(d->domain_id); > > + if ( IS_ERR_OR_NULL(agent_channel) ) > > + return PTR_ERR(agent_channel); > > + > > + channel = get_channel_by_id(HYP_CHANNEL); > > + if ( IS_ERR_OR_NULL(channel) ) > > + return PTR_ERR(channel); > > + > > + hdr.id = SCMI_BASE_SET_DEVICE_PERMISSIONS; > > + hdr.type = 0; > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > + > > + tx.agent_id = agent_channel->agent_id; > > + tx.device_id = scmi_devid; > > + tx.flags = SCMI_ALLOW_ACCESS; > > + > > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(&rx)); > > + if ( IS_ERR_VALUE(ret) ) > > + return ret; > > + > > + ret = check_scmi_status(rx.status); > > + if ( IS_ERR_VALUE(ret) ) > > + return ret; > > + > > + return 0; > > +} > > + > > +static int scmi_add_dt_device(struct domain *d, struct dt_device_node *dev) > > +{ > > + uint32_t scmi_devid; > > + > > + if ( (!scmi_data.initialized) || (!d->arch.sci) ) > > + return 0; > > + > > + if ( !dt_property_read_u32(dev, "scmi_devid", &scmi_devid) ) > > + return 0; > > scmi_devid is another property that is not documented in the binding. > This property should be added to the device nodes, which are using scmi to work with clocks\resets\power-domains etc. This id should match the device_id, defined in Firmware. Hypervisor send this device_id to the Firmware as parameter to the permission request. Firmware set permissions to clocks\resets\power-domains, related to this device. > > > + printk(XENLOG_INFO "scmi: dt_node = %s\n", dt_node_full_name(dev)); > > + > > + return scmi_add_device_by_devid(d, scmi_devid); > > +} > > + > > +static int scmi_relinquish_resources(struct domain *d) > > +{ > > + int ret; > > + struct scmi_channel *channel, *agent_channel; > > + scmi_msg_header_t hdr; > > + struct reset_agent_tx { > > + uint32_t agent_id; > > + uint32_t flags; > > + } tx; > > + uint32_t rx; > > + > > + if ( !d->arch.sci ) > > + return 0; > > + > > + agent_channel = d->arch.sci; > > + > > + spin_lock(&agent_channel->lock); > > + tx.agent_id = agent_channel->agent_id; > > + spin_unlock(&agent_channel->lock); > > + > > + channel = get_channel_by_id(HYP_CHANNEL); > > + if ( !channel ) > > + { > > + printk(XENLOG_ERR > > + "scmi: Unable to get Hypervisor scmi channel for domain %d\n", > > + d->domain_id); > > + return -EINVAL; > > + } > > + > > + hdr.id = SCMI_BASE_RESET_AGENT_CONFIGURATION; > > + hdr.type = 0; > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > + > > + tx.flags = 0; > > + > > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(rx)); > > + if ( ret ) > > + return ret; > > + > > + ret = check_scmi_status(rx); > > + > > + return ret; > > +} > > + > > +static void scmi_domain_destroy(struct domain *d) > > +{ > > + struct scmi_channel *channel; > > + > > + if ( !d->arch.sci ) > > + return; > > + > > + channel = d->arch.sci; > > + spin_lock(&channel->lock); > > + > > + relinquish_scmi_channel(channel); > > + printk(XENLOG_DEBUG "scmi: Free domain %d\n", d->domain_id); > > + > > + d->arch.sci = NULL; > > + > > + unmap_memory_from_domain(d, channel->paddr, PAGE_SIZE); > > + spin_unlock(&channel->lock); > > + return; > > +} > > + > > +static bool scmi_handle_call(struct domain *d, void *args) > > +{ > > + bool res = false; > > + struct scmi_channel *agent_channel; > > + struct arm_smccc_res resp; > > + struct cpu_user_regs *regs = args; > > + > > + if ( !d->arch.sci ) > > + return false; > > + > > + agent_channel = d->arch.sci; > > + spin_lock(&agent_channel->lock); > > + > > + if ( agent_channel->func_id != regs->x0 ) > > + { > > + printk(XENLOG_ERR "scmi: func_id mismatch, exiting\n"); > > + goto unlock; > > + } > > + > > + arm_smccc_smc(agent_channel->func_id, 0, 0, 0, 0, 0, 0, > > + agent_channel->chan_id, &resp); > > + > > + set_user_reg(regs, 0, resp.a0); > > + set_user_reg(regs, 1, resp.a1); > > + set_user_reg(regs, 2, resp.a2); > > + set_user_reg(regs, 3, resp.a3); > > + res = true; > > +unlock: > > + spin_unlock(&agent_channel->lock); > > + > > + return res; > > +} > > + > > +static int scmi_get_channel_paddr(void *scmi_ops, > > + struct xen_arch_domainconfig *config) > > +{ > > + struct scmi_channel *agent_channel = scmi_ops; > > + > > + if ( !agent_channel ) > > + return -EINVAL; > > + > > + config->sci_agent_paddr = agent_channel->paddr; > > + return 0; > > +} > > I am still not sure why it couldn't be done by scmi_domain_init. > I can move this logic to scmi_domain_init, but in this case I have to add struct xen_arch_domainconfig *config as input parameter to scmi_domain_init and pass NULL from construct_dom0. Do you think this approach would be better? Also I think it's reasonable to pass xen_arch_domainconfig so different implementations could set another data they would probably need. > > > +static const struct dt_device_match scmi_smc_match[] __initconst = > > +{ > > + DT_MATCH_SCMI_SMC, > > + { /* sentinel */ }, > > +}; > > + > > +static const struct sci_mediator_ops scmi_ops = > > +{ > > + .probe = scmi_probe, > > + .domain_init = scmi_domain_init, > > + .domain_destroy = scmi_domain_destroy, > > + .add_dt_device = scmi_add_dt_device, > > + .relinquish_resources = scmi_relinquish_resources, > > + .handle_call = scmi_handle_call, > > + .get_channel_info = scmi_get_channel_paddr > > +}; > > + > > +REGISTER_SCI_MEDIATOR(scmi_smc, "SCMI-SMC", XEN_DOMCTL_CONFIG_SCI_SCMI_SMC, > > + scmi_smc_match, &scmi_ops); > > + > > +/* > > + * Local variables: > > + * mode: C > > + * c-file-style: "BSD" > > + * c-basic-offset: 4 > > + * indent-tabs-mode: nil > > + * End: > > + */ > > diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h > > index 9180be5e86..a67237942d 100644 > > --- a/xen/include/public/arch-arm.h > > +++ b/xen/include/public/arch-arm.h > > @@ -315,6 +315,7 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t); > > #define XEN_DOMCTL_CONFIG_TEE_OPTEE 1 > > > > #define XEN_DOMCTL_CONFIG_SCI_NONE 0 > > +#define XEN_DOMCTL_CONFIG_SCI_SCMI_SMC 1 > > > > struct xen_arch_domainconfig { > > /* IN/OUT */ > > -- > > 2.27.0 > >
On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > Hi Stefano, > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > This allows devices from the Domains to work with clocks, resets and > > > power-domains without access to CPG. > > > > > > The following features are implemented: > > > - request SCMI channels from ATF and pass channels to Domains; > > > - set device permissions for Domains based on the Domain partial > > > device-tree. Devices with permissions are able to work with clocks, > > > resets and power-domains via SCMI; > > > - redirect scmi messages from Domains to ATF. > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > --- > > > xen/arch/arm/Kconfig | 2 + > > > xen/arch/arm/sci/Kconfig | 10 + > > > xen/arch/arm/sci/Makefile | 1 + > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > xen/include/public/arch-arm.h | 1 + > > > 5 files changed, 809 insertions(+) > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > index 186e1db389..02d96c6cfc 100644 > > > --- a/xen/arch/arm/Kconfig > > > +++ b/xen/arch/arm/Kconfig > > > @@ -114,6 +114,8 @@ config SCI > > > support. It allows guests to control system resourcess via one of > > > SCI mediators implemented in XEN. > > > > > > +source "arch/arm/sci/Kconfig" > > > + > > > endmenu > > > > > > menu "ARM errata workaround via the alternative framework" > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > new file mode 100644 > > > index 0000000000..9563067ddc > > > --- /dev/null > > > +++ b/xen/arch/arm/sci/Kconfig > > > @@ -0,0 +1,10 @@ > > > +config SCMI_SMC > > > + bool "Enable SCMI-SMC mediator driver" > > > + default n > > > + depends on SCI > > > + ---help--- > > > + > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > + This feature allows drivers from Domains to work with System > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > + for communication. > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > index 837dc7492b..67f2611872 100644 > > > --- a/xen/arch/arm/sci/Makefile > > > +++ b/xen/arch/arm/sci/Makefile > > > @@ -1 +1,2 @@ > > > obj-y += sci.o > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > new file mode 100644 > > > index 0000000000..2eb01ea82d > > > --- /dev/null > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > @@ -0,0 +1,795 @@ > > > +/* > > > + * xen/arch/arm/sci/scmi_smc.c > > > + * > > > + * SCMI mediator driver, using SCP as transport. > > > + * > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > + * Copyright (C) 2021, EPAM Systems. > > > + * > > > + * This program is free software; you can redistribute it and/or modify > > > + * it under the terms of the GNU General Public License as published by > > > + * the Free Software Foundation; either version 2 of the License, or > > > + * (at your option) any later version. > > > + * > > > + * This program is distributed in the hope that it will be useful, > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > + * GNU General Public License for more details. > > > + */ > > > + > > > +#include <asm/sci/sci.h> > > > +#include <asm/smccc.h> > > > +#include <asm/io.h> > > > +#include <xen/bitops.h> > > > +#include <xen/config.h> > > > +#include <xen/sched.h> > > > +#include <xen/device_tree.h> > > > +#include <xen/iocap.h> > > > +#include <xen/init.h> > > > +#include <xen/err.h> > > > +#include <xen/lib.h> > > > +#include <xen/list.h> > > > +#include <xen/mm.h> > > > +#include <xen/string.h> > > > +#include <xen/time.h> > > > +#include <xen/vmap.h> > > > + > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > + > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > +#define SCMI_SUCCESS 0 > > > +#define SCMI_NOT_SUPPORTED (-1) > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > +#define SCMI_DENIED (-3) > > > +#define SCMI_NOT_FOUND (-4) > > > +#define SCMI_OUT_OF_RANGE (-5) > > > +#define SCMI_BUSY (-6) > > > +#define SCMI_COMMS_ERROR (-7) > > > +#define SCMI_GENERIC_ERROR (-8) > > > +#define SCMI_HARDWARE_ERROR (-9) > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > + > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > + > > > +#define SCMI_SMC_ID "arm,smc-id" > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > I could find the following SCMI binding in Linux, which describes > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > the "shmem" property instead? And the compatible string used for this > > seems to be "arm,scmi-shmem". > > > > We use linux,scmi_mem node to reserve memory, needed for all > channels: > > reserved-memory { > /* reserved region for scmi channels*/ > scmi_memory: linux,scmi_mem@53FF0000 { > no-map; > reg = <0x0 0x53FF0000 0x0 0x10000>; > }; > }; > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > the current scmi channel: > > cpu_scp_shm: scp-shmem@0x53FF0000 { > compatible = "arm,scmi-shmem"; > reg = <0x0 0x53FF0000 0x0 0x1000>; > }; > > For each Domain reg points to unigue page from linux,scmi_mem region, > assigned to this agent. If we were to use "linux,scmi_mem" we would have to introduce it as a compatible string, not as a node name, and it would need to be described in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. But from your description I don't think it is necessary. We can just use "arm,scmi-shmem" to describe all the required regions: reserved-memory { scp-shmem@0x53FF0000 { compatible = "arm,scmi-shmem"; reg = <0x0 0x53FF0000 0x0 0x1000>; }; scp-shmem@0x53FF1000 { compatible = "arm,scmi-shmem"; reg = <0x0 0x53FF1000 0x0 0x1000>; }; scp-shmem@0x53FF2000 { compatible = "arm,scmi-shmem"; reg = <0x0 0x53FF2000 0x0 0x1000>; }; ... In other words, if all the individual channel pages are described as "arm,scmi-shmem", why do we also need a single larger region as "linux,scmi_mem"? > > > +#define SCMI_SHMEM "shmem" > > > + > > > +#define HYP_CHANNEL 0x0 > > > + > > > +#define HDR_ID GENMASK(7,0) > > > +#define HDR_TYPE GENMASK(9, 8) > > > +#define HDR_PROTO GENMASK(17, 10) > > > + > > > +/* SCMI protocol, refer to section 4.2.2.2 (DEN0056C) */ > > > +#define MSG_N_AGENTS_MASK GENMASK(15, 8) > > > + > > > +#define FIELD_GET(_mask, _reg)\ > > > + ((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1))) > > > +#define FIELD_PREP(_mask, _val)\ > > > + (((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask)) > > > + > > > +typedef struct scmi_msg_header { > > > + uint8_t id; > > > + uint8_t type; > > > + uint8_t protocol; > > > +} scmi_msg_header_t; > > > + > > > +typedef struct scmi_perms_tx { > > > + uint32_t agent_id; > > > + uint32_t device_id; > > > + uint32_t flags; > > > +} scmi_perms_tx_t; > > > + > > > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE BIT(0, UL) > > > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_ERROR BIT(1, UL) > > > + > > > +#define SCMI_ALLOW_ACCESS BIT(0, UL) > > > + > > > +struct scmi_shared_mem { > > > + uint32_t reserved; > > > + uint32_t channel_status; > > > + uint32_t reserved1[2]; > > > + uint32_t flags; > > > + uint32_t length; > > > + uint32_t msg_header; > > > + uint8_t msg_payload[]; > > > +}; > > > + > > > +struct scmi_channel { > > > + int chan_id; > > > + int agent_id; > > > + uint32_t func_id; > > > + int domain_id; > > > + uint64_t paddr; > > > + struct scmi_shared_mem *shmem; > > > + spinlock_t lock; > > > + struct list_head list; > > > +}; > > > + > > > +struct scmi_data { > > > + struct list_head channel_list; > > > + spinlock_t channel_list_lock; > > > + bool initialized; > > > + u64 shmem_addr, shmem_size; > > > +}; > > > + > > > +static struct scmi_data scmi_data; > > > + > > > +/* > > > + * pack_scmi_header() - packs and returns 32-bit header > > > + * > > > + * @hdr: pointer to header containing all the information on message id, > > > + * protocol id and type id. > > > + * > > > + * Return: 32-bit packed message header to be sent to the platform. > > > + */ > > > +static inline uint32_t pack_scmi_header(scmi_msg_header_t *hdr) > > > +{ > > > + return FIELD_PREP(HDR_ID, hdr->id) | > > > + FIELD_PREP(HDR_TYPE, hdr->type) | > > > + FIELD_PREP(HDR_PROTO, hdr->protocol); > > > +} > > > + > > > +/* > > > + * unpack_scmi_header() - unpacks and records message and protocol id > > > + * > > > + * @msg_hdr: 32-bit packed message header sent from the platform > > > + * @hdr: pointer to header to fetch message and protocol id. > > > + */ > > > +static inline void unpack_scmi_header(uint32_t msg_hdr, scmi_msg_header_t *hdr) > > > +{ > > > + hdr->id = FIELD_GET(HDR_ID, msg_hdr); > > > + hdr->type = FIELD_GET(HDR_TYPE, msg_hdr); > > > + hdr->protocol = FIELD_GET(HDR_PROTO, msg_hdr); > > > +} > > > + > > > +static inline int channel_is_free(struct scmi_channel *chan_info) > > > +{ > > > + return ( chan_info->shmem->channel_status > > > + & SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE ) ? 0 : -EBUSY; > > > > Does this need a memory barrier? Or not, because the other end always > > runs on the same CPU at a different execution level so the > > channel_status would be always guaranteed to be read as updated? > > > > It don't because the other end runs on the same CPU. Other mediator > implemetaions, which uses different areas may need memory barrier. > > > > > > +} > > > + > > > +static int send_smc_message(struct scmi_channel *chan_info, > > > + scmi_msg_header_t *hdr, void *data, int len) > > > +{ > > > + struct arm_smccc_res resp; > > > + int ret; > > > + > > > + printk(XENLOG_DEBUG "scmi: status =%d len=%d\n", > > > + chan_info->shmem->channel_status, len); > > > + printk(XENLOG_DEBUG "scmi: header id = %d type = %d, proto = %d\n", > > > + hdr->id, hdr->type, hdr->protocol); > > > + > > > + ret = channel_is_free(chan_info); > > > + if ( IS_ERR_VALUE(ret) ) > > > + return ret; > > > + > > > + chan_info->shmem->channel_status = 0x0; > > > + /* Writing 0x0 right now, but SCMI_SHMEM_FLAG_INTR_ENABLED can be set */ > > > + chan_info->shmem->flags = 0x0; > > > + chan_info->shmem->length = sizeof(chan_info->shmem->msg_header) + len; > > > + chan_info->shmem->msg_header = pack_scmi_header(hdr); > > > + > > > + printk(XENLOG_DEBUG "scmi: Writing to shmem address %p\n", > > > + chan_info->shmem); > > > + if ( len > 0 && data ) > > > + memcpy((void *)(chan_info->shmem->msg_payload), data, len); > > > > Again, here we don't need a barrier because it is implicit in the SMC? > > > > As I mentioned before, the other end runs on the same CPU. > > > Don't we need to check that "len" fits in the shared memory? > > > > I think it's a good point. I'll add len check in v2. > > > > > > + arm_smccc_smc(chan_info->func_id, 0, 0, 0, 0, 0, 0, chan_info->chan_id, > > > + &resp); > > > + > > > + printk(XENLOG_DEBUG "scmi: scmccc_smc response %d\n", (int)(resp.a0)); > > > + > > > + if ( resp.a0 ) > > > + return -EOPNOTSUPP; > > > > Why is that? > > > > This change was presented in kernel by Sudeep Holla in commit: > f7199cf489027ae38a9a82312d13025f7aefa0b8 > > However, link posted in the commit: > https://lore.kernel.org/r/20200417103232.6896-1-sudeep.holla@arm.com > > Leads to slightly different patch: > > + if (res.a0 == SMCCC_RET_NOT_SUPPORTED) > + return -EOPNOTSUPP; > + else if (res.a0) > + return -EINVAL; > + return 0; > > I don't know why it differs from the original commit, but I'll check and place > the correct implementation in v2. > > > > > > + return 0; > > > +} > > > + > > > +static int check_scmi_status(int scmi_status) > > > +{ > > > + if ( scmi_status == SCMI_SUCCESS ) > > > + return 0; > > > + > > > + printk(XENLOG_DEBUG "scmi: Error received: %d\n", scmi_status); > > > + > > > + switch ( scmi_status ) > > > + { > > > + case SCMI_NOT_SUPPORTED: > > > + return -EOPNOTSUPP; > > > + case SCMI_INVALID_PARAMETERS: > > > + return -EINVAL; > > > + case SCMI_DENIED: > > > + return -EACCES; > > > + case SCMI_NOT_FOUND: > > > + return -ENOENT; > > > + case SCMI_OUT_OF_RANGE: > > > + return -ERANGE; > > > + case SCMI_BUSY: > > > + return -EBUSY; > > > + case SCMI_COMMS_ERROR: > > > + return -ENOTCONN; > > > + case SCMI_GENERIC_ERROR: > > > + return -EIO; > > > + case SCMI_HARDWARE_ERROR: > > > + return -ENXIO; > > > + case SCMI_PROTOCOL_ERROR: > > > + return -EBADMSG; > > > + } > > > + > > > + return -EINVAL; > > > +} > > > + > > > +static int get_smc_response(struct scmi_channel *chan_info, > > > + scmi_msg_header_t *hdr, void *data, int len) > > > +{ > > > + int recv_len; > > > + int ret; > > > + > > > + printk(XENLOG_DEBUG "scmi: get smc responce msgid %d\n", hdr->id); > > > + > > > + ret = channel_is_free(chan_info); > > > + if ( IS_ERR_VALUE(ret) ) > > > + return ret; > > > > I am not familiar with the spec (do you have a link?) but is it expected > > that the channel is "free" when actually we want to read a message on > > the channel? > > > > Here is the link https://developer.arm.com/documentation/den0056/latest > Figure 6 in Section 5.1.1. > Caller marks channel as busy, then callee process message and marks channel as free. > We are implementing polling based communication flow. OK > > > + recv_len = chan_info->shmem->length - sizeof(chan_info->shmem->msg_header); > > > + > > > + if ( recv_len < 0 ) > > > + { > > > + printk(XENLOG_ERR > > > + "scmi: Wrong size of smc message. Data may be invalid\n"); > > > + return -EINVAL; > > > + } > > > + > > > + if ( recv_len > len ) > > > + { > > > + printk(XENLOG_ERR > > > + "scmi: Not enough buffer for message %d, expecting %d\n", > > > + recv_len, len); > > > + return -EINVAL; > > > + } > > > + > > > + unpack_scmi_header(chan_info->shmem->msg_header, hdr); > > > + > > > + if ( recv_len > 0 ) > > > + { > > > + memcpy(data, chan_info->shmem->msg_payload, recv_len); > > > + } > > > + > > > + return 0; > > > +} > > > + > > > +static int do_smc_xfer(struct scmi_channel *channel, scmi_msg_header_t *hdr, void *tx_data, int tx_size, > > > + void *rx_data, int rx_size) > > > +{ > > > + int ret = 0; > > > + > > > + if ( !hdr ) > > > + return -EINVAL; > > > + > > > + spin_lock(&channel->lock); > > > + > > > + ret = send_smc_message(channel, hdr, tx_data, tx_size); > > > + if ( ret ) > > > + goto clean; > > > + > > > + ret = get_smc_response(channel, hdr, rx_data, rx_size); > > > +clean: > > > + spin_unlock(&channel->lock); > > > + > > > + return ret; > > > +} > > > + > > > +static struct scmi_channel *get_channel_by_id(uint8_t chan_id) > > > +{ > > > + struct scmi_channel *curr; > > > + bool found = false; > > > + > > > + spin_lock(&scmi_data.channel_list_lock); > > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > > > please use parenthesis around the inner if (also in other places) > > > > Thank you for the remark. I will fix it in v2. > > > > > > + if ( curr->chan_id == chan_id ) > > > + { > > > + found = true; > > > + break; > > > + } > > > + > > > + spin_unlock(&scmi_data.channel_list_lock); > > > + if ( found ) > > > + return curr; > > > + > > > + return NULL; > > > +} > > > + > > > +static struct scmi_channel *get_channel_by_domain(uint8_t domain_id) > > > > Use domid_t for domain ids. > > Thanks, I will fix it in v2. > > > > Also, wouldn't it be better to implement it as: > > > > static inline struct scmi_channel *get_channel_by_domain(struct domain *d) { > > return d->arch.sci > > } > > > That's a good point. I will take a look on it and fix in v2. > > > > > +{ > > > + struct scmi_channel *curr; > > > + bool found = false; > > > + > > > + spin_lock(&scmi_data.channel_list_lock); > > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > > + if ( curr->domain_id == domain_id ) > > > + { > > > + found = true; > > > + break; > > > + } > > > + > > > + spin_unlock(&scmi_data.channel_list_lock); > > > + if ( found ) > > > + return curr; > > > + > > > + return NULL; > > > +} > > > + > > > +static struct scmi_channel *aquire_scmi_channel(int domain_id) > > > +{ > > > + struct scmi_channel *curr; > > > + bool found = false; > > > + > > > + ASSERT(domain_id != DOMID_INVALID && domain_id >= 0); > > > + > > > + spin_lock(&scmi_data.channel_list_lock); > > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > > + if ( (curr->domain_id == DOMID_INVALID) > > > + && (curr->chan_id != HYP_CHANNEL) ) > > > > If you use DOMID_XEN for HYP_CHANNEL, then this check becomes more > > intuitive > > > > We do not have direct relation between channel id and domain id. > One channel id can be reused by different domain_ids. So from my standpoint, > DOMID_XEN doesn't fit here. Below you wrote that you used DOMID_INVALID to mark a channel as free. That is fine, but then DOMID_INVALID shouldn't be used for HYP_CHANNEL because HYP_CHANNEL is not "free". In this function the check for curr->chan_id != HYP_CHANNEL should be unnecessary and I think the code would look as follows: list_for_each_entry(curr, &scmi_data.channel_list, list) { if ( (curr->domain_id == DOMID_INVALID) ) { > > > + { > > > + curr->domain_id = domain_id; > > > + found = true; > > > + break; > > > + } > > > + > > > + spin_unlock(&scmi_data.channel_list_lock); > > > + if ( found ) > > > + return curr; > > > + > > > + return NULL; > > > +} > > > + > > > +static void relinquish_scmi_channel(struct scmi_channel *channel) > > > +{ > > > + spin_lock(&scmi_data.channel_list_lock); > > > + ASSERT(channel != NULL); > > > > the ASSERT could be before the spin_lock > > > > Thank you. I will fix it in v2. > > > > > > + channel->domain_id = DOMID_INVALID; > > > + spin_unlock(&scmi_data.channel_list_lock); > > > +} > > > + > > > +static struct scmi_channel *smc_create_channel(uint8_t chan_id, > > > + uint32_t func_id, uint64_t addr) > > > +{ > > > + struct scmi_channel *channel; > > > + mfn_t mfn; > > > + > > > + channel = get_channel_by_id(chan_id); > > > + if ( channel ) > > > + return ERR_PTR(EEXIST); > > > + > > > + channel = xmalloc(struct scmi_channel); > > > + if ( !channel ) > > > + return ERR_PTR(ENOMEM); > > > + > > > + channel->chan_id = chan_id; > > > + channel->func_id = func_id; > > > + channel->domain_id = DOMID_INVALID; > > > > I take you are using DOMID_INVALID to mark a channel used by Xen itself? > > If so, then DOMID_XEN would be more appropriate. > > > > I use DOMID_INVALID to mark channel as free. > > > > > > + mfn = maddr_to_mfn(addr); > > > + channel->shmem = vmap(&mfn, 1); > > > > One thing to be careful is the mapping attributes, for a couple of > > reasons. As you might be aware, the ARM architecture forbids mismatching > > attributes for mapping memory in different places in the system. So the > > attributes that we use here must be the same used by the firmware > > (and/or the guest.) > > > > The second reason to be careful is that in the bindings example > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml the shared > > memory is "mmio-sram", which is special. It is not supposed to be normal > > memory, but it is OK to map it cacheable. Still, it might be more > > appropriate to use ioremap_cache. > > > > Originally, I used vmap here to have memcpy and it works fine in our > setup. But I will do some research and email you with the results. > > > > > > + if ( !channel->shmem ) > > > + { > > > + xfree(channel); > > > + return ERR_PTR(ENOMEM); > > > + } > > > + > > > + printk(XENLOG_DEBUG "scmi: Got shmem after vmap %p\n", channel->shmem); > > > + channel->paddr = addr; > > > + channel->shmem->channel_status = SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE; > > > + spin_lock_init(&channel->lock); > > > + spin_lock(&scmi_data.channel_list_lock); > > > + list_add(&channel->list, &scmi_data.channel_list); > > > + spin_unlock(&scmi_data.channel_list_lock); > > > + return channel; > > > +} > > > + > > > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > > > +{ > > > + return iomem_permit_access(d, paddr_to_pfn(addr), > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > +} > > > + > > > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > > > + uint64_t len) > > > +{ > > > + return iomem_deny_access(d, paddr_to_pfn(addr), > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > +} > > > + > > > +static int dt_update_domain_range(struct domain *d, uint64_t addr, > > > + uint64_t size) > > > +{ > > > + struct dt_device_node *shmem_node; > > > + __be32 *hw_reg; > > > + const struct dt_property *pp; > > > + uint32_t len; > > > + > > > + shmem_node = dt_find_compatible_node(NULL, NULL, "arm,scmi-shmem"); > > > > Here we are using "arm,scmi-shmem" while below we are checking for > > "linux,scmi_mem". What's the difference? > > linux,scmi_mem (I posted nodes examples above) describes memory region, allocated > for all channels, while arm,scmi-shmem points to the exact channel (page in > linux,scmi_mem region). > > > > Also, this function is looking for "arm,scmi-shmem" in dt_host and > > replaces its value. For dom0less domUs we'll probably need a > > make_scmi_node function to create the node from scratch like for > > instance xen/arch/arm/domain_build.c:make_gic_domU_node. > > > > I wonder if we had such a function whether it wouldn't be better to also > > use it for dom0 (and blacklist the physical "arm,scmi-shmem" in > > handle_node so that dom0 doesn't get the real shared memory information > > by accident). > > > > Thank you for the remark. I will rework this in v2. > > > > > > + > > > + if ( !shmem_node ) > > > + { > > > + printk(XENLOG_ERR "scmi: Unable to find %s node in DT\n", SCMI_SHMEM); > > > + return -EINVAL; > > > + } > > > + > > > + pp = dt_find_property(shmem_node, "reg", &len); > > > + if ( !pp ) > > > + { > > > + printk(XENLOG_ERR "scmi: Unable to find regs entry in shmem node\n"); > > > + return -ENOENT; > > > + } > > > + > > > + hw_reg = pp->value; > > > + dt_set_range(&hw_reg, shmem_node, addr, size); > > > + > > > + return 0; > > > +} > > > + > > > +static void free_channel_list(void) > > > +{ > > > + struct scmi_channel *curr, *_curr; > > > + > > > + spin_lock(&scmi_data.channel_list_lock); > > > + list_for_each_entry_safe (curr, _curr, &scmi_data.channel_list, list) > > > + { > > > + vunmap(curr->shmem); > > > + list_del(&curr->list); > > > + xfree(curr); > > > + } > > > + > > > + spin_unlock(&scmi_data.channel_list_lock); > > > +} > > > + > > > +static __init bool scmi_probe(struct dt_device_node *scmi_node) > > > +{ > > > + struct dt_device_node *shmem_node; > > > + int ret, i; > > > + struct scmi_channel *channel, *agent_channel; > > > + int n_agents; > > > + scmi_msg_header_t hdr; > > > + struct rx_t { > > > + int32_t status; > > > + uint32_t attributes; > > > + } rx; > > > > Should rx be defined at the top together with scmi_perms_tx_t and > > others? > > > > I'd rather move scmi_perms_tx_t to scmi_add_device_by_devid because it's > used only in 1 place. > So we will have rx and tx in scmi_add_device_by_devid and rx ( which > differs from rx in scmi_add_device_by_devid ) in scmi_probe. > I think it will be more understandable and no need to make unique names. > What do you think about that? I think that's OK > > > + uint32_t func_id; > > > + > > > + ASSERT(scmi_node != NULL); > > > + > > > + INIT_LIST_HEAD(&scmi_data.channel_list); > > > + spin_lock_init(&scmi_data.channel_list_lock); > > > + > > > + if ( !dt_property_read_u32(scmi_node, SCMI_SMC_ID, &func_id) ) > > > + { > > > + printk(XENLOG_ERR "scmi: Unable to read smc-id from DT\n"); > > > + return false; > > > + } > > > + > > > + shmem_node = dt_find_node_by_name(NULL, SCMI_SHARED_MEMORY); > > > > From the spec, it looks like you should be getting the shared memory > > area from the phandle list "shmem". > > > > We use SCMI_SHARED_MEMORY to get whole memory region (0x10 pages in my case), > we can use for the agents. As you can see below - Hypervisor received number of > agents from Firmware and split this region between agents. In general we can't use properties that are not part of the device tree spec, either https://www.devicetree.org/specifications/ or https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings "linux,scmi_mem" is currently absent. Are you aware of any upstreaming activities to get "linux,scmi_mem" upstream under Documentation/devicetree/bindings in Linux? If "linux,scmi_mem" is going upstream in Linux, then we could use it. Otherwise, first "linux,scmi_mem" needs to be added somewhere under Documentation/devicetree/bindings (probably Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can work on the Xen code that makes use of it. Does it make sense? > > > + if ( IS_ERR_OR_NULL(shmem_node) ) > > > + { > > > + printk(XENLOG_ERR > > > + "scmi: Device tree error, can't parse shmem phandle %ld\n", > > > + PTR_ERR(shmem_node)); > > > + return false; > > > + } > > > + > > > + ret = dt_device_get_address(shmem_node, 0, &scmi_data.shmem_addr, > > > + &scmi_data.shmem_size); > > > + if ( IS_ERR_VALUE(ret) ) > > > + return false; > > > + > > > + channel = smc_create_channel(HYP_CHANNEL, func_id, scmi_data.shmem_addr); > > > + if ( IS_ERR(channel) ) > > > + return false; > > > + > > > + hdr.id = SCMI_BASE_PROTOCOL_ATTIBUTES; > > > + hdr.type = 0; > > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > > + > > > + ret = do_smc_xfer(channel, &hdr, NULL, 0, &rx, sizeof(rx)); > > > + if ( ret ) > > > + goto clean; > > > + > > > + ret = check_scmi_status(rx.status); > > > + if ( ret ) > > > + goto clean; > > > + > > > + n_agents = FIELD_GET(MSG_N_AGENTS_MASK, rx.attributes); > > > + printk(XENLOG_DEBUG "scmi: Got agent count %d\n", n_agents); > > > + > > > + n_agents = (n_agents > scmi_data.shmem_size / PAGE_SIZE) ? > > > + scmi_data.shmem_size / PAGE_SIZE : n_agents; > > > + > > > + for ( i = 1; i < n_agents; i++ ) > > > + { > > > > Given that HYP_CHANNEL is actually zero, it looks like we could do > > everything here in this loop but starting from i=0? > > > > We allocate HYP_CHANNEL before loop because we need it to request number > of agents. And we don't need to send SCMI_BASE_DISCOVER_AGENT to > HYP_CHANNEL. OK > > > + uint32_t tx_agent_id = 0xFFFFFFFF; > > > + struct { > > > + int32_t status; > > > + uint32_t agent_id; > > > + char name[16]; > > > + } da_rx; > > > + > > > + agent_channel = smc_create_channel(i, func_id, scmi_data.shmem_addr + > > > + i * PAGE_SIZE); > > > + if ( IS_ERR(agent_channel) ) > > > + { > > > + ret = PTR_ERR(agent_channel); > > > + goto clean; > > > + } > > > + > > > + hdr.id = SCMI_BASE_DISCOVER_AGENT; > > > + hdr.type = 0; > > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > > + > > > + ret = do_smc_xfer(agent_channel, &hdr, &tx_agent_id, > > > + sizeof(tx_agent_id), &da_rx, sizeof(da_rx)); > > > + if ( ret ) > > > + goto clean; > > > + > > > + ret = check_scmi_status(da_rx.status); > > > + if ( ret ) > > > + goto clean; > > > + > > > + printk(XENLOG_DEBUG "scmi: status=0x%x id=0x%x name=%s\n", > > > + da_rx.status, da_rx.agent_id, da_rx.name); > > > + > > > + agent_channel->agent_id = da_rx.agent_id; > > > + } > > > + > > > + scmi_data.initialized = true; > > > + return true; > > > + > > > +clean: > > > + free_channel_list(); > > > + return ret == 0; > > > +} > > > + > > > +static int scmi_domain_init(struct domain *d) > > > +{ > > > + struct scmi_channel *channel; > > > + int ret; > > > + > > > + if ( !scmi_data.initialized ) > > > + return 0; > > > + > > > + channel = aquire_scmi_channel(d->domain_id); > > > + if ( IS_ERR_OR_NULL(channel) ) > > > + return -ENOENT; > > > + > > > + printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = %d" > > > + "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, > > > + channel->paddr); > > > + > > > + if ( is_hardware_domain(d) ) > > > + { > > > + ret = map_memory_to_domain(d, scmi_data.shmem_addr, > > > + scmi_data.shmem_size); > > > + if ( IS_ERR_VALUE(ret) ) > > > + goto error; > > > + > > > + ret = dt_update_domain_range(d, channel->paddr, PAGE_SIZE); > > > + if ( IS_ERR_VALUE(ret) ) > > > + { > > > + int rc = unmap_memory_from_domain(d, scmi_data.shmem_addr, > > > + scmi_data.shmem_size); > > > + if ( rc ) > > > + printk(XENLOG_ERR "Unable to unmap_memory_from_domain\n"); > > > + > > > + goto error; > > > + } > > > + } > > > > Is dom0 the only domain to get direct access to the shared memory > > region? If so, I don't think it is a good idea to make Dom0 "special" in > > this case. > > > > Let me make an example: if we assign a device to a domU since boot, and > > dom0 wants to change the frequency of a clock that affects the assigned > > device (likely because it doesn't know it is assigned), then dom0 > > shouldn't be able to. We might have to perform checks in Xen to make > > sure dom0 cannot stop the clock for the assigned device. > > > > So I think it would be better if all domains are treated the same way in > > the mediator unless really necessary. > > > > On the other hand, if all domains get access to the shared memory > > region, then I don't think this is likely the right place to create the > > dom0 mapping. We probably want to do it in domain_build.c in a way that > > can be reused for dom0less domUs. > > > > The idea is that all domains have their own page in shared memory > region and unigue agent_id. Agent_id is used to set permissions for > clocks\resets\power-domains etc. So during creation of domain (domUs or > dom0) device-tree is processed using scmi_add_dt_device and > clocks\resets\power-domains which are related to dom0 devices will be > requested by using SCMI_BASE_SET_DEVICE_PERMISSIONS message. > All passed-through devices will be requested during DomU creation. Which > means dom0 do not have an access to clocks\resets\power-domains, which > are related to DomU. OK, excellent > > In regards to shared memory: it looks like the only two functions to > > access the real shared memory are send_smc_message and get_smc_response. > > If that is the case, then we actually don't need to expose the real > > shared memory to any of the domains. > > > > We could simply: > > > > - expose a regular normal memory region as dom0/domU channel memory > > - on SMC trap, read from the "fake" shared memory and set the > > corresponding real shared memory on the appropriate channel > > - issue the SMC call > > - on return from SMC, copy over data from the real shared memory to the > > "fake" channel reagion > > Hypervisor redirects only SMC calls from guests and set agent_id to SMC > parameters as a7. The idea was to give page for each agent, so we don't > need to make additional read/write each time we receive SMC call. > All we povide from hypervisor is agent_id. Firmware is responsible for > reading memory from the correct address and place the response. > > > > > This is useful if we need to "filter" any of the SCMI commands and > > options from the domains to the firmware, and also it is useful if the > > channel memory is not page aligned. But if the permissions are > > fine-grained enough and also the channel memory is page aligned (and > > multiple of 4K in size) then we could map the memory. > > > > In current implementation we suppose that channel memory is page aligned. > I think that Firmware should be responsible for permissions handling and > "filtering", that's why permission calls were added to SCMI spec. > I tried to make mediator as thin as possible. OK. Can we check that the channel memory is page aligned at init time and throw an error if it is not the case? > > > + > > > + d->arch.sci = channel; > > > + > > > + return 0; > > > +error: > > > + relinquish_scmi_channel(channel); > > > + > > > + return ret; > > > +} > > > + > > > +static int scmi_add_device_by_devid(struct domain *d, uint32_t scmi_devid) > > > +{ > > > + struct scmi_channel *channel, *agent_channel; > > > + scmi_msg_header_t hdr; > > > + scmi_perms_tx_t tx; > > > + struct rx_t { > > > + int32_t status; > > > + uint32_t attributes; > > > + } rx; > > > + int ret; > > > + > > > + if ( !scmi_data.initialized ) > > > + return 0; > > > + > > > + printk(XENLOG_DEBUG "scmi: scmi_devid = %d\n", scmi_devid); > > > + > > > + agent_channel = get_channel_by_domain(d->domain_id); > > > + if ( IS_ERR_OR_NULL(agent_channel) ) > > > + return PTR_ERR(agent_channel); > > > + > > > + channel = get_channel_by_id(HYP_CHANNEL); > > > + if ( IS_ERR_OR_NULL(channel) ) > > > + return PTR_ERR(channel); > > > + > > > + hdr.id = SCMI_BASE_SET_DEVICE_PERMISSIONS; > > > + hdr.type = 0; > > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > > + > > > + tx.agent_id = agent_channel->agent_id; > > > + tx.device_id = scmi_devid; > > > + tx.flags = SCMI_ALLOW_ACCESS; > > > + > > > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(&rx)); > > > + if ( IS_ERR_VALUE(ret) ) > > > + return ret; > > > + > > > + ret = check_scmi_status(rx.status); > > > + if ( IS_ERR_VALUE(ret) ) > > > + return ret; > > > + > > > + return 0; > > > +} > > > + > > > +static int scmi_add_dt_device(struct domain *d, struct dt_device_node *dev) > > > +{ > > > + uint32_t scmi_devid; > > > + > > > + if ( (!scmi_data.initialized) || (!d->arch.sci) ) > > > + return 0; > > > + > > > + if ( !dt_property_read_u32(dev, "scmi_devid", &scmi_devid) ) > > > + return 0; > > > > scmi_devid is another property that is not documented in the binding. > > > > This property should be added to the device nodes, which are using scmi > to work with clocks\resets\power-domains etc. This id should match the > device_id, defined in Firmware. Hypervisor send this device_id to the Firmware > as parameter to the permission request. Firmware set permissions to > clocks\resets\power-domains, related to this device. OK, I see. Unfortunately, scmi_devid is also not described under Documentation/devicetree/bindings/. This property seems to be actually required for the system to work correctly. Unless somebody else is already working on this, please send a patch to the Linux kernel mailing list CCing the SCMI maintainers and Rob Herring to introduce scmi_devid as a new property. > > > + printk(XENLOG_INFO "scmi: dt_node = %s\n", dt_node_full_name(dev)); > > > + > > > + return scmi_add_device_by_devid(d, scmi_devid); > > > +} > > > + > > > +static int scmi_relinquish_resources(struct domain *d) > > > +{ > > > + int ret; > > > + struct scmi_channel *channel, *agent_channel; > > > + scmi_msg_header_t hdr; > > > + struct reset_agent_tx { > > > + uint32_t agent_id; > > > + uint32_t flags; > > > + } tx; > > > + uint32_t rx; > > > + > > > + if ( !d->arch.sci ) > > > + return 0; > > > + > > > + agent_channel = d->arch.sci; > > > + > > > + spin_lock(&agent_channel->lock); > > > + tx.agent_id = agent_channel->agent_id; > > > + spin_unlock(&agent_channel->lock); > > > + > > > + channel = get_channel_by_id(HYP_CHANNEL); > > > + if ( !channel ) > > > + { > > > + printk(XENLOG_ERR > > > + "scmi: Unable to get Hypervisor scmi channel for domain %d\n", > > > + d->domain_id); > > > + return -EINVAL; > > > + } > > > + > > > + hdr.id = SCMI_BASE_RESET_AGENT_CONFIGURATION; > > > + hdr.type = 0; > > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > > + > > > + tx.flags = 0; > > > + > > > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(rx)); > > > + if ( ret ) > > > + return ret; > > > + > > > + ret = check_scmi_status(rx); > > > + > > > + return ret; > > > +} > > > + > > > +static void scmi_domain_destroy(struct domain *d) > > > +{ > > > + struct scmi_channel *channel; > > > + > > > + if ( !d->arch.sci ) > > > + return; > > > + > > > + channel = d->arch.sci; > > > + spin_lock(&channel->lock); > > > + > > > + relinquish_scmi_channel(channel); > > > + printk(XENLOG_DEBUG "scmi: Free domain %d\n", d->domain_id); > > > + > > > + d->arch.sci = NULL; > > > + > > > + unmap_memory_from_domain(d, channel->paddr, PAGE_SIZE); > > > + spin_unlock(&channel->lock); > > > + return; > > > +} > > > + > > > +static bool scmi_handle_call(struct domain *d, void *args) > > > +{ > > > + bool res = false; > > > + struct scmi_channel *agent_channel; > > > + struct arm_smccc_res resp; > > > + struct cpu_user_regs *regs = args; > > > + > > > + if ( !d->arch.sci ) > > > + return false; > > > + > > > + agent_channel = d->arch.sci; > > > + spin_lock(&agent_channel->lock); > > > + > > > + if ( agent_channel->func_id != regs->x0 ) > > > + { > > > + printk(XENLOG_ERR "scmi: func_id mismatch, exiting\n"); > > > + goto unlock; > > > + } > > > + > > > + arm_smccc_smc(agent_channel->func_id, 0, 0, 0, 0, 0, 0, > > > + agent_channel->chan_id, &resp); > > > + > > > + set_user_reg(regs, 0, resp.a0); > > > + set_user_reg(regs, 1, resp.a1); > > > + set_user_reg(regs, 2, resp.a2); > > > + set_user_reg(regs, 3, resp.a3); > > > + res = true; > > > +unlock: > > > + spin_unlock(&agent_channel->lock); > > > + > > > + return res; > > > +} > > > + > > > +static int scmi_get_channel_paddr(void *scmi_ops, > > > + struct xen_arch_domainconfig *config) > > > +{ > > > + struct scmi_channel *agent_channel = scmi_ops; > > > + > > > + if ( !agent_channel ) > > > + return -EINVAL; > > > + > > > + config->sci_agent_paddr = agent_channel->paddr; > > > + return 0; > > > +} > > > > I am still not sure why it couldn't be done by scmi_domain_init. > > > > I can move this logic to scmi_domain_init, but in this case I have to add > struct xen_arch_domainconfig *config as input parameter to > scmi_domain_init and pass NULL from construct_dom0. > Do you think this approach would be better? I think it is OK to pass struct xen_arch_domainconfig *config as input parameter to scmi_domain_init. For dom0, why is sci_agent_paddr not supposed to be set? > Also I think it's reasonable to pass xen_arch_domainconfig so different > implementations could set another data they would probably need. > > > > > > +static const struct dt_device_match scmi_smc_match[] __initconst = > > > +{ > > > + DT_MATCH_SCMI_SMC, > > > + { /* sentinel */ }, > > > +}; > > > + > > > +static const struct sci_mediator_ops scmi_ops = > > > +{ > > > + .probe = scmi_probe, > > > + .domain_init = scmi_domain_init, > > > + .domain_destroy = scmi_domain_destroy, > > > + .add_dt_device = scmi_add_dt_device, > > > + .relinquish_resources = scmi_relinquish_resources, > > > + .handle_call = scmi_handle_call, > > > + .get_channel_info = scmi_get_channel_paddr > > > +}; > > > + > > > +REGISTER_SCI_MEDIATOR(scmi_smc, "SCMI-SMC", XEN_DOMCTL_CONFIG_SCI_SCMI_SMC, > > > + scmi_smc_match, &scmi_ops); > > > + > > > +/* > > > + * Local variables: > > > + * mode: C > > > + * c-file-style: "BSD" > > > + * c-basic-offset: 4 > > > + * indent-tabs-mode: nil > > > + * End: > > > + */ > > > diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h > > > index 9180be5e86..a67237942d 100644 > > > --- a/xen/include/public/arch-arm.h > > > +++ b/xen/include/public/arch-arm.h > > > @@ -315,6 +315,7 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t); > > > #define XEN_DOMCTL_CONFIG_TEE_OPTEE 1 > > > > > > #define XEN_DOMCTL_CONFIG_SCI_NONE 0 > > > +#define XEN_DOMCTL_CONFIG_SCI_SCMI_SMC 1 > > > > > > struct xen_arch_domainconfig { > > > /* IN/OUT */
Hi Stefano, On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > Hi Stefano, > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > This allows devices from the Domains to work with clocks, resets and > > > > power-domains without access to CPG. > > > > > > > > The following features are implemented: > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > - set device permissions for Domains based on the Domain partial > > > > device-tree. Devices with permissions are able to work with clocks, > > > > resets and power-domains via SCMI; > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > --- > > > > xen/arch/arm/Kconfig | 2 + > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > xen/arch/arm/sci/Makefile | 1 + > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > xen/include/public/arch-arm.h | 1 + > > > > 5 files changed, 809 insertions(+) > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > index 186e1db389..02d96c6cfc 100644 > > > > --- a/xen/arch/arm/Kconfig > > > > +++ b/xen/arch/arm/Kconfig > > > > @@ -114,6 +114,8 @@ config SCI > > > > support. It allows guests to control system resourcess via one of > > > > SCI mediators implemented in XEN. > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > + > > > > endmenu > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > new file mode 100644 > > > > index 0000000000..9563067ddc > > > > --- /dev/null > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > @@ -0,0 +1,10 @@ > > > > +config SCMI_SMC > > > > + bool "Enable SCMI-SMC mediator driver" > > > > + default n > > > > + depends on SCI > > > > + ---help--- > > > > + > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > + This feature allows drivers from Domains to work with System > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > + for communication. > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > index 837dc7492b..67f2611872 100644 > > > > --- a/xen/arch/arm/sci/Makefile > > > > +++ b/xen/arch/arm/sci/Makefile > > > > @@ -1 +1,2 @@ > > > > obj-y += sci.o > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > new file mode 100644 > > > > index 0000000000..2eb01ea82d > > > > --- /dev/null > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > @@ -0,0 +1,795 @@ > > > > +/* > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > + * > > > > + * SCMI mediator driver, using SCP as transport. > > > > + * > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > + * Copyright (C) 2021, EPAM Systems. > > > > + * > > > > + * This program is free software; you can redistribute it and/or modify > > > > + * it under the terms of the GNU General Public License as published by > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > + * (at your option) any later version. > > > > + * > > > > + * This program is distributed in the hope that it will be useful, > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > + * GNU General Public License for more details. > > > > + */ > > > > + > > > > +#include <asm/sci/sci.h> > > > > +#include <asm/smccc.h> > > > > +#include <asm/io.h> > > > > +#include <xen/bitops.h> > > > > +#include <xen/config.h> > > > > +#include <xen/sched.h> > > > > +#include <xen/device_tree.h> > > > > +#include <xen/iocap.h> > > > > +#include <xen/init.h> > > > > +#include <xen/err.h> > > > > +#include <xen/lib.h> > > > > +#include <xen/list.h> > > > > +#include <xen/mm.h> > > > > +#include <xen/string.h> > > > > +#include <xen/time.h> > > > > +#include <xen/vmap.h> > > > > + > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > + > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > +#define SCMI_SUCCESS 0 > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > +#define SCMI_DENIED (-3) > > > > +#define SCMI_NOT_FOUND (-4) > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > +#define SCMI_BUSY (-6) > > > > +#define SCMI_COMMS_ERROR (-7) > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > + > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > + > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > I could find the following SCMI binding in Linux, which describes > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > the "shmem" property instead? And the compatible string used for this > > > seems to be "arm,scmi-shmem". > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > channels: > > > > reserved-memory { > > /* reserved region for scmi channels*/ > > scmi_memory: linux,scmi_mem@53FF0000 { > > no-map; > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > }; > > }; > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > the current scmi channel: > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > compatible = "arm,scmi-shmem"; > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > }; > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > assigned to this agent. > > If we were to use "linux,scmi_mem" we would have to introduce it as a > compatible string, not as a node name, and it would need to be described > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > But from your description I don't think it is necessary. We can just use > "arm,scmi-shmem" to describe all the required regions: > > reserved-memory { > scp-shmem@0x53FF0000 { > compatible = "arm,scmi-shmem"; > reg = <0x0 0x53FF0000 0x0 0x1000>; > }; > scp-shmem@0x53FF1000 { > compatible = "arm,scmi-shmem"; > reg = <0x0 0x53FF1000 0x0 0x1000>; > }; > scp-shmem@0x53FF2000 { > compatible = "arm,scmi-shmem"; > reg = <0x0 0x53FF2000 0x0 0x1000>; > }; > ... > > In other words, if all the individual channel pages are described as > "arm,scmi-shmem", why do we also need a single larger region as > "linux,scmi_mem"? > That was my first implementation. But I've met a problem with scmi driver in kernel. I don't remember the exact place, but I remember there were some if, checking if memory weren't reserved. That's why I ended up splitting nodes reserved memory region and actual shmem page. For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, which has no compatible string and provides no-map property. linux,scmi_shmem node is needed to prevent xen from allocating this space for the domain. Very interesting question about should I introduce linux,scmi_mem node and scmi_devid property to the Documentation/devicetree/bindings/firmware/arm,scmi.yaml? Those node and property are needed only for Xen and useless for non-virtualized systems. I can add this node and property description to arm,scmi.yaml, but leave a note that this is Xen specific params. What do you think about it? > > > > > +#define SCMI_SHMEM "shmem" > > > > + > > > > +#define HYP_CHANNEL 0x0 > > > > + > > > > +#define HDR_ID GENMASK(7,0) > > > > +#define HDR_TYPE GENMASK(9, 8) > > > > +#define HDR_PROTO GENMASK(17, 10) > > > > + > > > > +/* SCMI protocol, refer to section 4.2.2.2 (DEN0056C) */ > > > > +#define MSG_N_AGENTS_MASK GENMASK(15, 8) > > > > + > > > > +#define FIELD_GET(_mask, _reg)\ > > > > + ((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1))) > > > > +#define FIELD_PREP(_mask, _val)\ > > > > + (((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask)) > > > > + > > > > +typedef struct scmi_msg_header { > > > > + uint8_t id; > > > > + uint8_t type; > > > > + uint8_t protocol; > > > > +} scmi_msg_header_t; > > > > + > > > > +typedef struct scmi_perms_tx { > > > > + uint32_t agent_id; > > > > + uint32_t device_id; > > > > + uint32_t flags; > > > > +} scmi_perms_tx_t; > > > > + > > > > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE BIT(0, UL) > > > > +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_ERROR BIT(1, UL) > > > > + > > > > +#define SCMI_ALLOW_ACCESS BIT(0, UL) > > > > + > > > > +struct scmi_shared_mem { > > > > + uint32_t reserved; > > > > + uint32_t channel_status; > > > > + uint32_t reserved1[2]; > > > > + uint32_t flags; > > > > + uint32_t length; > > > > + uint32_t msg_header; > > > > + uint8_t msg_payload[]; > > > > +}; > > > > + > > > > +struct scmi_channel { > > > > + int chan_id; > > > > + int agent_id; > > > > + uint32_t func_id; > > > > + int domain_id; > > > > + uint64_t paddr; > > > > + struct scmi_shared_mem *shmem; > > > > + spinlock_t lock; > > > > + struct list_head list; > > > > +}; > > > > + > > > > +struct scmi_data { > > > > + struct list_head channel_list; > > > > + spinlock_t channel_list_lock; > > > > + bool initialized; > > > > + u64 shmem_addr, shmem_size; > > > > +}; > > > > + > > > > +static struct scmi_data scmi_data; > > > > + > > > > +/* > > > > + * pack_scmi_header() - packs and returns 32-bit header > > > > + * > > > > + * @hdr: pointer to header containing all the information on message id, > > > > + * protocol id and type id. > > > > + * > > > > + * Return: 32-bit packed message header to be sent to the platform. > > > > + */ > > > > +static inline uint32_t pack_scmi_header(scmi_msg_header_t *hdr) > > > > +{ > > > > + return FIELD_PREP(HDR_ID, hdr->id) | > > > > + FIELD_PREP(HDR_TYPE, hdr->type) | > > > > + FIELD_PREP(HDR_PROTO, hdr->protocol); > > > > +} > > > > + > > > > +/* > > > > + * unpack_scmi_header() - unpacks and records message and protocol id > > > > + * > > > > + * @msg_hdr: 32-bit packed message header sent from the platform > > > > + * @hdr: pointer to header to fetch message and protocol id. > > > > + */ > > > > +static inline void unpack_scmi_header(uint32_t msg_hdr, scmi_msg_header_t *hdr) > > > > +{ > > > > + hdr->id = FIELD_GET(HDR_ID, msg_hdr); > > > > + hdr->type = FIELD_GET(HDR_TYPE, msg_hdr); > > > > + hdr->protocol = FIELD_GET(HDR_PROTO, msg_hdr); > > > > +} > > > > + > > > > +static inline int channel_is_free(struct scmi_channel *chan_info) > > > > +{ > > > > + return ( chan_info->shmem->channel_status > > > > + & SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE ) ? 0 : -EBUSY; > > > > > > Does this need a memory barrier? Or not, because the other end always > > > runs on the same CPU at a different execution level so the > > > channel_status would be always guaranteed to be read as updated? > > > > > > > It don't because the other end runs on the same CPU. Other mediator > > implemetaions, which uses different areas may need memory barrier. > > > > > > > > > +} > > > > + > > > > +static int send_smc_message(struct scmi_channel *chan_info, > > > > + scmi_msg_header_t *hdr, void *data, int len) > > > > +{ > > > > + struct arm_smccc_res resp; > > > > + int ret; > > > > + > > > > + printk(XENLOG_DEBUG "scmi: status =%d len=%d\n", > > > > + chan_info->shmem->channel_status, len); > > > > + printk(XENLOG_DEBUG "scmi: header id = %d type = %d, proto = %d\n", > > > > + hdr->id, hdr->type, hdr->protocol); > > > > + > > > > + ret = channel_is_free(chan_info); > > > > + if ( IS_ERR_VALUE(ret) ) > > > > + return ret; > > > > + > > > > + chan_info->shmem->channel_status = 0x0; > > > > + /* Writing 0x0 right now, but SCMI_SHMEM_FLAG_INTR_ENABLED can be set */ > > > > + chan_info->shmem->flags = 0x0; > > > > + chan_info->shmem->length = sizeof(chan_info->shmem->msg_header) + len; > > > > + chan_info->shmem->msg_header = pack_scmi_header(hdr); > > > > + > > > > + printk(XENLOG_DEBUG "scmi: Writing to shmem address %p\n", > > > > + chan_info->shmem); > > > > + if ( len > 0 && data ) > > > > + memcpy((void *)(chan_info->shmem->msg_payload), data, len); > > > > > > Again, here we don't need a barrier because it is implicit in the SMC? > > > > > > > As I mentioned before, the other end runs on the same CPU. > > > > > Don't we need to check that "len" fits in the shared memory? > > > > > > > I think it's a good point. I'll add len check in v2. > > > > > > > > > + arm_smccc_smc(chan_info->func_id, 0, 0, 0, 0, 0, 0, chan_info->chan_id, > > > > + &resp); > > > > + > > > > + printk(XENLOG_DEBUG "scmi: scmccc_smc response %d\n", (int)(resp.a0)); > > > > + > > > > + if ( resp.a0 ) > > > > + return -EOPNOTSUPP; > > > > > > Why is that? > > > > > > > This change was presented in kernel by Sudeep Holla in commit: > > f7199cf489027ae38a9a82312d13025f7aefa0b8 > > > > However, link posted in the commit: > > https://urldefense.com/v3/__https://lore.kernel.org/r/20200417103232.6896-1-sudeep.holla@arm.com__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7Bnhb7jXoB3$ [lore[.]kernel[.]org] > > > > Leads to slightly different patch: > > > > + if (res.a0 == SMCCC_RET_NOT_SUPPORTED) > > + return -EOPNOTSUPP; > > + else if (res.a0) > > + return -EINVAL; > > + return 0; > > > > I don't know why it differs from the original commit, but I'll check and place > > the correct implementation in v2. > > > > > > > > > + return 0; > > > > +} > > > > + > > > > +static int check_scmi_status(int scmi_status) > > > > +{ > > > > + if ( scmi_status == SCMI_SUCCESS ) > > > > + return 0; > > > > + > > > > + printk(XENLOG_DEBUG "scmi: Error received: %d\n", scmi_status); > > > > + > > > > + switch ( scmi_status ) > > > > + { > > > > + case SCMI_NOT_SUPPORTED: > > > > + return -EOPNOTSUPP; > > > > + case SCMI_INVALID_PARAMETERS: > > > > + return -EINVAL; > > > > + case SCMI_DENIED: > > > > + return -EACCES; > > > > + case SCMI_NOT_FOUND: > > > > + return -ENOENT; > > > > + case SCMI_OUT_OF_RANGE: > > > > + return -ERANGE; > > > > + case SCMI_BUSY: > > > > + return -EBUSY; > > > > + case SCMI_COMMS_ERROR: > > > > + return -ENOTCONN; > > > > + case SCMI_GENERIC_ERROR: > > > > + return -EIO; > > > > + case SCMI_HARDWARE_ERROR: > > > > + return -ENXIO; > > > > + case SCMI_PROTOCOL_ERROR: > > > > + return -EBADMSG; > > > > + } > > > > + > > > > + return -EINVAL; > > > > +} > > > > + > > > > +static int get_smc_response(struct scmi_channel *chan_info, > > > > + scmi_msg_header_t *hdr, void *data, int len) > > > > +{ > > > > + int recv_len; > > > > + int ret; > > > > + > > > > + printk(XENLOG_DEBUG "scmi: get smc responce msgid %d\n", hdr->id); > > > > + > > > > + ret = channel_is_free(chan_info); > > > > + if ( IS_ERR_VALUE(ret) ) > > > > + return ret; > > > > > > I am not familiar with the spec (do you have a link?) but is it expected > > > that the channel is "free" when actually we want to read a message on > > > the channel? > > > > > > > Here is the link https://urldefense.com/v3/__https://developer.arm.com/documentation/den0056/latest__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhRd1Yyp8$ [developer[.]arm[.]com] > > Figure 6 in Section 5.1.1. > > Caller marks channel as busy, then callee process message and marks channel as free. > > We are implementing polling based communication flow. > > OK > > > > > > + recv_len = chan_info->shmem->length - sizeof(chan_info->shmem->msg_header); > > > > + > > > > + if ( recv_len < 0 ) > > > > + { > > > > + printk(XENLOG_ERR > > > > + "scmi: Wrong size of smc message. Data may be invalid\n"); > > > > + return -EINVAL; > > > > + } > > > > + > > > > + if ( recv_len > len ) > > > > + { > > > > + printk(XENLOG_ERR > > > > + "scmi: Not enough buffer for message %d, expecting %d\n", > > > > + recv_len, len); > > > > + return -EINVAL; > > > > + } > > > > + > > > > + unpack_scmi_header(chan_info->shmem->msg_header, hdr); > > > > + > > > > + if ( recv_len > 0 ) > > > > + { > > > > + memcpy(data, chan_info->shmem->msg_payload, recv_len); > > > > + } > > > > + > > > > + return 0; > > > > +} > > > > + > > > > +static int do_smc_xfer(struct scmi_channel *channel, scmi_msg_header_t *hdr, void *tx_data, int tx_size, > > > > + void *rx_data, int rx_size) > > > > +{ > > > > + int ret = 0; > > > > + > > > > + if ( !hdr ) > > > > + return -EINVAL; > > > > + > > > > + spin_lock(&channel->lock); > > > > + > > > > + ret = send_smc_message(channel, hdr, tx_data, tx_size); > > > > + if ( ret ) > > > > + goto clean; > > > > + > > > > + ret = get_smc_response(channel, hdr, rx_data, rx_size); > > > > +clean: > > > > + spin_unlock(&channel->lock); > > > > + > > > > + return ret; > > > > +} > > > > + > > > > +static struct scmi_channel *get_channel_by_id(uint8_t chan_id) > > > > +{ > > > > + struct scmi_channel *curr; > > > > + bool found = false; > > > > + > > > > + spin_lock(&scmi_data.channel_list_lock); > > > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > > > > > please use parenthesis around the inner if (also in other places) > > > > > > > Thank you for the remark. I will fix it in v2. > > > > > > > > > + if ( curr->chan_id == chan_id ) > > > > + { > > > > + found = true; > > > > + break; > > > > + } > > > > + > > > > + spin_unlock(&scmi_data.channel_list_lock); > > > > + if ( found ) > > > > + return curr; > > > > + > > > > + return NULL; > > > > +} > > > > + > > > > +static struct scmi_channel *get_channel_by_domain(uint8_t domain_id) > > > > > > Use domid_t for domain ids. > > > > Thanks, I will fix it in v2. > > > > > > Also, wouldn't it be better to implement it as: > > > > > > static inline struct scmi_channel *get_channel_by_domain(struct domain *d) { > > > return d->arch.sci > > > } > > > > > That's a good point. I will take a look on it and fix in v2. > > > > > > > +{ > > > > + struct scmi_channel *curr; > > > > + bool found = false; > > > > + > > > > + spin_lock(&scmi_data.channel_list_lock); > > > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > > > + if ( curr->domain_id == domain_id ) > > > > + { > > > > + found = true; > > > > + break; > > > > + } > > > > + > > > > + spin_unlock(&scmi_data.channel_list_lock); > > > > + if ( found ) > > > > + return curr; > > > > + > > > > + return NULL; > > > > +} > > > > + > > > > +static struct scmi_channel *aquire_scmi_channel(int domain_id) > > > > +{ > > > > + struct scmi_channel *curr; > > > > + bool found = false; > > > > + > > > > + ASSERT(domain_id != DOMID_INVALID && domain_id >= 0); > > > > + > > > > + spin_lock(&scmi_data.channel_list_lock); > > > > + list_for_each_entry(curr, &scmi_data.channel_list, list) > > > > + if ( (curr->domain_id == DOMID_INVALID) > > > > + && (curr->chan_id != HYP_CHANNEL) ) > > > > > > If you use DOMID_XEN for HYP_CHANNEL, then this check becomes more > > > intuitive > > > > > > > We do not have direct relation between channel id and domain id. > > One channel id can be reused by different domain_ids. So from my standpoint, > > DOMID_XEN doesn't fit here. > > Below you wrote that you used DOMID_INVALID to mark a channel as free. > That is fine, but then DOMID_INVALID shouldn't be used for HYP_CHANNEL > because HYP_CHANNEL is not "free". In this function the check for > curr->chan_id != HYP_CHANNEL should be unnecessary and I think the code > would look as follows: > > list_for_each_entry(curr, &scmi_data.channel_list, list) > { > if ( (curr->domain_id == DOMID_INVALID) ) > { > Oh, now I understood your point. Thank you for that. I will use DOMID_XEN for HYP_CHANNEL and remove curr->chan_id != HYP_CHANNEL. This will be fixed in v2. > > > > > + { > > > > + curr->domain_id = domain_id; > > > > + found = true; > > > > + break; > > > > + } > > > > + > > > > + spin_unlock(&scmi_data.channel_list_lock); > > > > + if ( found ) > > > > + return curr; > > > > + > > > > + return NULL; > > > > +} > > > > + > > > > +static void relinquish_scmi_channel(struct scmi_channel *channel) > > > > +{ > > > > + spin_lock(&scmi_data.channel_list_lock); > > > > + ASSERT(channel != NULL); > > > > > > the ASSERT could be before the spin_lock > > > > > > > Thank you. I will fix it in v2. > > > > > > > > > + channel->domain_id = DOMID_INVALID; > > > > + spin_unlock(&scmi_data.channel_list_lock); > > > > +} > > > > + > > > > +static struct scmi_channel *smc_create_channel(uint8_t chan_id, > > > > + uint32_t func_id, uint64_t addr) > > > > +{ > > > > + struct scmi_channel *channel; > > > > + mfn_t mfn; > > > > + > > > > + channel = get_channel_by_id(chan_id); > > > > + if ( channel ) > > > > + return ERR_PTR(EEXIST); > > > > + > > > > + channel = xmalloc(struct scmi_channel); > > > > + if ( !channel ) > > > > + return ERR_PTR(ENOMEM); > > > > + > > > > + channel->chan_id = chan_id; > > > > + channel->func_id = func_id; > > > > + channel->domain_id = DOMID_INVALID; > > > > > > I take you are using DOMID_INVALID to mark a channel used by Xen itself? > > > If so, then DOMID_XEN would be more appropriate. > > > > > > > I use DOMID_INVALID to mark channel as free. > > > > > > > > > + mfn = maddr_to_mfn(addr); > > > > + channel->shmem = vmap(&mfn, 1); > > > > > > One thing to be careful is the mapping attributes, for a couple of > > > reasons. As you might be aware, the ARM architecture forbids mismatching > > > attributes for mapping memory in different places in the system. So the > > > attributes that we use here must be the same used by the firmware > > > (and/or the guest.) > > > > > > The second reason to be careful is that in the bindings example > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml the shared > > > memory is "mmio-sram", which is special. It is not supposed to be normal > > > memory, but it is OK to map it cacheable. Still, it might be more > > > appropriate to use ioremap_cache. > > > > > > > Originally, I used vmap here to have memcpy and it works fine in our > > setup. But I will do some research and email you with the results. > > > > > > > > > + if ( !channel->shmem ) > > > > + { > > > > + xfree(channel); > > > > + return ERR_PTR(ENOMEM); > > > > + } > > > > + > > > > + printk(XENLOG_DEBUG "scmi: Got shmem after vmap %p\n", channel->shmem); > > > > + channel->paddr = addr; > > > > + channel->shmem->channel_status = SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE; > > > > + spin_lock_init(&channel->lock); > > > > + spin_lock(&scmi_data.channel_list_lock); > > > > + list_add(&channel->list, &scmi_data.channel_list); > > > > + spin_unlock(&scmi_data.channel_list_lock); > > > > + return channel; > > > > +} > > > > + > > > > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > > > > +{ > > > > + return iomem_permit_access(d, paddr_to_pfn(addr), > > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > > +} > > > > + > > > > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > > > > + uint64_t len) > > > > +{ > > > > + return iomem_deny_access(d, paddr_to_pfn(addr), > > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > > +} > > > > + > > > > +static int dt_update_domain_range(struct domain *d, uint64_t addr, > > > > + uint64_t size) > > > > +{ > > > > + struct dt_device_node *shmem_node; > > > > + __be32 *hw_reg; > > > > + const struct dt_property *pp; > > > > + uint32_t len; > > > > + > > > > + shmem_node = dt_find_compatible_node(NULL, NULL, "arm,scmi-shmem"); > > > > > > Here we are using "arm,scmi-shmem" while below we are checking for > > > "linux,scmi_mem". What's the difference? > > > > linux,scmi_mem (I posted nodes examples above) describes memory region, allocated > > for all channels, while arm,scmi-shmem points to the exact channel (page in > > linux,scmi_mem region). > > > > > > > Also, this function is looking for "arm,scmi-shmem" in dt_host and > > > replaces its value. For dom0less domUs we'll probably need a > > > make_scmi_node function to create the node from scratch like for > > > instance xen/arch/arm/domain_build.c:make_gic_domU_node. > > > > > > I wonder if we had such a function whether it wouldn't be better to also > > > use it for dom0 (and blacklist the physical "arm,scmi-shmem" in > > > handle_node so that dom0 doesn't get the real shared memory information > > > by accident). > > > > > > > Thank you for the remark. I will rework this in v2. > > > > > > > > > + > > > > + if ( !shmem_node ) > > > > + { > > > > + printk(XENLOG_ERR "scmi: Unable to find %s node in DT\n", SCMI_SHMEM); > > > > + return -EINVAL; > > > > + } > > > > + > > > > + pp = dt_find_property(shmem_node, "reg", &len); > > > > + if ( !pp ) > > > > + { > > > > + printk(XENLOG_ERR "scmi: Unable to find regs entry in shmem node\n"); > > > > + return -ENOENT; > > > > + } > > > > + > > > > + hw_reg = pp->value; > > > > + dt_set_range(&hw_reg, shmem_node, addr, size); > > > > + > > > > + return 0; > > > > +} > > > > + > > > > +static void free_channel_list(void) > > > > +{ > > > > + struct scmi_channel *curr, *_curr; > > > > + > > > > + spin_lock(&scmi_data.channel_list_lock); > > > > + list_for_each_entry_safe (curr, _curr, &scmi_data.channel_list, list) > > > > + { > > > > + vunmap(curr->shmem); > > > > + list_del(&curr->list); > > > > + xfree(curr); > > > > + } > > > > + > > > > + spin_unlock(&scmi_data.channel_list_lock); > > > > +} > > > > + > > > > +static __init bool scmi_probe(struct dt_device_node *scmi_node) > > > > +{ > > > > + struct dt_device_node *shmem_node; > > > > + int ret, i; > > > > + struct scmi_channel *channel, *agent_channel; > > > > + int n_agents; > > > > + scmi_msg_header_t hdr; > > > > + struct rx_t { > > > > + int32_t status; > > > > + uint32_t attributes; > > > > + } rx; > > > > > > Should rx be defined at the top together with scmi_perms_tx_t and > > > others? > > > > > > > I'd rather move scmi_perms_tx_t to scmi_add_device_by_devid because it's > > used only in 1 place. > > So we will have rx and tx in scmi_add_device_by_devid and rx ( which > > differs from rx in scmi_add_device_by_devid ) in scmi_probe. > > I think it will be more understandable and no need to make unique names. > > What do you think about that? > > I think that's OK > > > > > > + uint32_t func_id; > > > > + > > > > + ASSERT(scmi_node != NULL); > > > > + > > > > + INIT_LIST_HEAD(&scmi_data.channel_list); > > > > + spin_lock_init(&scmi_data.channel_list_lock); > > > > + > > > > + if ( !dt_property_read_u32(scmi_node, SCMI_SMC_ID, &func_id) ) > > > > + { > > > > + printk(XENLOG_ERR "scmi: Unable to read smc-id from DT\n"); > > > > + return false; > > > > + } > > > > + > > > > + shmem_node = dt_find_node_by_name(NULL, SCMI_SHARED_MEMORY); > > > > > > From the spec, it looks like you should be getting the shared memory > > > area from the phandle list "shmem". > > > > > > > We use SCMI_SHARED_MEMORY to get whole memory region (0x10 pages in my case), > > we can use for the agents. As you can see below - Hypervisor received number of > > agents from Firmware and split this region between agents. > > In general we can't use properties that are not part of the device tree > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > activities to get "linux,scmi_mem" upstream under > Documentation/devicetree/bindings in Linux? > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > Documentation/devicetree/bindings (probably > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > work on the Xen code that makes use of it. > > Does it make sense? > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > + if ( IS_ERR_OR_NULL(shmem_node) ) > > > > + { > > > > + printk(XENLOG_ERR > > > > + "scmi: Device tree error, can't parse shmem phandle %ld\n", > > > > + PTR_ERR(shmem_node)); > > > > + return false; > > > > + } > > > > + > > > > + ret = dt_device_get_address(shmem_node, 0, &scmi_data.shmem_addr, > > > > + &scmi_data.shmem_size); > > > > + if ( IS_ERR_VALUE(ret) ) > > > > + return false; > > > > + > > > > + channel = smc_create_channel(HYP_CHANNEL, func_id, scmi_data.shmem_addr); > > > > + if ( IS_ERR(channel) ) > > > > + return false; > > > > + > > > > + hdr.id = SCMI_BASE_PROTOCOL_ATTIBUTES; > > > > + hdr.type = 0; > > > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > > > + > > > > + ret = do_smc_xfer(channel, &hdr, NULL, 0, &rx, sizeof(rx)); > > > > + if ( ret ) > > > > + goto clean; > > > > + > > > > + ret = check_scmi_status(rx.status); > > > > + if ( ret ) > > > > + goto clean; > > > > + > > > > + n_agents = FIELD_GET(MSG_N_AGENTS_MASK, rx.attributes); > > > > + printk(XENLOG_DEBUG "scmi: Got agent count %d\n", n_agents); > > > > + > > > > + n_agents = (n_agents > scmi_data.shmem_size / PAGE_SIZE) ? > > > > + scmi_data.shmem_size / PAGE_SIZE : n_agents; > > > > + > > > > + for ( i = 1; i < n_agents; i++ ) > > > > + { > > > > > > Given that HYP_CHANNEL is actually zero, it looks like we could do > > > everything here in this loop but starting from i=0? > > > > > > > We allocate HYP_CHANNEL before loop because we need it to request number > > of agents. And we don't need to send SCMI_BASE_DISCOVER_AGENT to > > HYP_CHANNEL. > > OK > > > > > > + uint32_t tx_agent_id = 0xFFFFFFFF; > > > > + struct { > > > > + int32_t status; > > > > + uint32_t agent_id; > > > > + char name[16]; > > > > + } da_rx; > > > > + > > > > + agent_channel = smc_create_channel(i, func_id, scmi_data.shmem_addr + > > > > + i * PAGE_SIZE); > > > > + if ( IS_ERR(agent_channel) ) > > > > + { > > > > + ret = PTR_ERR(agent_channel); > > > > + goto clean; > > > > + } > > > > + > > > > + hdr.id = SCMI_BASE_DISCOVER_AGENT; > > > > + hdr.type = 0; > > > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > > > + > > > > + ret = do_smc_xfer(agent_channel, &hdr, &tx_agent_id, > > > > + sizeof(tx_agent_id), &da_rx, sizeof(da_rx)); > > > > + if ( ret ) > > > > + goto clean; > > > > + > > > > + ret = check_scmi_status(da_rx.status); > > > > + if ( ret ) > > > > + goto clean; > > > > + > > > > + printk(XENLOG_DEBUG "scmi: status=0x%x id=0x%x name=%s\n", > > > > + da_rx.status, da_rx.agent_id, da_rx.name); > > > > + > > > > + agent_channel->agent_id = da_rx.agent_id; > > > > + } > > > > + > > > > + scmi_data.initialized = true; > > > > + return true; > > > > + > > > > +clean: > > > > + free_channel_list(); > > > > + return ret == 0; > > > > +} > > > > + > > > > +static int scmi_domain_init(struct domain *d) > > > > +{ > > > > + struct scmi_channel *channel; > > > > + int ret; > > > > + > > > > + if ( !scmi_data.initialized ) > > > > + return 0; > > > > + > > > > + channel = aquire_scmi_channel(d->domain_id); > > > > + if ( IS_ERR_OR_NULL(channel) ) > > > > + return -ENOENT; > > > > + > > > > + printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = %d" > > > > + "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, > > > > + channel->paddr); > > > > + > > > > + if ( is_hardware_domain(d) ) > > > > + { > > > > + ret = map_memory_to_domain(d, scmi_data.shmem_addr, > > > > + scmi_data.shmem_size); > > > > + if ( IS_ERR_VALUE(ret) ) > > > > + goto error; > > > > + > > > > + ret = dt_update_domain_range(d, channel->paddr, PAGE_SIZE); > > > > + if ( IS_ERR_VALUE(ret) ) > > > > + { > > > > + int rc = unmap_memory_from_domain(d, scmi_data.shmem_addr, > > > > + scmi_data.shmem_size); > > > > + if ( rc ) > > > > + printk(XENLOG_ERR "Unable to unmap_memory_from_domain\n"); > > > > + > > > > + goto error; > > > > + } > > > > + } > > > > > > Is dom0 the only domain to get direct access to the shared memory > > > region? If so, I don't think it is a good idea to make Dom0 "special" in > > > this case. > > > > > > Let me make an example: if we assign a device to a domU since boot, and > > > dom0 wants to change the frequency of a clock that affects the assigned > > > device (likely because it doesn't know it is assigned), then dom0 > > > shouldn't be able to. We might have to perform checks in Xen to make > > > sure dom0 cannot stop the clock for the assigned device. > > > > > > So I think it would be better if all domains are treated the same way in > > > the mediator unless really necessary. > > > > > > On the other hand, if all domains get access to the shared memory > > > region, then I don't think this is likely the right place to create the > > > dom0 mapping. We probably want to do it in domain_build.c in a way that > > > can be reused for dom0less domUs. > > > > > > > The idea is that all domains have their own page in shared memory > > region and unigue agent_id. Agent_id is used to set permissions for > > clocks\resets\power-domains etc. So during creation of domain (domUs or > > dom0) device-tree is processed using scmi_add_dt_device and > > clocks\resets\power-domains which are related to dom0 devices will be > > requested by using SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > All passed-through devices will be requested during DomU creation. Which > > means dom0 do not have an access to clocks\resets\power-domains, which > > are related to DomU. > > OK, excellent > > > > > In regards to shared memory: it looks like the only two functions to > > > access the real shared memory are send_smc_message and get_smc_response. > > > If that is the case, then we actually don't need to expose the real > > > shared memory to any of the domains. > > > > > > We could simply: > > > > > > - expose a regular normal memory region as dom0/domU channel memory > > > - on SMC trap, read from the "fake" shared memory and set the > > > corresponding real shared memory on the appropriate channel > > > - issue the SMC call > > > - on return from SMC, copy over data from the real shared memory to the > > > "fake" channel reagion > > > > Hypervisor redirects only SMC calls from guests and set agent_id to SMC > > parameters as a7. The idea was to give page for each agent, so we don't > > need to make additional read/write each time we receive SMC call. > > All we povide from hypervisor is agent_id. Firmware is responsible for > > reading memory from the correct address and place the response. > > > > > > > > This is useful if we need to "filter" any of the SCMI commands and > > > options from the domains to the firmware, and also it is useful if the > > > channel memory is not page aligned. But if the permissions are > > > fine-grained enough and also the channel memory is page aligned (and > > > multiple of 4K in size) then we could map the memory. > > > > > > > In current implementation we suppose that channel memory is page aligned. > > I think that Firmware should be responsible for permissions handling and > > "filtering", that's why permission calls were added to SCMI spec. > > I tried to make mediator as thin as possible. > > OK. Can we check that the channel memory is page aligned at init time > and throw an error if it is not the case? > Yes. I will add this check in v2. > > > > > + > > > > + d->arch.sci = channel; > > > > + > > > > + return 0; > > > > +error: > > > > + relinquish_scmi_channel(channel); > > > > + > > > > + return ret; > > > > +} > > > > + > > > > +static int scmi_add_device_by_devid(struct domain *d, uint32_t scmi_devid) > > > > +{ > > > > + struct scmi_channel *channel, *agent_channel; > > > > + scmi_msg_header_t hdr; > > > > + scmi_perms_tx_t tx; > > > > + struct rx_t { > > > > + int32_t status; > > > > + uint32_t attributes; > > > > + } rx; > > > > + int ret; > > > > + > > > > + if ( !scmi_data.initialized ) > > > > + return 0; > > > > + > > > > + printk(XENLOG_DEBUG "scmi: scmi_devid = %d\n", scmi_devid); > > > > + > > > > + agent_channel = get_channel_by_domain(d->domain_id); > > > > + if ( IS_ERR_OR_NULL(agent_channel) ) > > > > + return PTR_ERR(agent_channel); > > > > + > > > > + channel = get_channel_by_id(HYP_CHANNEL); > > > > + if ( IS_ERR_OR_NULL(channel) ) > > > > + return PTR_ERR(channel); > > > > + > > > > + hdr.id = SCMI_BASE_SET_DEVICE_PERMISSIONS; > > > > + hdr.type = 0; > > > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > > > + > > > > + tx.agent_id = agent_channel->agent_id; > > > > + tx.device_id = scmi_devid; > > > > + tx.flags = SCMI_ALLOW_ACCESS; > > > > + > > > > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(&rx)); > > > > + if ( IS_ERR_VALUE(ret) ) > > > > + return ret; > > > > + > > > > + ret = check_scmi_status(rx.status); > > > > + if ( IS_ERR_VALUE(ret) ) > > > > + return ret; > > > > + > > > > + return 0; > > > > +} > > > > + > > > > +static int scmi_add_dt_device(struct domain *d, struct dt_device_node *dev) > > > > +{ > > > > + uint32_t scmi_devid; > > > > + > > > > + if ( (!scmi_data.initialized) || (!d->arch.sci) ) > > > > + return 0; > > > > + > > > > + if ( !dt_property_read_u32(dev, "scmi_devid", &scmi_devid) ) > > > > + return 0; > > > > > > scmi_devid is another property that is not documented in the binding. > > > > > > > This property should be added to the device nodes, which are using scmi > > to work with clocks\resets\power-domains etc. This id should match the > > device_id, defined in Firmware. Hypervisor send this device_id to the Firmware > > as parameter to the permission request. Firmware set permissions to > > clocks\resets\power-domains, related to this device. > > OK, I see. Unfortunately, scmi_devid is also not described under > Documentation/devicetree/bindings/. > > This property seems to be actually required for the system to work > correctly. > > Unless somebody else is already working on this, please send a patch to > the Linux kernel mailing list CCing the SCMI maintainers and Rob Herring > to introduce scmi_devid as a new property. > I will prepare and send patch introducing scmi_devid and linux,scmi_mem. > > > > > + printk(XENLOG_INFO "scmi: dt_node = %s\n", dt_node_full_name(dev)); > > > > + > > > > + return scmi_add_device_by_devid(d, scmi_devid); > > > > +} > > > > + > > > > +static int scmi_relinquish_resources(struct domain *d) > > > > +{ > > > > + int ret; > > > > + struct scmi_channel *channel, *agent_channel; > > > > + scmi_msg_header_t hdr; > > > > + struct reset_agent_tx { > > > > + uint32_t agent_id; > > > > + uint32_t flags; > > > > + } tx; > > > > + uint32_t rx; > > > > + > > > > + if ( !d->arch.sci ) > > > > + return 0; > > > > + > > > > + agent_channel = d->arch.sci; > > > > + > > > > + spin_lock(&agent_channel->lock); > > > > + tx.agent_id = agent_channel->agent_id; > > > > + spin_unlock(&agent_channel->lock); > > > > + > > > > + channel = get_channel_by_id(HYP_CHANNEL); > > > > + if ( !channel ) > > > > + { > > > > + printk(XENLOG_ERR > > > > + "scmi: Unable to get Hypervisor scmi channel for domain %d\n", > > > > + d->domain_id); > > > > + return -EINVAL; > > > > + } > > > > + > > > > + hdr.id = SCMI_BASE_RESET_AGENT_CONFIGURATION; > > > > + hdr.type = 0; > > > > + hdr.protocol = SCMI_BASE_PROTOCOL; > > > > + > > > > + tx.flags = 0; > > > > + > > > > + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(rx)); > > > > + if ( ret ) > > > > + return ret; > > > > + > > > > + ret = check_scmi_status(rx); > > > > + > > > > + return ret; > > > > +} > > > > + > > > > +static void scmi_domain_destroy(struct domain *d) > > > > +{ > > > > + struct scmi_channel *channel; > > > > + > > > > + if ( !d->arch.sci ) > > > > + return; > > > > + > > > > + channel = d->arch.sci; > > > > + spin_lock(&channel->lock); > > > > + > > > > + relinquish_scmi_channel(channel); > > > > + printk(XENLOG_DEBUG "scmi: Free domain %d\n", d->domain_id); > > > > + > > > > + d->arch.sci = NULL; > > > > + > > > > + unmap_memory_from_domain(d, channel->paddr, PAGE_SIZE); > > > > + spin_unlock(&channel->lock); > > > > + return; > > > > +} > > > > + > > > > +static bool scmi_handle_call(struct domain *d, void *args) > > > > +{ > > > > + bool res = false; > > > > + struct scmi_channel *agent_channel; > > > > + struct arm_smccc_res resp; > > > > + struct cpu_user_regs *regs = args; > > > > + > > > > + if ( !d->arch.sci ) > > > > + return false; > > > > + > > > > + agent_channel = d->arch.sci; > > > > + spin_lock(&agent_channel->lock); > > > > + > > > > + if ( agent_channel->func_id != regs->x0 ) > > > > + { > > > > + printk(XENLOG_ERR "scmi: func_id mismatch, exiting\n"); > > > > + goto unlock; > > > > + } > > > > + > > > > + arm_smccc_smc(agent_channel->func_id, 0, 0, 0, 0, 0, 0, > > > > + agent_channel->chan_id, &resp); > > > > + > > > > + set_user_reg(regs, 0, resp.a0); > > > > + set_user_reg(regs, 1, resp.a1); > > > > + set_user_reg(regs, 2, resp.a2); > > > > + set_user_reg(regs, 3, resp.a3); > > > > + res = true; > > > > +unlock: > > > > + spin_unlock(&agent_channel->lock); > > > > + > > > > + return res; > > > > +} > > > > + > > > > +static int scmi_get_channel_paddr(void *scmi_ops, > > > > + struct xen_arch_domainconfig *config) > > > > +{ > > > > + struct scmi_channel *agent_channel = scmi_ops; > > > > + > > > > + if ( !agent_channel ) > > > > + return -EINVAL; > > > > + > > > > + config->sci_agent_paddr = agent_channel->paddr; > > > > + return 0; > > > > +} > > > > > > I am still not sure why it couldn't be done by scmi_domain_init. > > > > > > > I can move this logic to scmi_domain_init, but in this case I have to add > > struct xen_arch_domainconfig *config as input parameter to > > scmi_domain_init and pass NULL from construct_dom0. > > Do you think this approach would be better? > > I think it is OK to pass struct xen_arch_domainconfig *config as input > parameter to scmi_domain_init. Ok, I will fix it in v2. > > For dom0, why is sci_agent_paddr not supposed to be set? > paddr is the channel address. We need this address to update reg address of arm,scmi-shmem node in guest device-tree, so guest scmi driver will use shmem related to it. > > > > Also I think it's reasonable to pass xen_arch_domainconfig so different > > implementations could set another data they would probably need. > > > > > > > > > +static const struct dt_device_match scmi_smc_match[] __initconst = > > > > +{ > > > > + DT_MATCH_SCMI_SMC, > > > > + { /* sentinel */ }, > > > > +}; > > > > + > > > > +static const struct sci_mediator_ops scmi_ops = > > > > +{ > > > > + .probe = scmi_probe, > > > > + .domain_init = scmi_domain_init, > > > > + .domain_destroy = scmi_domain_destroy, > > > > + .add_dt_device = scmi_add_dt_device, > > > > + .relinquish_resources = scmi_relinquish_resources, > > > > + .handle_call = scmi_handle_call, > > > > + .get_channel_info = scmi_get_channel_paddr > > > > +}; > > > > + > > > > +REGISTER_SCI_MEDIATOR(scmi_smc, "SCMI-SMC", XEN_DOMCTL_CONFIG_SCI_SCMI_SMC, > > > > + scmi_smc_match, &scmi_ops); > > > > + > > > > +/* > > > > + * Local variables: > > > > + * mode: C > > > > + * c-file-style: "BSD" > > > > + * c-basic-offset: 4 > > > > + * indent-tabs-mode: nil > > > > + * End: > > > > + */ > > > > diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h > > > > index 9180be5e86..a67237942d 100644 > > > > --- a/xen/include/public/arch-arm.h > > > > +++ b/xen/include/public/arch-arm.h > > > > @@ -315,6 +315,7 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t); > > > > #define XEN_DOMCTL_CONFIG_TEE_OPTEE 1 > > > > > > > > #define XEN_DOMCTL_CONFIG_SCI_NONE 0 > > > > +#define XEN_DOMCTL_CONFIG_SCI_SCMI_SMC 1 > > > > > > > > struct xen_arch_domainconfig { > > > > /* IN/OUT */
On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > Hi Stefano, > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > Hi Stefano, > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > power-domains without access to CPG. > > > > > > > > > > The following features are implemented: > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > - set device permissions for Domains based on the Domain partial > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > resets and power-domains via SCMI; > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > --- > > > > > xen/arch/arm/Kconfig | 2 + > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > xen/include/public/arch-arm.h | 1 + > > > > > 5 files changed, 809 insertions(+) > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > --- a/xen/arch/arm/Kconfig > > > > > +++ b/xen/arch/arm/Kconfig > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > support. It allows guests to control system resourcess via one of > > > > > SCI mediators implemented in XEN. > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > + > > > > > endmenu > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > new file mode 100644 > > > > > index 0000000000..9563067ddc > > > > > --- /dev/null > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > @@ -0,0 +1,10 @@ > > > > > +config SCMI_SMC > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > + default n > > > > > + depends on SCI > > > > > + ---help--- > > > > > + > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > + This feature allows drivers from Domains to work with System > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > + for communication. > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > index 837dc7492b..67f2611872 100644 > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > @@ -1 +1,2 @@ > > > > > obj-y += sci.o > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > new file mode 100644 > > > > > index 0000000000..2eb01ea82d > > > > > --- /dev/null > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > @@ -0,0 +1,795 @@ > > > > > +/* > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > + * > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > + * > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > + * > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > + * it under the terms of the GNU General Public License as published by > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > + * (at your option) any later version. > > > > > + * > > > > > + * This program is distributed in the hope that it will be useful, > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > + * GNU General Public License for more details. > > > > > + */ > > > > > + > > > > > +#include <asm/sci/sci.h> > > > > > +#include <asm/smccc.h> > > > > > +#include <asm/io.h> > > > > > +#include <xen/bitops.h> > > > > > +#include <xen/config.h> > > > > > +#include <xen/sched.h> > > > > > +#include <xen/device_tree.h> > > > > > +#include <xen/iocap.h> > > > > > +#include <xen/init.h> > > > > > +#include <xen/err.h> > > > > > +#include <xen/lib.h> > > > > > +#include <xen/list.h> > > > > > +#include <xen/mm.h> > > > > > +#include <xen/string.h> > > > > > +#include <xen/time.h> > > > > > +#include <xen/vmap.h> > > > > > + > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > + > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > +#define SCMI_SUCCESS 0 > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > +#define SCMI_DENIED (-3) > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > +#define SCMI_BUSY (-6) > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > + > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > + > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > the "shmem" property instead? And the compatible string used for this > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > channels: > > > > > > reserved-memory { > > > /* reserved region for scmi channels*/ > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > no-map; > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > }; > > > }; > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > the current scmi channel: > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > compatible = "arm,scmi-shmem"; > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > }; > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > assigned to this agent. > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > compatible string, not as a node name, and it would need to be described > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > But from your description I don't think it is necessary. We can just use > > "arm,scmi-shmem" to describe all the required regions: > > > > reserved-memory { > > scp-shmem@0x53FF0000 { > > compatible = "arm,scmi-shmem"; > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > }; > > scp-shmem@0x53FF1000 { > > compatible = "arm,scmi-shmem"; > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > }; > > scp-shmem@0x53FF2000 { > > compatible = "arm,scmi-shmem"; > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > }; > > ... > > > > In other words, if all the individual channel pages are described as > > "arm,scmi-shmem", why do we also need a single larger region as > > "linux,scmi_mem"? > > > > That was my first implementation. But I've met a problem with > scmi driver in kernel. I don't remember the exact place, but I remember > there were some if, checking if memory weren't reserved. > That's why I ended up splitting nodes reserved memory region and actual > shmem page. > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > which has no compatible string and provides no-map property. > linux,scmi_shmem node is needed to prevent xen from allocating this > space for the domain. > > Very interesting question about should I introduce linux,scmi_mem node > and scmi_devid property to the > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > Those node and property are needed only for Xen and useless for > non-virtualized systems. I can add this node and property description to > arm,scmi.yaml, but leave a note that this is Xen specific params. > What do you think about it? Reply below [...] > > In general we can't use properties that are not part of the device tree > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > activities to get "linux,scmi_mem" upstream under > > Documentation/devicetree/bindings in Linux? > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > Documentation/devicetree/bindings (probably > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > work on the Xen code that makes use of it. > > > > Does it make sense? > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. I didn't realize that linux,scmi_mem and scmi_devid are supposed to be Xen specific. In general, it would be best not to introduce Xen specific properties into generic bindings. It is a problem both from a specification perspective (because it has hard to handle Xen specific cases in fully generic bindings, especially as those bindings are maintained as part of the Linux kernel) and from a user perspective (because now the user has to deal with a Xen-specific dtb, or has to modify the host dtb to add Xen-specific information by hand.) Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It looks like a generic property that should be needed for the Linux SCMI driver too. Why the Linux driver doesn't need it? In regards to linux,scmi_mem, I think it would be best to do without it and fix the Linux SCMI driver if we need to do so. Xen should be able to parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either way, I don't think we should need linux,scmi_mem.
On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > Hi Stefano, > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > Hi Stefano, > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > power-domains without access to CPG. > > > > > > > > > > > > The following features are implemented: > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > resets and power-domains via SCMI; > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > --- > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > 5 files changed, 809 insertions(+) > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > support. It allows guests to control system resourcess via one of > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > + > > > > > > endmenu > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > new file mode 100644 > > > > > > index 0000000000..9563067ddc > > > > > > --- /dev/null > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > @@ -0,0 +1,10 @@ > > > > > > +config SCMI_SMC > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > + default n > > > > > > + depends on SCI > > > > > > + ---help--- > > > > > > + > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > + This feature allows drivers from Domains to work with System > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > + for communication. > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > @@ -1 +1,2 @@ > > > > > > obj-y += sci.o > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > new file mode 100644 > > > > > > index 0000000000..2eb01ea82d > > > > > > --- /dev/null > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > @@ -0,0 +1,795 @@ > > > > > > +/* > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > + * > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > + * > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > + * > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > + * (at your option) any later version. > > > > > > + * > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > + * GNU General Public License for more details. > > > > > > + */ > > > > > > + > > > > > > +#include <asm/sci/sci.h> > > > > > > +#include <asm/smccc.h> > > > > > > +#include <asm/io.h> > > > > > > +#include <xen/bitops.h> > > > > > > +#include <xen/config.h> > > > > > > +#include <xen/sched.h> > > > > > > +#include <xen/device_tree.h> > > > > > > +#include <xen/iocap.h> > > > > > > +#include <xen/init.h> > > > > > > +#include <xen/err.h> > > > > > > +#include <xen/lib.h> > > > > > > +#include <xen/list.h> > > > > > > +#include <xen/mm.h> > > > > > > +#include <xen/string.h> > > > > > > +#include <xen/time.h> > > > > > > +#include <xen/vmap.h> > > > > > > + > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > + > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > +#define SCMI_SUCCESS 0 > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > +#define SCMI_DENIED (-3) > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > +#define SCMI_BUSY (-6) > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > + > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > + > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > the "shmem" property instead? And the compatible string used for this > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > channels: > > > > > > > > reserved-memory { > > > > /* reserved region for scmi channels*/ > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > no-map; > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > }; > > > > }; > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > the current scmi channel: > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > compatible = "arm,scmi-shmem"; > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > }; > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > assigned to this agent. > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > compatible string, not as a node name, and it would need to be described > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > But from your description I don't think it is necessary. We can just use > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > reserved-memory { > > > scp-shmem@0x53FF0000 { > > > compatible = "arm,scmi-shmem"; > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > }; > > > scp-shmem@0x53FF1000 { > > > compatible = "arm,scmi-shmem"; > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > }; > > > scp-shmem@0x53FF2000 { > > > compatible = "arm,scmi-shmem"; > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > }; > > > ... > > > > > > In other words, if all the individual channel pages are described as > > > "arm,scmi-shmem", why do we also need a single larger region as > > > "linux,scmi_mem"? > > > > > > > That was my first implementation. But I've met a problem with > > scmi driver in kernel. I don't remember the exact place, but I remember > > there were some if, checking if memory weren't reserved. > > That's why I ended up splitting nodes reserved memory region and actual > > shmem page. > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > which has no compatible string and provides no-map property. > > linux,scmi_shmem node is needed to prevent xen from allocating this > > space for the domain. > > > > Very interesting question about should I introduce linux,scmi_mem node > > and scmi_devid property to the > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > Those node and property are needed only for Xen and useless for > > non-virtualized systems. I can add this node and property description to > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > What do you think about it? > > Reply below > > [...] > > > > > In general we can't use properties that are not part of the device tree > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > activities to get "linux,scmi_mem" upstream under > > > Documentation/devicetree/bindings in Linux? > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > Documentation/devicetree/bindings (probably > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > work on the Xen code that makes use of it. > > > > > > Does it make sense? > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > Xen specific. In general, it would be best not to introduce Xen specific > properties into generic bindings. It is a problem both from a > specification perspective (because it has hard to handle Xen specific > cases in fully generic bindings, especially as those bindings are > maintained as part of the Linux kernel) and from a user perspective > (because now the user has to deal with a Xen-specific dtb, or has to > modify the host dtb to add Xen-specific information by hand.) > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > looks like a generic property that should be needed for the Linux SCMI > driver too. Why the Linux driver doesn't need it? > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. On non-virtualized systems - there is no need of this call, because OS is the only one entity, running on the system. I've chatted with Volodymyr_Babchuk and he gave a great idea to add a list of device_ids to dom.cfg, such as: sci_devs = [ 0, 1, 15, 35 ]; Using this approach, we can remove scmi_devid from the device tree and just pass a list of scmi_devids to XEN using additional hypercall. We can probably make hypercall taking devid list as input parameter. This will take only 1 hypercall to setup sci permissions. > > In regards to linux,scmi_mem, I think it would be best to do without it > and fix the Linux SCMI driver if we need to do so. Xen should be able to > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > way, I don't think we should need linux,scmi_mem. This requires further investigation. I will try to make implementation without linux,scmi_mem, using only arm,scmi-shmem nodes and share reuslts with you.
On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > Hi Stefano, > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > Hi Stefano, > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > The following features are implemented: > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > resets and power-domains via SCMI; > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > --- > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > + > > > > > > > endmenu > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > new file mode 100644 > > > > > > > index 0000000000..9563067ddc > > > > > > > --- /dev/null > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > +config SCMI_SMC > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > + default n > > > > > > > + depends on SCI > > > > > > > + ---help--- > > > > > > > + > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > + for communication. > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > @@ -1 +1,2 @@ > > > > > > > obj-y += sci.o > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > new file mode 100644 > > > > > > > index 0000000000..2eb01ea82d > > > > > > > --- /dev/null > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > +/* > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > + * > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > + * > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > + * > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > + * (at your option) any later version. > > > > > > > + * > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > + * GNU General Public License for more details. > > > > > > > + */ > > > > > > > + > > > > > > > +#include <asm/sci/sci.h> > > > > > > > +#include <asm/smccc.h> > > > > > > > +#include <asm/io.h> > > > > > > > +#include <xen/bitops.h> > > > > > > > +#include <xen/config.h> > > > > > > > +#include <xen/sched.h> > > > > > > > +#include <xen/device_tree.h> > > > > > > > +#include <xen/iocap.h> > > > > > > > +#include <xen/init.h> > > > > > > > +#include <xen/err.h> > > > > > > > +#include <xen/lib.h> > > > > > > > +#include <xen/list.h> > > > > > > > +#include <xen/mm.h> > > > > > > > +#include <xen/string.h> > > > > > > > +#include <xen/time.h> > > > > > > > +#include <xen/vmap.h> > > > > > > > + > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > + > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > + > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > + > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > channels: > > > > > > > > > > reserved-memory { > > > > > /* reserved region for scmi channels*/ > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > no-map; > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > }; > > > > > }; > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > the current scmi channel: > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > }; > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > assigned to this agent. > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > compatible string, not as a node name, and it would need to be described > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > reserved-memory { > > > > scp-shmem@0x53FF0000 { > > > > compatible = "arm,scmi-shmem"; > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > }; > > > > scp-shmem@0x53FF1000 { > > > > compatible = "arm,scmi-shmem"; > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > }; > > > > scp-shmem@0x53FF2000 { > > > > compatible = "arm,scmi-shmem"; > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > }; > > > > ... > > > > > > > > In other words, if all the individual channel pages are described as > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > "linux,scmi_mem"? > > > > > > > > > > That was my first implementation. But I've met a problem with > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > there were some if, checking if memory weren't reserved. > > > That's why I ended up splitting nodes reserved memory region and actual > > > shmem page. > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > which has no compatible string and provides no-map property. > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > space for the domain. > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > and scmi_devid property to the > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > Those node and property are needed only for Xen and useless for > > > non-virtualized systems. I can add this node and property description to > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > What do you think about it? > > > > Reply below > > > > [...] > > > > > > > > In general we can't use properties that are not part of the device tree > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > activities to get "linux,scmi_mem" upstream under > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > Documentation/devicetree/bindings (probably > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > work on the Xen code that makes use of it. > > > > > > > > Does it make sense? > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > Xen specific. In general, it would be best not to introduce Xen specific > > properties into generic bindings. It is a problem both from a > > specification perspective (because it has hard to handle Xen specific > > cases in fully generic bindings, especially as those bindings are > > maintained as part of the Linux kernel) and from a user perspective > > (because now the user has to deal with a Xen-specific dtb, or has to > > modify the host dtb to add Xen-specific information by hand.) > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > looks like a generic property that should be needed for the Linux SCMI > > driver too. Why the Linux driver doesn't need it? > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > On non-virtualized systems - there is no need of this call, because OS > is the only one entity, running on the system. OK. Even if it is only required for virtualized systems, I think that scmi_devid is important enough that should be part of the upstream binding. I think it is worth starting an email thread on the LKML with Rob Herring and the SCMI maintainers to discuss the addition of scmi_devid to the binding. > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > list of device_ids to dom.cfg, such as: > sci_devs = [ 0, 1, 15, 35 ]; > > Using this approach, we can remove scmi_devid from the device tree and > just pass a list of scmi_devids to XEN using additional hypercall. > We can probably make hypercall taking devid list as input parameter. > This will take only 1 hypercall to setup sci permissions. But how would a user know which are the right SCMI IDs to add to the sci_devs list? Would the user have to go and read the reference manual of the platform to find the SCMI IDs and then write sci_devs by hand? If that is the case, then I think that it would be better to add scmi_devid to device tree. In general, I think this configuration should happen automatically without user intervention. The user should just specify "enable SCMI" and it should work. > > In regards to linux,scmi_mem, I think it would be best to do without it > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > way, I don't think we should need linux,scmi_mem. > > This requires further investigation. I will try to make implementation > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > reuslts with you. OK, thanks.
Hi Stefano, Oleksii, Stefano Stabellini <sstabellini@kernel.org> writes: > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: >> On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: >> > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: >> > > Hi Stefano, >> > > >> > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: >> > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: >> > > > > Hi Stefano, >> > > > > >> > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: >> > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: [...] >> > > > In general we can't use properties that are not part of the device tree >> > > > spec, either >> > > > https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ >> > > > [devicetree[.]org] or >> > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ >> > > > [git[.]kernel[.]org] >> > > > >> > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming >> > > > activities to get "linux,scmi_mem" upstream under >> > > > Documentation/devicetree/bindings in Linux? >> > > > >> > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. >> > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under >> > > > Documentation/devicetree/bindings (probably >> > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can >> > > > work on the Xen code that makes use of it. >> > > > >> > > > Does it make sense? >> > > > >> > > >> > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. >> > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. >> > >> > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be >> > Xen specific. In general, it would be best not to introduce Xen specific >> > properties into generic bindings. It is a problem both from a >> > specification perspective (because it has hard to handle Xen specific >> > cases in fully generic bindings, especially as those bindings are >> > maintained as part of the Linux kernel) and from a user perspective >> > (because now the user has to deal with a Xen-specific dtb, or has to >> > modify the host dtb to add Xen-specific information by hand.) >> > >> > >> > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It >> > looks like a generic property that should be needed for the Linux SCMI >> > driver too. Why the Linux driver doesn't need it? >> > >> >> scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. >> On non-virtualized systems - there is no need of this call, because OS >> is the only one entity, running on the system. > > OK. Even if it is only required for virtualized systems, I think that > scmi_devid is important enough that should be part of the upstream > binding. I think it is worth starting an email thread on the LKML with > Rob Herring and the SCMI maintainers to discuss the addition of > scmi_devid to the binding. Agree there. Also I want to point that there are other hypervisors, like KVM, which may benefit from this. Another approach is to name this node "xen,scmdi_devid", but I don't like it. >> I've chatted with Volodymyr_Babchuk and he gave a great idea to add a >> list of device_ids to dom.cfg, such as: >> sci_devs = [ 0, 1, 15, 35 ]; >> Well, yes. We discussed this possibility, but personally I stick to idea of re-using dt_dev, as we discussed in the different thread. >> Using this approach, we can remove scmi_devid from the device tree and >> just pass a list of scmi_devids to XEN using additional hypercall. >> We can probably make hypercall taking devid list as input parameter. >> This will take only 1 hypercall to setup sci permissions. > > But how would a user know which are the right SCMI IDs to add to the > sci_devs list? Would the user have to go and read the reference manual > of the platform to find the SCMI IDs and then write sci_devs by hand? > If that is the case, then I think that it would be better to add > scmi_devid to device tree. > Yes, ideally this needs to be done by BSP vendor in the device tree.
Hi Stefano, On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > Hi Stefano, > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > Hi Stefano, > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > resets and power-domains via SCMI; > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > --- > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > + > > > > > > > > endmenu > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > new file mode 100644 > > > > > > > > index 0000000000..9563067ddc > > > > > > > > --- /dev/null > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > +config SCMI_SMC > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > + default n > > > > > > > > + depends on SCI > > > > > > > > + ---help--- > > > > > > > > + > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > + for communication. > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > obj-y += sci.o > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > new file mode 100644 > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > --- /dev/null > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > +/* > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > + * > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > + * > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > + * > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > + * (at your option) any later version. > > > > > > > > + * > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > + * GNU General Public License for more details. > > > > > > > > + */ > > > > > > > > + > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > +#include <asm/smccc.h> > > > > > > > > +#include <asm/io.h> > > > > > > > > +#include <xen/bitops.h> > > > > > > > > +#include <xen/config.h> > > > > > > > > +#include <xen/sched.h> > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > +#include <xen/iocap.h> > > > > > > > > +#include <xen/init.h> > > > > > > > > +#include <xen/err.h> > > > > > > > > +#include <xen/lib.h> > > > > > > > > +#include <xen/list.h> > > > > > > > > +#include <xen/mm.h> > > > > > > > > +#include <xen/string.h> > > > > > > > > +#include <xen/time.h> > > > > > > > > +#include <xen/vmap.h> > > > > > > > > + > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > + > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > + > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > + > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > channels: > > > > > > > > > > > > reserved-memory { > > > > > > /* reserved region for scmi channels*/ > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > no-map; > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > }; > > > > > > }; > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > the current scmi channel: > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > }; > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > assigned to this agent. > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > compatible string, not as a node name, and it would need to be described > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > reserved-memory { > > > > > scp-shmem@0x53FF0000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > }; > > > > > scp-shmem@0x53FF1000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > }; > > > > > scp-shmem@0x53FF2000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > }; > > > > > ... > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > there were some if, checking if memory weren't reserved. > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > shmem page. > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > which has no compatible string and provides no-map property. > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > space for the domain. > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > and scmi_devid property to the > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > Those node and property are needed only for Xen and useless for > > > > non-virtualized systems. I can add this node and property description to > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > What do you think about it? > > > > > > Reply below > > > > > > [...] > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > activities to get "linux,scmi_mem" upstream under > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > Documentation/devicetree/bindings (probably > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > work on the Xen code that makes use of it. > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > Xen specific. In general, it would be best not to introduce Xen specific > > > properties into generic bindings. It is a problem both from a > > > specification perspective (because it has hard to handle Xen specific > > > cases in fully generic bindings, especially as those bindings are > > > maintained as part of the Linux kernel) and from a user perspective > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > looks like a generic property that should be needed for the Linux SCMI > > > driver too. Why the Linux driver doesn't need it? > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > On non-virtualized systems - there is no need of this call, because OS > > is the only one entity, running on the system. > > OK. Even if it is only required for virtualized systems, I think that > scmi_devid is important enough that should be part of the upstream > binding. I think it is worth starting an email thread on the LKML with > Rob Herring and the SCMI maintainers to discuss the addition of > scmi_devid to the binding. > Ok I will start the thread about scmi_devid. > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > list of device_ids to dom.cfg, such as: > > sci_devs = [ 0, 1, 15, 35 ]; > > > > Using this approach, we can remove scmi_devid from the device tree and > > just pass a list of scmi_devids to XEN using additional hypercall. > > We can probably make hypercall taking devid list as input parameter. > > This will take only 1 hypercall to setup sci permissions. > > But how would a user know which are the right SCMI IDs to add to the > sci_devs list? Would the user have to go and read the reference manual > of the platform to find the SCMI IDs and then write sci_devs by hand? > If that is the case, then I think that it would be better to add > scmi_devid to device tree. > > In general, I think this configuration should happen automatically > without user intervention. The user should just specify "enable SCMI" > and it should work. > Ok. This sounds reasonable. > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > way, I don't think we should need linux,scmi_mem. > > > > This requires further investigation. I will try to make implementation > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > reuslts with you. > > OK, thanks. One more question: As you probably seen - Jan had a complains about SCI term. He said SCI is ambiguous with ACPI's System Control Interrupt. I think of using SC (as System Control) instead. What do you think about it? Best regards, Oleksii.
On Thu, 23 Dec 2021, Oleksii Moisieiev wrote: > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > Hi Stefano, > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > Hi Stefano, > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > --- > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > + > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > new file mode 100644 > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > --- /dev/null > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > +config SCMI_SMC > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > + default n > > > > > > > > > + depends on SCI > > > > > > > > > + ---help--- > > > > > > > > > + > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > + for communication. > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > obj-y += sci.o > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > new file mode 100644 > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > --- /dev/null > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > +/* > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > + * > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > + * > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > + * > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > + * (at your option) any later version. > > > > > > > > > + * > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > + */ > > > > > > > > > + > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > +#include <asm/io.h> > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > +#include <xen/config.h> > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > +#include <xen/init.h> > > > > > > > > > +#include <xen/err.h> > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > +#include <xen/list.h> > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > +#include <xen/string.h> > > > > > > > > > +#include <xen/time.h> > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > + > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > + > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > + > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > + > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > channels: > > > > > > > > > > > > > > reserved-memory { > > > > > > > /* reserved region for scmi channels*/ > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > no-map; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > }; > > > > > > > }; > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > the current scmi channel: > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > }; > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > assigned to this agent. > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > reserved-memory { > > > > > > scp-shmem@0x53FF0000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > }; > > > > > > scp-shmem@0x53FF1000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > }; > > > > > > scp-shmem@0x53FF2000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > }; > > > > > > ... > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > there were some if, checking if memory weren't reserved. > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > shmem page. > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > which has no compatible string and provides no-map property. > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > space for the domain. > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > and scmi_devid property to the > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > Those node and property are needed only for Xen and useless for > > > > > non-virtualized systems. I can add this node and property description to > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > What do you think about it? > > > > > > > > Reply below > > > > > > > > [...] > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > Documentation/devicetree/bindings (probably > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > properties into generic bindings. It is a problem both from a > > > > specification perspective (because it has hard to handle Xen specific > > > > cases in fully generic bindings, especially as those bindings are > > > > maintained as part of the Linux kernel) and from a user perspective > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > looks like a generic property that should be needed for the Linux SCMI > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > On non-virtualized systems - there is no need of this call, because OS > > > is the only one entity, running on the system. > > > > OK. Even if it is only required for virtualized systems, I think that > > scmi_devid is important enough that should be part of the upstream > > binding. I think it is worth starting an email thread on the LKML with > > Rob Herring and the SCMI maintainers to discuss the addition of > > scmi_devid to the binding. > > > > Ok I will start the thread about scmi_devid. > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > list of device_ids to dom.cfg, such as: > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > We can probably make hypercall taking devid list as input parameter. > > > This will take only 1 hypercall to setup sci permissions. > > > > But how would a user know which are the right SCMI IDs to add to the > > sci_devs list? Would the user have to go and read the reference manual > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > If that is the case, then I think that it would be better to add > > scmi_devid to device tree. > > > > In general, I think this configuration should happen automatically > > without user intervention. The user should just specify "enable SCMI" > > and it should work. > > > > Ok. This sounds reasonable. > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > way, I don't think we should need linux,scmi_mem. > > > > > > This requires further investigation. I will try to make implementation > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > reuslts with you. > > > > OK, thanks. > > One more question: As you probably seen - Jan had a complains about SCI > term. He said SCI is ambiguous with ACPI's System > Control Interrupt. I see his point. As a term I see "SCMI" often and sometimes "SCPI" but "SCI" is the first time I saw it with this patch series. > I think of using SC (as System Control) instead. What do you think > about it? Yeah, I am not great at naming things but maybe "ARM_SCI"? "SC" alone doesn't give me enough context to guess what it is. Or we could broaden the scope and call it "firmware_interface"?
Hi, On 24/12/2021 01:16, Stefano Stabellini wrote: >> One more question: As you probably seen - Jan had a complains about SCI >> term. He said SCI is ambiguous with ACPI's System >> Control Interrupt. > > I see his point. As a term I see "SCMI" often and sometimes "SCPI" but > "SCI" is the first time I saw it with this patch series. > > >> I think of using SC (as System Control) instead. What do you think >> about it? > > Yeah, I am not great at naming things but maybe "ARM_SCI"? "SC" alone > doesn't give me enough context to guess what it is. I might be missing some context. Why are naming everything SCI rather than SMCI? > > Or we could broaden the scope and call it "firmware_interface"? How would this be used? Will it be a list of interface that will be exposed to the guest? Cheers,
Hi Julien, On Fri, Dec 24, 2021 at 02:29:13PM +0100, Julien Grall wrote: > Hi, > > On 24/12/2021 01:16, Stefano Stabellini wrote: > > > One more question: As you probably seen - Jan had a complains about SCI > > > term. He said SCI is ambiguous with ACPI's System > > > Control Interrupt. > > > > I see his point. As a term I see "SCMI" often and sometimes "SCPI" but > > "SCI" is the first time I saw it with this patch series. > > > > > > > I think of using SC (as System Control) instead. What do you think > > > about it? > > > > Yeah, I am not great at naming things but maybe "ARM_SCI"? "SC" alone > > doesn't give me enough context to guess what it is. > > I might be missing some context. Why are naming everything SCI rather than > SMCI? Because we're expecting other interfaces and transport to be implemented, such as for example: scmi_mailbox, scpi_smc, scpi_mailbox, ti_sci_smc etc. > > > > > Or we could broaden the scope and call it "firmware_interface"? > How would this be used? Will it be a list of interface that will be exposed > to the guest? > The idea is to set mediator type for each Domain, so for example Xen can use scmi_mailbox to communicate with SCP, Dom0 and DomD are also using scmi_mailbox, but DomU using scmi_smc mediator because we have only 3 mailboxes in system. This is not implemented yet, right now, we are introducing only scmi_smc support. In future, multiple mediator support can be added to Xen. Best regards, Oleksii.
On Thu, Dec 23, 2021 at 04:16:45PM -0800, Stefano Stabellini wrote: > On Thu, 23 Dec 2021, Oleksii Moisieiev wrote: > > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > > Hi Stefano, > > > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > --- > > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > > + > > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > > new file mode 100644 > > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > > --- /dev/null > > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > > +config SCMI_SMC > > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > > + default n > > > > > > > > > > + depends on SCI > > > > > > > > > > + ---help--- > > > > > > > > > > + > > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > > + for communication. > > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > > obj-y += sci.o > > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > new file mode 100644 > > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > > --- /dev/null > > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > > +/* > > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > + * > > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > > + * > > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > > + * > > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > > + * (at your option) any later version. > > > > > > > > > > + * > > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > > + */ > > > > > > > > > > + > > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > > +#include <asm/io.h> > > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > > +#include <xen/config.h> > > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > > +#include <xen/init.h> > > > > > > > > > > +#include <xen/err.h> > > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > > +#include <xen/list.h> > > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > > +#include <xen/string.h> > > > > > > > > > > +#include <xen/time.h> > > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > > + > > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > > + > > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > > + > > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > > + > > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > > channels: > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > /* reserved region for scmi channels*/ > > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > > no-map; > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > > }; > > > > > > > > }; > > > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > > the current scmi channel: > > > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > }; > > > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > > assigned to this agent. > > > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > > > reserved-memory { > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > }; > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > }; > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > }; > > > > > > > ... > > > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > > there were some if, checking if memory weren't reserved. > > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > > shmem page. > > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > > which has no compatible string and provides no-map property. > > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > > space for the domain. > > > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > > and scmi_devid property to the > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > > Those node and property are needed only for Xen and useless for > > > > > > non-virtualized systems. I can add this node and property description to > > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > > What do you think about it? > > > > > > > > > > Reply below > > > > > > > > > > [...] > > > > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > > Documentation/devicetree/bindings (probably > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > > properties into generic bindings. It is a problem both from a > > > > > specification perspective (because it has hard to handle Xen specific > > > > > cases in fully generic bindings, especially as those bindings are > > > > > maintained as part of the Linux kernel) and from a user perspective > > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > > looks like a generic property that should be needed for the Linux SCMI > > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > > On non-virtualized systems - there is no need of this call, because OS > > > > is the only one entity, running on the system. > > > > > > OK. Even if it is only required for virtualized systems, I think that > > > scmi_devid is important enough that should be part of the upstream > > > binding. I think it is worth starting an email thread on the LKML with > > > Rob Herring and the SCMI maintainers to discuss the addition of > > > scmi_devid to the binding. > > > > > > > Ok I will start the thread about scmi_devid. > > > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > > list of device_ids to dom.cfg, such as: > > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > > We can probably make hypercall taking devid list as input parameter. > > > > This will take only 1 hypercall to setup sci permissions. > > > > > > But how would a user know which are the right SCMI IDs to add to the > > > sci_devs list? Would the user have to go and read the reference manual > > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > > If that is the case, then I think that it would be better to add > > > scmi_devid to device tree. > > > > > > In general, I think this configuration should happen automatically > > > without user intervention. The user should just specify "enable SCMI" > > > and it should work. > > > > > > > Ok. This sounds reasonable. > > > > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > > way, I don't think we should need linux,scmi_mem. > > > > > > > > This requires further investigation. I will try to make implementation > > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > > reuslts with you. > > > > > > OK, thanks. > > > > One more question: As you probably seen - Jan had a complains about SCI > > term. He said SCI is ambiguous with ACPI's System > > Control Interrupt. > > I see his point. As a term I see "SCMI" often and sometimes "SCPI" but > "SCI" is the first time I saw it with this patch series. > > > > I think of using SC (as System Control) instead. What do you think > > about it? > > Yeah, I am not great at naming things but maybe "ARM_SCI"? "SC" alone > doesn't give me enough context to guess what it is. > > Or we could broaden the scope and call it "firmware_interface"? ARM_SCI sounds good for me. Best regards, Oleksii.
On 24/12/2021 14:59, Oleksii Moisieiev wrote: > Hi Julien, Hello, > On Fri, Dec 24, 2021 at 02:29:13PM +0100, Julien Grall wrote: >> Hi, >> >> On 24/12/2021 01:16, Stefano Stabellini wrote: >>>> One more question: As you probably seen - Jan had a complains about SCI >>>> term. He said SCI is ambiguous with ACPI's System >>>> Control Interrupt. >>> >>> I see his point. As a term I see "SCMI" often and sometimes "SCPI" but >>> "SCI" is the first time I saw it with this patch series. >>> >>> >>>> I think of using SC (as System Control) instead. What do you think >>>> about it? >>> >>> Yeah, I am not great at naming things but maybe "ARM_SCI"? "SC" alone >>> doesn't give me enough context to guess what it is. >> >> I might be missing some context. Why are naming everything SCI rather than >> SMCI? > > Because we're expecting other interfaces and transport to be > implemented, such as for example: > scmi_mailbox, scpi_smc, scpi_mailbox, ti_sci_smc etc. Oh, now that explain why there is a layer of indirection in Xen. It wasn't very clear from the cover letter why it was present. > >> >>> >>> Or we could broaden the scope and call it "firmware_interface"? >> How would this be used? Will it be a list of interface that will be exposed >> to the guest? >> > > The idea is to set mediator type for each Domain, so for example Xen can > use scmi_mailbox to communicate with SCP, Dom0 and DomD are also using > scmi_mailbox, but DomU using scmi_smc mediator because we have only 3 > mailboxes in system. This is not implemented yet, right now, we are > introducing only scmi_smc support. In future, multiple mediator support > can be added to Xen. Ok. So will there be only one interface at the time for a given domain. Is that correct? Cheers,
On 20/12/2021 16:41, Oleksii Moisieiev wrote: > Hi Julien, Hello, > On Fri, Dec 17, 2021 at 04:38:31PM +0000, Julien Grall wrote: >> >> >> On 17/12/2021 13:58, Oleksii Moisieiev wrote: >>> Hi Julien, >> >> Hi, >> >>> On Fri, Dec 17, 2021 at 01:37:35PM +0000, Julien Grall wrote: >>>> Hi, >>>> >>>> On 17/12/2021 13:23, Oleksii Moisieiev wrote: >>>>>>> +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) >>>>>>> +{ >>>>>>> + return iomem_permit_access(d, paddr_to_pfn(addr), >>>>>>> + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); >>>>>>> +} >>>>>>> + >>>>>>> +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, >>>>>>> + uint64_t len) >>>>>>> +{ >>>>>>> + return iomem_deny_access(d, paddr_to_pfn(addr), >>>>>>> + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); >>>>>>> +} >>>>>> >>>>>> I wonder, why we need an extra level of indirection here. And if this is >>>>>> really needed, I wonder why map(unmap)_memory* name was chosen, as there is >>>>>> no memory mapping/unmapping really happens here. >>>>>> >>>>> >>>>> I've added extra indirection to hide math like >>>>> paddr_to_pfn(PAGE_ALIGN(addr + len -1) >>>>> so you don't have to math in each call. unmap_memory_from_domain called >>>>> from 2 places, so I moved both calls to separate function. >>>>> Although, I agree that map/unmap is not perfect name. I consider >>>>> renaming it to mem_permit_acces and mam_deny_access. >>>> >>>> I haven't looked at the rest of the series. But this discussion caught my >>>> eye. This code implies that the address is page-aligned but the length not. >>>> Is that intended? >>>> >>>> That said, if you give permission to the domain on a full page then it means >>>> it may be able to access address it should not. Can you explain why this is >>>> fine? >>>> >>> >>> The idea was that xen receives some memory from the dt_node linux,scmi_mem, >>> then we split memory between the agents, so each agent get 1 page (we >>> allocate 0x10 pages right now). >> >> Thanks for the clarification. Does this imply the guest will be able to >> write message directly to the firmware? > > We used DEN0056C Specification as base. Available on: https://developer.arm.com/documentation/den0056/latest. > SCMI transport is described in Section 5.1. We implemented Shared Memory transport. > Firmware has N pages of the shared memory, used to communicate with Agents. > It allocates N agents and assign a page for each agent, such as: > ------------------------------------- > | Agent H | Agent 1 | Agent 2 | ... | > ------------------------------------- > Agent H is the privilleged Hypervisor agent, which is used to do the base commands, > such as getting Agent list, set\unset permissions etc. > Hypervisor assign agent to the guest and maps page, related to the agent to the Guest. > So the Guest, which is Agent 1 will get an access to Agent 1 page. > > Guest places SCMI message to Agent 1 memory, then sends SMC message. > Hypervisor process SMC request, add agent id to the message parameters and redirects it to the Firmware. > Based on the agent_id Firmware knows which page it should read. > Then after permission check ( if the resetId/clockID/powerID etc from message > is assigned to agent_id ) it does changes to the HW and places response to Agent > shared memory and marks channel as FREE ( by setting free bit in shared memory ). > Once channel is marked as free - Guest read response from the shared memory. So, IIUC, the hypervisor will not control what is written in the shared memory. It will only control the SMC parameters. Is my understanding correct? > > Non-virtualized systems will work as well. OS should send SMC directly to the Firmware. > >> >> If so, this brings a few more questions: >> 1) What will the guest write in it? Can it contains addresses? > Guest can write scmi request to the shared memory, which include the following data: > 1) protocol_id - which protocol is requested, such as clock, power, reset etc > 2) message_id - action that should be done to HW, such as do_reset or get_clock > 3) message data - which includes reset_id/clock_id/power_id etc. that should be changed. > Reset IDs and Clock IDs are assigned in Firmware. Guest receives ID, corresponding to the device from the device-tree. > dt_node as an example: > &avb { > scmi_devid = <0>; > clocks = <&scmi_clock 0>; > power-domains = <&scmi_power 0>; > resets = <&scmi_reset 0>; > }; > >> 2) What are the expected memory attribute for the regions? > > xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. I think you misunderstood my comment. Memory can be mapped with various type (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will the firmware expect? What will the guest OS usually? The reason I am asking is the attributes have to matched to avoid any coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen will configure the stage-2 to use Device nGnRnE. As the result, the result memory access will be Device nGnRnE which is very strict. Cheers,
On Fri, Dec 24, 2021 at 03:28:56PM +0100, Julien Grall wrote: > > > On 24/12/2021 14:59, Oleksii Moisieiev wrote: > > Hi Julien, > > Hello, > > > On Fri, Dec 24, 2021 at 02:29:13PM +0100, Julien Grall wrote: > > > Hi, > > > > > > On 24/12/2021 01:16, Stefano Stabellini wrote: > > > > > One more question: As you probably seen - Jan had a complains about SCI > > > > > term. He said SCI is ambiguous with ACPI's System > > > > > Control Interrupt. > > > > > > > > I see his point. As a term I see "SCMI" often and sometimes "SCPI" but > > > > "SCI" is the first time I saw it with this patch series. > > > > > > > > > > > > > I think of using SC (as System Control) instead. What do you think > > > > > about it? > > > > > > > > Yeah, I am not great at naming things but maybe "ARM_SCI"? "SC" alone > > > > doesn't give me enough context to guess what it is. > > > > > > I might be missing some context. Why are naming everything SCI rather than > > > SMCI? > > > > Because we're expecting other interfaces and transport to be > > implemented, such as for example: > > scmi_mailbox, scpi_smc, scpi_mailbox, ti_sci_smc etc. > > Oh, now that explain why there is a layer of indirection in Xen. It wasn't > very clear from the cover letter why it was present. > Please see below. > > > > > > > > > > > > > Or we could broaden the scope and call it "firmware_interface"? > > > How would this be used? Will it be a list of interface that will be exposed > > > to the guest? > > > > > > > The idea is to set mediator type for each Domain, so for example Xen can > > use scmi_mailbox to communicate with SCP, Dom0 and DomD are also using > > scmi_mailbox, but DomU using scmi_smc mediator because we have only 3 > > mailboxes in system. This is not implemented yet, right now, we are > > introducing only scmi_smc support. In future, multiple mediator support > > can be added to Xen. > > Ok. So will there be only one interface at the time for a given domain. Is > that correct? > Correct. The idea is that we provice only one interface to the Domain, so different domains can use different protocols and transport. Those interfaces can be different than the interface Xen uses to connect to SCP. This allows us to vary the configuration. So for example: Let's take system, that support only 2 mailboxes and communication with SCP can use only mailboxes as transport. We intent to use scmi protocol to manage HW. In this case we use 2 mailboxes for Xen-> SCP communication, and for Dom0 -> Xen. Domu can be configured to use scmi_smc, so the communication should be the following: DomU --smc--> Xen -mailbox--> SCP Firmware. Let's say we want to add DomainX with OS XXX, which using yyy protocol with zzz transport. Then we can configure DomX wuth yyy_zzz mediator, so the communication will be the following: DomX --yyy--> Xen -mailbox--> SCP Firmware Where Xen knows how to convert message from yyy protocol to scmi protocol. I considered the alternative way, when we can configure domain with several mediators, so each Domain can be configured to use, for example, scmi_smc for power-domains and scpi_smc for clocks and resets. But I don't see real use-cases for this configuration. What do you think about that? Best regards, Oleksii
On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: > On 20/12/2021 16:41, Oleksii Moisieiev wrote: > > Hi Julien, > > Hello, > > > On Fri, Dec 17, 2021 at 04:38:31PM +0000, Julien Grall wrote: > > > > > > > > > On 17/12/2021 13:58, Oleksii Moisieiev wrote: > > > > Hi Julien, > > > > > > Hi, > > > > > > > On Fri, Dec 17, 2021 at 01:37:35PM +0000, Julien Grall wrote: > > > > > Hi, > > > > > > > > > > On 17/12/2021 13:23, Oleksii Moisieiev wrote: > > > > > > > > +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) > > > > > > > > +{ > > > > > > > > + return iomem_permit_access(d, paddr_to_pfn(addr), > > > > > > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > > > > > > +} > > > > > > > > + > > > > > > > > +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, > > > > > > > > + uint64_t len) > > > > > > > > +{ > > > > > > > > + return iomem_deny_access(d, paddr_to_pfn(addr), > > > > > > > > + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); > > > > > > > > +} > > > > > > > > > > > > > > I wonder, why we need an extra level of indirection here. And if this is > > > > > > > really needed, I wonder why map(unmap)_memory* name was chosen, as there is > > > > > > > no memory mapping/unmapping really happens here. > > > > > > > > > > > > > > > > > > > I've added extra indirection to hide math like > > > > > > paddr_to_pfn(PAGE_ALIGN(addr + len -1) > > > > > > so you don't have to math in each call. unmap_memory_from_domain called > > > > > > from 2 places, so I moved both calls to separate function. > > > > > > Although, I agree that map/unmap is not perfect name. I consider > > > > > > renaming it to mem_permit_acces and mam_deny_access. > > > > > > > > > > I haven't looked at the rest of the series. But this discussion caught my > > > > > eye. This code implies that the address is page-aligned but the length not. > > > > > Is that intended? > > > > > > > > > > That said, if you give permission to the domain on a full page then it means > > > > > it may be able to access address it should not. Can you explain why this is > > > > > fine? > > > > > > > > > > > > > The idea was that xen receives some memory from the dt_node linux,scmi_mem, > > > > then we split memory between the agents, so each agent get 1 page (we > > > > allocate 0x10 pages right now). > > > > > > Thanks for the clarification. Does this imply the guest will be able to > > > write message directly to the firmware? > > > > We used DEN0056C Specification as base. Available on: https://urldefense.com/v3/__https://developer.arm.com/documentation/den0056/latest__;!!GF_29dbcQIUBPA!m9pWoxBEjb8Sd1CoV5cpU8MbmLCjohYQxv2ci9tDvMmZ9oCEitqyydZ3rQWXCM5bxvIn$ [developer[.]arm[.]com]. > > SCMI transport is described in Section 5.1. We implemented Shared Memory transport. > > Firmware has N pages of the shared memory, used to communicate with Agents. > > It allocates N agents and assign a page for each agent, such as: > > ------------------------------------- > > | Agent H | Agent 1 | Agent 2 | ... | > > ------------------------------------- > > Agent H is the privilleged Hypervisor agent, which is used to do the base commands, > > such as getting Agent list, set\unset permissions etc. > > Hypervisor assign agent to the guest and maps page, related to the agent to the Guest. > > So the Guest, which is Agent 1 will get an access to Agent 1 page. > > > > Guest places SCMI message to Agent 1 memory, then sends SMC message. > > Hypervisor process SMC request, add agent id to the message parameters and redirects it to the Firmware. > > Based on the agent_id Firmware knows which page it should read. > > Then after permission check ( if the resetId/clockID/powerID etc from message > > is assigned to agent_id ) it does changes to the HW and places response to Agent > > shared memory and marks channel as FREE ( by setting free bit in shared memory ). > > Once channel is marked as free - Guest read response from the shared memory. > > So, IIUC, the hypervisor will not control what is written in the shared > memory. It will only control the SMC parameters. Is my understanding > correct? > For scmi_smc it will not. But potentially it can make some changes, or convert to the different protocol. > > > > Non-virtualized systems will work as well. OS should send SMC directly to the Firmware. > > > > > > > > If so, this brings a few more questions: > > > 1) What will the guest write in it? Can it contains addresses? > > Guest can write scmi request to the shared memory, which include the following data: > > 1) protocol_id - which protocol is requested, such as clock, power, reset etc > > 2) message_id - action that should be done to HW, such as do_reset or get_clock > > 3) message data - which includes reset_id/clock_id/power_id etc. that should be changed. > > Reset IDs and Clock IDs are assigned in Firmware. Guest receives ID, corresponding to the device from the device-tree. > > dt_node as an example: > > &avb { > > scmi_devid = <0>; > > clocks = <&scmi_clock 0>; > > power-domains = <&scmi_power 0>; > > resets = <&scmi_reset 0>; > > }; > > > > > 2) What are the expected memory attribute for the regions? > > > > xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. > > I think you misunderstood my comment. Memory can be mapped with various type > (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will > the firmware expect? What will the guest OS usually? > > The reason I am asking is the attributes have to matched to avoid any > coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen > will configure the stage-2 to use Device nGnRnE. As the result, the result > memory access will be Device nGnRnE which is very strict. > Let me share with you the configuration example: scmi expects memory to be configured in the device-tree: cpu_scp_shm: scp-shmem@0xXXXXXXX { compatible = "arm,scmi-shmem"; reg = <0x0 0xXXXXXX 0x0 0x1000>; }; where XXXXXX address I allocate in alloc_magic_pages function. Then I get paddr of the scmi channel for this domain and use XEN_DOMCTL_memory_mapping to map scmi channel address to gfn. Hope I wass able to answer your question. Best regards, Oleksii.
Hi, On 24/12/2021 17:02, Oleksii Moisieiev wrote: > On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: >> On 20/12/2021 16:41, Oleksii Moisieiev wrote: >>>> 2) What are the expected memory attribute for the regions? >>> >>> xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. >> >> I think you misunderstood my comment. Memory can be mapped with various type >> (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will >> the firmware expect? What will the guest OS usually? >> >> The reason I am asking is the attributes have to matched to avoid any >> coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen >> will configure the stage-2 to use Device nGnRnE. As the result, the result >> memory access will be Device nGnRnE which is very strict. >> > > Let me share with you the configuration example: > scmi expects memory to be configured in the device-tree: > > cpu_scp_shm: scp-shmem@0xXXXXXXX { > compatible = "arm,scmi-shmem"; > reg = <0x0 0xXXXXXX 0x0 0x1000>; > }; > > where XXXXXX address I allocate in alloc_magic_pages function. The goal of alloc_magic_pages() is to allocate RAM. However, what you want is a guest physical address and then map a part of the shared page. I can see two options here: 1) Hardcode the SCMI region in the memory map 2) Create a new region in the memory map that can be used for reserving memory for mapping. We still have plenty of space in the guest memory map. So the former is probably going to be fine for now. > Then I get paddr of the scmi channel for this domain and use > XEN_DOMCTL_memory_mapping to map scmi channel address to gfn. > > Hope I wass able to answer your question. What you provided me is how the guest OS will locate the shared memory. This still doesn't tell me which memory attribute will be used to map the page in Stage-1 (guest page-tables). To find that out, you want to look at the driver and how the mapping is done. The Linux driver (drivers/firmware/arm_scmi) is using devm_ioremap() (see smc_chan_setup()). Under the hood, the function devm_ioremap() is using PROT_DEVICE_nGnRE (arm64) which is one of the most restrictive memory attribute. This means the firmware should be able to deal with the most restrictive attribute and therefore using XEN_DOMCTL_memory_mapping to map the shared page in stage-2 should be fine. Cheers,
Hi, On 24/12/2021 16:49, Oleksii Moisieiev wrote: > On Fri, Dec 24, 2021 at 03:28:56PM +0100, Julien Grall wrote: >> On 24/12/2021 14:59, Oleksii Moisieiev wrote: >>> Hi Julien, >> >> Hello, >> >>> On Fri, Dec 24, 2021 at 02:29:13PM +0100, Julien Grall wrote: >>>> Hi, >>>> >>>> On 24/12/2021 01:16, Stefano Stabellini wrote: >>>>>> One more question: As you probably seen - Jan had a complains about SCI >>>>>> term. He said SCI is ambiguous with ACPI's System >>>>>> Control Interrupt. >>>>> >>>>> I see his point. As a term I see "SCMI" often and sometimes "SCPI" but >>>>> "SCI" is the first time I saw it with this patch series. >>>>> >>>>> >>>>>> I think of using SC (as System Control) instead. What do you think >>>>>> about it? >>>>> >>>>> Yeah, I am not great at naming things but maybe "ARM_SCI"? "SC" alone >>>>> doesn't give me enough context to guess what it is. >>>> >>>> I might be missing some context. Why are naming everything SCI rather than >>>> SMCI? >>> >>> Because we're expecting other interfaces and transport to be >>> implemented, such as for example: >>> scmi_mailbox, scpi_smc, scpi_mailbox, ti_sci_smc etc. >> >> Oh, now that explain why there is a layer of indirection in Xen. It wasn't >> very clear from the cover letter why it was present. >> > Please see below. >>> >>>> >>>>> >>>>> Or we could broaden the scope and call it "firmware_interface"? >>>> How would this be used? Will it be a list of interface that will be exposed >>>> to the guest? >>>> >>> >>> The idea is to set mediator type for each Domain, so for example Xen can >>> use scmi_mailbox to communicate with SCP, Dom0 and DomD are also using >>> scmi_mailbox, but DomU using scmi_smc mediator because we have only 3 >>> mailboxes in system. This is not implemented yet, right now, we are >>> introducing only scmi_smc support. In future, multiple mediator support >>> can be added to Xen. >> >> Ok. So will there be only one interface at the time for a given domain. Is >> that correct? >> > Correct. The idea is that we provice only one interface to the Domain, > so different domains can use different protocols and transport I think this is similar to TEE mediator. Specifying in the guest configuration the TEE protocol (e.g. OP-TEE) allows to sanity check what the guest wants match the host. However, I am not convinced there is a need to expose different protocols for domains running on the same platform. > Those > interfaces can be different than the interface Xen uses to connect to SCP. I can understand how this looks appealing. However, it may not always be possible to convert the guest protocol to the host protocol. And even if it is possible, it is increasing the complexity and risk. > This allows us to vary the configuration. > So for example: > Let's take system, that support only 2 mailboxes and communication with > SCP can use only mailboxes as transport. We intent to use scmi protocol > to manage HW. In this case we use 2 mailboxes for Xen-> SCP > communication, and for Dom0 -> Xen. > Domu can be configured to use > scmi_smc, so the communication should be the following: > DomU --smc--> Xen -mailbox--> SCP Firmware. > Let's say we want to add DomainX with OS XXX, which using yyy protocol > with zzz transport. Then we can configure DomX wuth yyy_zzz mediator, so > the communication will be the following: > DomX --yyy--> Xen -mailbox--> SCP Firmware I am a bit confused with your example here. From my understanding, I would say 'smc' is the transport. But in your example above, you suggest this is the protocol. What did I miss? > Where Xen knows how to convert message from yyy protocol to scmi protocol As I wrote above, when possible, converting protocol could be complex and risky. So do you have a concrete use case for that? That said, changing the transport (e.g. smc <-> mailbox) would make more sense to me. > > I considered the alternative way, when we can configure domain with > several mediators, so each Domain can be configured to use, for example, > scmi_smc for power-domains and scpi_smc for clocks and resets. But I > don't see real use-cases for this configuration. Do you mean a platform using both or exposing both to the guest? > > What do you think about that? So I am a bit unsure how "firmware_interface" would be specified. IMO, the user should only specificy which interface the guest will use (e.g. SCMI via SMC) even if the host end up to use a different transport (e.g. SCMI via mailbox). IOW, the option would not tell how to convert it. Is it what you had in mind? Cheers,
Hi Julien, On Mon, Jan 03, 2022 at 01:14:17PM +0000, Julien Grall wrote: > Hi, > > On 24/12/2021 17:02, Oleksii Moisieiev wrote: > > On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: > > > On 20/12/2021 16:41, Oleksii Moisieiev wrote: > > > > > 2) What are the expected memory attribute for the regions? > > > > > > > > xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. > > > > > > I think you misunderstood my comment. Memory can be mapped with various type > > > (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will > > > the firmware expect? What will the guest OS usually? > > > > > > The reason I am asking is the attributes have to matched to avoid any > > > coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen > > > will configure the stage-2 to use Device nGnRnE. As the result, the result > > > memory access will be Device nGnRnE which is very strict. > > > > > > > Let me share with you the configuration example: > > scmi expects memory to be configured in the device-tree: > > > > cpu_scp_shm: scp-shmem@0xXXXXXXX { > > compatible = "arm,scmi-shmem"; > > reg = <0x0 0xXXXXXX 0x0 0x1000>; > > }; > > > > where XXXXXX address I allocate in alloc_magic_pages function. > > The goal of alloc_magic_pages() is to allocate RAM. However, what you want > is a guest physical address and then map a part of the shared page. Do you mean that I can't use alloc_magic_pages to allocate shared memory region for SCMI? > > I can see two options here: > 1) Hardcode the SCMI region in the memory map > 2) Create a new region in the memory map that can be used for reserving > memory for mapping. Could you please explain what do you mean under the "new region in the memory map"? > > We still have plenty of space in the guest memory map. So the former is > probably going to be fine for now. > > > Then I get paddr of the scmi channel for this domain and use > > XEN_DOMCTL_memory_mapping to map scmi channel address to gfn. > > > Hope I wass able to answer your question. > > What you provided me is how the guest OS will locate the shared memory. This > still doesn't tell me which memory attribute will be used to map the page in > Stage-1 (guest page-tables). > > To find that out, you want to look at the driver and how the mapping is > done. The Linux driver (drivers/firmware/arm_scmi) is using devm_ioremap() > (see smc_chan_setup()). > > Under the hood, the function devm_ioremap() is using PROT_DEVICE_nGnRE > (arm64) which is one of the most restrictive memory attribute. > > This means the firmware should be able to deal with the most restrictive > attribute and therefore using XEN_DOMCTL_memory_mapping to map the shared > page in stage-2 should be fine. > I'm using vmap call to map channel memory (see smc_create_channel()). vmap call sets PAGE_HYPERVISOR flag which sets MT_NORMAL (0x7) flag. Considering that protocol is synchronous and only one agent per channel is expected - this works fine for now. But I agree that the same memory attributes should be used in xen and kernel. I fill fix that in v2. Best regards, Oleksii.
On 06/01/2022 13:53, Oleksii Moisieiev wrote: > Hi Julien, Hi, > > On Mon, Jan 03, 2022 at 01:14:17PM +0000, Julien Grall wrote: >> Hi, >> >> On 24/12/2021 17:02, Oleksii Moisieiev wrote: >>> On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: >>>> On 20/12/2021 16:41, Oleksii Moisieiev wrote: >>>>>> 2) What are the expected memory attribute for the regions? >>>>> >>>>> xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. >>>> >>>> I think you misunderstood my comment. Memory can be mapped with various type >>>> (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will >>>> the firmware expect? What will the guest OS usually? >>>> >>>> The reason I am asking is the attributes have to matched to avoid any >>>> coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen >>>> will configure the stage-2 to use Device nGnRnE. As the result, the result >>>> memory access will be Device nGnRnE which is very strict. >>>> >>> >>> Let me share with you the configuration example: >>> scmi expects memory to be configured in the device-tree: >>> >>> cpu_scp_shm: scp-shmem@0xXXXXXXX { >>> compatible = "arm,scmi-shmem"; >>> reg = <0x0 0xXXXXXX 0x0 0x1000>; >>> }; >>> >>> where XXXXXX address I allocate in alloc_magic_pages function. >> >> The goal of alloc_magic_pages() is to allocate RAM. However, what you want >> is a guest physical address and then map a part of the shared page. > > Do you mean that I can't use alloc_magic_pages to allocate shared > memory region for SCMI? Correct. alloc_magic_pages() will allocate a RAM page and then assign to the guest. From your description, this is not what you want because you will call XEN_DOMCTL_memory_mapping (and therefore replace the mapping). > >> >> I can see two options here: >> 1) Hardcode the SCMI region in the memory map >> 2) Create a new region in the memory map that can be used for reserving >> memory for mapping. > > Could you please explain what do you mean under the "new region in the > memory map"? I mean reserving some guest physical address that could be used for map host physical address (e.g. SCMI region, GIC CPU interface...). So rather than hardcoding the address, we have something more flexible. > >> >> We still have plenty of space in the guest memory map. So the former is >> probably going to be fine for now. >> >>> Then I get paddr of the scmi channel for this domain and use >>> XEN_DOMCTL_memory_mapping to map scmi channel address to gfn. >>> > Hope I wass able to answer your question. >> >> What you provided me is how the guest OS will locate the shared memory. This >> still doesn't tell me which memory attribute will be used to map the page in >> Stage-1 (guest page-tables). >> >> To find that out, you want to look at the driver and how the mapping is >> done. The Linux driver (drivers/firmware/arm_scmi) is using devm_ioremap() >> (see smc_chan_setup()). >> >> Under the hood, the function devm_ioremap() is using PROT_DEVICE_nGnRE >> (arm64) which is one of the most restrictive memory attribute. >> >> This means the firmware should be able to deal with the most restrictive >> attribute and therefore using XEN_DOMCTL_memory_mapping to map the shared >> page in stage-2 should be fine. >> > > I'm using vmap call to map channel memory (see smc_create_channel()). > vmap call sets PAGE_HYPERVISOR flag which sets MT_NORMAL (0x7) flag. You want to use ioremap(). > Considering that protocol is synchronous and only one agent per channel is > expected - this works fine for now. > But I agree that the same memory attributes should be used in xen and > kernel. I fill fix that in v2. I am a bit confused. Are you mapping the full shared memory area in Xen? If yes, why do you need to map the memory that is going to be shared with a domain? Cheers,
On Mon, Jan 03, 2022 at 02:23:01PM +0000, Julien Grall wrote: > Hi, > > On 24/12/2021 16:49, Oleksii Moisieiev wrote: > > On Fri, Dec 24, 2021 at 03:28:56PM +0100, Julien Grall wrote: > > > On 24/12/2021 14:59, Oleksii Moisieiev wrote: > > > > Hi Julien, > > > > > > Hello, > > > > > > > On Fri, Dec 24, 2021 at 02:29:13PM +0100, Julien Grall wrote: > > > > > Hi, > > > > > > > > > > On 24/12/2021 01:16, Stefano Stabellini wrote: > > > > > > > One more question: As you probably seen - Jan had a complains about SCI > > > > > > > term. He said SCI is ambiguous with ACPI's System > > > > > > > Control Interrupt. > > > > > > > > > > > > I see his point. As a term I see "SCMI" often and sometimes "SCPI" but > > > > > > "SCI" is the first time I saw it with this patch series. > > > > > > > > > > > > > > > > > > > I think of using SC (as System Control) instead. What do you think > > > > > > > about it? > > > > > > > > > > > > Yeah, I am not great at naming things but maybe "ARM_SCI"? "SC" alone > > > > > > doesn't give me enough context to guess what it is. > > > > > > > > > > I might be missing some context. Why are naming everything SCI rather than > > > > > SMCI? > > > > > > > > Because we're expecting other interfaces and transport to be > > > > implemented, such as for example: > > > > scmi_mailbox, scpi_smc, scpi_mailbox, ti_sci_smc etc. > > > > > > Oh, now that explain why there is a layer of indirection in Xen. It wasn't > > > very clear from the cover letter why it was present. > > > > > Please see below. > > > > > > > > > > > > > > > > > > > > > Or we could broaden the scope and call it "firmware_interface"? > > > > > How would this be used? Will it be a list of interface that will be exposed > > > > > to the guest? > > > > > > > > > > > > > The idea is to set mediator type for each Domain, so for example Xen can > > > > use scmi_mailbox to communicate with SCP, Dom0 and DomD are also using > > > > scmi_mailbox, but DomU using scmi_smc mediator because we have only 3 > > > > mailboxes in system. This is not implemented yet, right now, we are > > > > introducing only scmi_smc support. In future, multiple mediator support > > > > can be added to Xen. > > > > > > Ok. So will there be only one interface at the time for a given domain. Is > > > that correct? > > > > > Correct. The idea is that we provice only one interface to the Domain, > > so different domains can use different protocols and transport > > I think this is similar to TEE mediator. Specifying in the guest > configuration the TEE protocol (e.g. OP-TEE) allows to sanity check what the > guest wants match the host. However, I am not convinced there is a need to > expose different protocols for domains running on the same platform. > Yes. You're right, I based on TEE mediator during implementation. Please see below my thoughts about different protocols and transports. > > Those > > interfaces can be different than the interface Xen uses to connect to SCP. > > I can understand how this looks appealing. However, it may not always be > possible to convert the guest protocol to the host protocol. And even if it > is possible, it is increasing the complexity and risk. > > > This allows us to vary the configuration. > > So for example: > > Let's take system, that support only 2 mailboxes and communication with > > SCP can use only mailboxes as transport. We intent to use scmi protocol > > to manage HW. In this case we use 2 mailboxes for Xen-> SCP > > communication, and for Dom0 -> Xen. > > Domu can be configured to use > > scmi_smc, so the communication should be the following: > > DomU --smc--> Xen -mailbox--> SCP Firmware. > > Let's say we want to add DomainX with OS XXX, which using yyy protocol > > with zzz transport. Then we can configure DomX wuth yyy_zzz mediator, so > > the communication will be the following: > > DomX --yyy--> Xen -mailbox--> SCP Firmware > > I am a bit confused with your example here. From my understanding, I would > say 'smc' is the transport. But in your example above, you suggest this is > the protocol. What did I miss? Sorry, that's my fault. I merged 2 exapmles in 1, in which the first example shows different transports and the second one shows different protocols. The first expample describes the use-case when we have firmware, using mailboxes as transport, but the system we have has number of Domains > than number of the mailboxes. Let's say Firmware and all Domains are using scmi protocol. In this case I see 2 possible configurations: 1) Firmware and some Domains are using mailbox as transport, other Domains, when all mailboxes are occupied, are using smc as transport. Xen will be responsible to receive smc message and send request to Firmware using mailbox and vice versa. 2) Firmware using mailbox to communicate with Xen, all Domains are using smc. In this case only one mailbox will be used. Number 2 looks better from my standpoint. In that case we have scmi_smc mediator, which exposes smc transport to domains and uses mailbox to communicate with Firmware. The second part is that we can change protocol when it's simple and safe. But now I see that this is not a good Idea to make Xen responsible for protocol converting. > > > Where Xen knows how to convert message from yyy protocol to scmi protocol > > As I wrote above, when possible, converting protocol could be complex and > risky. So do you have a concrete use case for that? > > That said, changing the transport (e.g. smc <-> mailbox) would make more > sense to me. > I think you're right. > > > > I considered the alternative way, when we can configure domain with > > several mediators, so each Domain can be configured to use, for example, > > scmi_smc for power-domains and scpi_smc for clocks and resets. But I > > don't see real use-cases for this configuration. > > Do you mean a platform using both or exposing both to the guest? > Yes, that's the alternative way I see. Similar to linux kernel, where you are allowed to use for example scmi for clocks and scpi for resets. I don't see the real use-cases for this config, but Xen can follow Linux kernel configuration approach and allow to register 2 mediators: one for scmi_smc and one for scpi_smc. In this case Domain configuration will look like this: sci = [ "scmi_smc", "scpi_smc", ]; So both scmi_smc and scpi_smc mediators are exposing protocols to the domains and communicating with Firmware. Current implementation supports only 1 mediator, but in future I think we should consider the posibility of having several mediators. Where each mediator has it's own channel (transport+protocol) to Firmware and exposes transport+protocol to the guests. > > > > What do you think about that? > So I am a bit unsure how "firmware_interface" would be specified. IMO, the > user should only specificy which interface the guest will use (e.g. SCMI via > SMC) even if the host end up to use a different transport (e.g. SCMI via > mailbox). IOW, the option would not tell how to convert it. > > Is it what you had in mind? > Yes. That's what I mean. firmware_interface can be specified in xen Device-tree. Different SCI mediators can be implemented and added to the device-tree. Each mediator has channel (protocol + transports) to the Firmware and exposes protocol + transport to the guests. Protocol to the Firmware should be the same as exposed protocol, transport may differ and is implementation specific. Best regards, Oleksii.
On Thu, Jan 06, 2022 at 02:02:10PM +0000, Julien Grall wrote: > > > On 06/01/2022 13:53, Oleksii Moisieiev wrote: > > Hi Julien, > > Hi, > > > > > On Mon, Jan 03, 2022 at 01:14:17PM +0000, Julien Grall wrote: > > > Hi, > > > > > > On 24/12/2021 17:02, Oleksii Moisieiev wrote: > > > > On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: > > > > > On 20/12/2021 16:41, Oleksii Moisieiev wrote: > > > > > > > 2) What are the expected memory attribute for the regions? > > > > > > > > > > > > xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. > > > > > > > > > > I think you misunderstood my comment. Memory can be mapped with various type > > > > > (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will > > > > > the firmware expect? What will the guest OS usually? > > > > > > > > > > The reason I am asking is the attributes have to matched to avoid any > > > > > coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen > > > > > will configure the stage-2 to use Device nGnRnE. As the result, the result > > > > > memory access will be Device nGnRnE which is very strict. > > > > > > > > > > > > > Let me share with you the configuration example: > > > > scmi expects memory to be configured in the device-tree: > > > > > > > > cpu_scp_shm: scp-shmem@0xXXXXXXX { > > > > compatible = "arm,scmi-shmem"; > > > > reg = <0x0 0xXXXXXX 0x0 0x1000>; > > > > }; > > > > > > > > where XXXXXX address I allocate in alloc_magic_pages function. > > > > > > The goal of alloc_magic_pages() is to allocate RAM. However, what you want > > > is a guest physical address and then map a part of the shared page. > > > > Do you mean that I can't use alloc_magic_pages to allocate shared > > memory region for SCMI? > Correct. alloc_magic_pages() will allocate a RAM page and then assign to the > guest. From your description, this is not what you want because you will > call XEN_DOMCTL_memory_mapping (and therefore replace the mapping). > Ok thanks, I will refactor this part in v2. > > > > > > > > I can see two options here: > > > 1) Hardcode the SCMI region in the memory map > > > 2) Create a new region in the memory map that can be used for reserving > > > memory for mapping. > > > > Could you please explain what do you mean under the "new region in the > > memory map"? > > I mean reserving some guest physical address that could be used for map host > physical address (e.g. SCMI region, GIC CPU interface...). > > So rather than hardcoding the address, we have something more flexible. > Ok, I will fix that in v2. > > > > > > > > We still have plenty of space in the guest memory map. So the former is > > > probably going to be fine for now. > > > > > > > Then I get paddr of the scmi channel for this domain and use > > > > XEN_DOMCTL_memory_mapping to map scmi channel address to gfn. > > > > > Hope I wass able to answer your question. > > > > > > What you provided me is how the guest OS will locate the shared memory. This > > > still doesn't tell me which memory attribute will be used to map the page in > > > Stage-1 (guest page-tables). > > > > > > To find that out, you want to look at the driver and how the mapping is > > > done. The Linux driver (drivers/firmware/arm_scmi) is using devm_ioremap() > > > (see smc_chan_setup()). > > > > > > Under the hood, the function devm_ioremap() is using PROT_DEVICE_nGnRE > > > (arm64) which is one of the most restrictive memory attribute. > > > > > > This means the firmware should be able to deal with the most restrictive > > > attribute and therefore using XEN_DOMCTL_memory_mapping to map the shared > > > page in stage-2 should be fine. > > > > > > > I'm using vmap call to map channel memory (see smc_create_channel()). > > vmap call sets PAGE_HYPERVISOR flag which sets MT_NORMAL (0x7) flag. > > You want to use ioremap(). > I've used ioremap originally, but changed it to vmap because ioremap doesn't support memcpy. What if I use __vmap with MT_DEVICE_nGnRE flag? > > Considering that protocol is synchronous and only one agent per channel is > > expected - this works fine for now. > > But I agree that the same memory attributes should be used in xen and > > kernel. I fill fix that in v2. > > I am a bit confused. Are you mapping the full shared memory area in Xen? If > yes, why do you need to map the memory that is going to be shared with a > domain? > Xen should have an access to all agent channels because it should send SCMI_BASE_DISCOVER_AGENT to each channel and receive agent_id during scmi_probe call. Best regards, Oleksii
Hi, On 06/01/2022 15:43, Oleksii Moisieiev wrote: > On Thu, Jan 06, 2022 at 02:02:10PM +0000, Julien Grall wrote: >> >> >> On 06/01/2022 13:53, Oleksii Moisieiev wrote: >>> Hi Julien, >> >> Hi, >> >>> >>> On Mon, Jan 03, 2022 at 01:14:17PM +0000, Julien Grall wrote: >>>> Hi, >>>> >>>> On 24/12/2021 17:02, Oleksii Moisieiev wrote: >>>>> On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: >>>>>> On 20/12/2021 16:41, Oleksii Moisieiev wrote: >>>>>>>> 2) What are the expected memory attribute for the regions? >>>>>>> >>>>>>> xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. >>>>>> >>>>>> I think you misunderstood my comment. Memory can be mapped with various type >>>>>> (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will >>>>>> the firmware expect? What will the guest OS usually? >>>>>> >>>>>> The reason I am asking is the attributes have to matched to avoid any >>>>>> coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen >>>>>> will configure the stage-2 to use Device nGnRnE. As the result, the result >>>>>> memory access will be Device nGnRnE which is very strict. >>>>>> >>>>> >>>>> Let me share with you the configuration example: >>>>> scmi expects memory to be configured in the device-tree: >>>>> >>>>> cpu_scp_shm: scp-shmem@0xXXXXXXX { >>>>> compatible = "arm,scmi-shmem"; >>>>> reg = <0x0 0xXXXXXX 0x0 0x1000>; >>>>> }; >>>>> >>>>> where XXXXXX address I allocate in alloc_magic_pages function. >>>> >>>> The goal of alloc_magic_pages() is to allocate RAM. However, what you want >>>> is a guest physical address and then map a part of the shared page. >>> >>> Do you mean that I can't use alloc_magic_pages to allocate shared >>> memory region for SCMI? >> Correct. alloc_magic_pages() will allocate a RAM page and then assign to the >> guest. From your description, this is not what you want because you will >> call XEN_DOMCTL_memory_mapping (and therefore replace the mapping). >> > > Ok thanks, I will refactor this part in v2. > >>> >>>> >>>> I can see two options here: >>>> 1) Hardcode the SCMI region in the memory map >>>> 2) Create a new region in the memory map that can be used for reserving >>>> memory for mapping. >>> >>> Could you please explain what do you mean under the "new region in the >>> memory map"? >> >> I mean reserving some guest physical address that could be used for map host >> physical address (e.g. SCMI region, GIC CPU interface...). >> >> So rather than hardcoding the address, we have something more flexible. >> > > Ok, I will fix that in v2. Just for avoidance of doubt. I was clarify option 2 and not requesting to implement. That said, if you want to implement option 2 I would be happy to review it. > >>> >>>> >>>> We still have plenty of space in the guest memory map. So the former is >>>> probably going to be fine for now. >>>> >>>>> Then I get paddr of the scmi channel for this domain and use >>>>> XEN_DOMCTL_memory_mapping to map scmi channel address to gfn. >>>>> > Hope I wass able to answer your question. >>>> >>>> What you provided me is how the guest OS will locate the shared memory. This >>>> still doesn't tell me which memory attribute will be used to map the page in >>>> Stage-1 (guest page-tables). >>>> >>>> To find that out, you want to look at the driver and how the mapping is >>>> done. The Linux driver (drivers/firmware/arm_scmi) is using devm_ioremap() >>>> (see smc_chan_setup()). >>>> >>>> Under the hood, the function devm_ioremap() is using PROT_DEVICE_nGnRE >>>> (arm64) which is one of the most restrictive memory attribute. >>>> >>>> This means the firmware should be able to deal with the most restrictive >>>> attribute and therefore using XEN_DOMCTL_memory_mapping to map the shared >>>> page in stage-2 should be fine. >>>> >>> >>> I'm using vmap call to map channel memory (see smc_create_channel()). >>> vmap call sets PAGE_HYPERVISOR flag which sets MT_NORMAL (0x7) flag. >> >> You want to use ioremap(). >> > > I've used ioremap originally, but changed it to vmap because ioremap > doesn't support memcpy. > What if I use __vmap with MT_DEVICE_nGnRE flag? That's not going to help. Our implementation of memcpy() is using unaligned access (which is forbidden on Device memory). You will need something similar to memcpy_toio() in Linux. I don't think we have one today in Xen, so I would suggest to import the implementation from Linux. > >>> Considering that protocol is synchronous and only one agent per channel is >>> expected - this works fine for now. >>> But I agree that the same memory attributes should be used in xen and >>> kernel. I fill fix that in v2. >> >> I am a bit confused. Are you mapping the full shared memory area in Xen? If >> yes, why do you need to map the memory that is going to be shared with a >> domain? >> > > Xen should have an access to all agent channels because it should send > SCMI_BASE_DISCOVER_AGENT to each channel and receive agent_id during > scmi_probe call. Hmmm... Just to confirm, this will only happen during Xen boot? IOW, Xen will never write to the channel when a domain is running? If yes, then I think it would be best to unmap the channel once they are used. This would prevent all sort of issues (e.g. Xen mistakenly written in them). Cheers,
Hi Julien, On Thu, Jan 06, 2022 at 04:04:34PM +0000, Julien Grall wrote: > Hi, > > On 06/01/2022 15:43, Oleksii Moisieiev wrote: > > On Thu, Jan 06, 2022 at 02:02:10PM +0000, Julien Grall wrote: > > > > > > > > > On 06/01/2022 13:53, Oleksii Moisieiev wrote: > > > > Hi Julien, > > > > > > Hi, > > > > > > > > > > > On Mon, Jan 03, 2022 at 01:14:17PM +0000, Julien Grall wrote: > > > > > Hi, > > > > > > > > > > On 24/12/2021 17:02, Oleksii Moisieiev wrote: > > > > > > On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: > > > > > > > On 20/12/2021 16:41, Oleksii Moisieiev wrote: > > > > > > > > > 2) What are the expected memory attribute for the regions? > > > > > > > > > > > > > > > > xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. > > > > > > > > > > > > > > I think you misunderstood my comment. Memory can be mapped with various type > > > > > > > (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will > > > > > > > the firmware expect? What will the guest OS usually? > > > > > > > > > > > > > > The reason I am asking is the attributes have to matched to avoid any > > > > > > > coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen > > > > > > > will configure the stage-2 to use Device nGnRnE. As the result, the result > > > > > > > memory access will be Device nGnRnE which is very strict. > > > > > > > > > > > > > > > > > > > Let me share with you the configuration example: > > > > > > scmi expects memory to be configured in the device-tree: > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0xXXXXXXX { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0xXXXXXX 0x0 0x1000>; > > > > > > }; > > > > > > > > > > > > where XXXXXX address I allocate in alloc_magic_pages function. > > > > > > > > > > The goal of alloc_magic_pages() is to allocate RAM. However, what you want > > > > > is a guest physical address and then map a part of the shared page. > > > > > > > > Do you mean that I can't use alloc_magic_pages to allocate shared > > > > memory region for SCMI? > > > Correct. alloc_magic_pages() will allocate a RAM page and then assign to the > > > guest. From your description, this is not what you want because you will > > > call XEN_DOMCTL_memory_mapping (and therefore replace the mapping). > > > > > > > Ok thanks, I will refactor this part in v2. > > > > > > > > > > > > > > > > I can see two options here: > > > > > 1) Hardcode the SCMI region in the memory map > > > > > 2) Create a new region in the memory map that can be used for reserving > > > > > memory for mapping. > > > > > > > > Could you please explain what do you mean under the "new region in the > > > > memory map"? > > > > > > I mean reserving some guest physical address that could be used for map host > > > physical address (e.g. SCMI region, GIC CPU interface...). > > > > > > So rather than hardcoding the address, we have something more flexible. > > > > > > > Ok, I will fix that in v2. > > Just for avoidance of doubt. I was clarify option 2 and not requesting to > implement. That said, if you want to implement option 2 I would be happy to > review it. > I think it's time for me to start working on v2. I think option 2 can be implemented. If not - I will look for an alternative way. > > > > > > > > > > > > > > > > We still have plenty of space in the guest memory map. So the former is > > > > > probably going to be fine for now. > > > > > > > > > > > Then I get paddr of the scmi channel for this domain and use > > > > > > XEN_DOMCTL_memory_mapping to map scmi channel address to gfn. > > > > > > > Hope I wass able to answer your question. > > > > > > > > > > What you provided me is how the guest OS will locate the shared memory. This > > > > > still doesn't tell me which memory attribute will be used to map the page in > > > > > Stage-1 (guest page-tables). > > > > > > > > > > To find that out, you want to look at the driver and how the mapping is > > > > > done. The Linux driver (drivers/firmware/arm_scmi) is using devm_ioremap() > > > > > (see smc_chan_setup()). > > > > > > > > > > Under the hood, the function devm_ioremap() is using PROT_DEVICE_nGnRE > > > > > (arm64) which is one of the most restrictive memory attribute. > > > > > > > > > > This means the firmware should be able to deal with the most restrictive > > > > > attribute and therefore using XEN_DOMCTL_memory_mapping to map the shared > > > > > page in stage-2 should be fine. > > > > > > > > > > > > > I'm using vmap call to map channel memory (see smc_create_channel()). > > > > vmap call sets PAGE_HYPERVISOR flag which sets MT_NORMAL (0x7) flag. > > > > > > You want to use ioremap(). > > > > > > > I've used ioremap originally, but changed it to vmap because ioremap > > doesn't support memcpy. > > What if I use __vmap with MT_DEVICE_nGnRE flag? > > That's not going to help. Our implementation of memcpy() is using unaligned > access (which is forbidden on Device memory). > > You will need something similar to memcpy_toio() in Linux. I don't think we > have one today in Xen, so I would suggest to import the implementation from > Linux. > Ok. Then I'll import memcpy_toio from Linux kernel. > > > > > > Considering that protocol is synchronous and only one agent per channel is > > > > expected - this works fine for now. > > > > But I agree that the same memory attributes should be used in xen and > > > > kernel. I fill fix that in v2. > > > > > > I am a bit confused. Are you mapping the full shared memory area in Xen? If > > > yes, why do you need to map the memory that is going to be shared with a > > > domain? > > > > > > > Xen should have an access to all agent channels because it should send > > SCMI_BASE_DISCOVER_AGENT to each channel and receive agent_id during > > scmi_probe call. > > Hmmm... Just to confirm, this will only happen during Xen boot? IOW, Xen > will never write to the channel when a domain is running? > Yes. Only during Xen boot. > If yes, then I think it would be best to unmap the channel once they are > used. This would prevent all sort of issues (e.g. Xen mistakenly written in > them). > That's a good advise. Thank you. Best regards, Oleksii.
Hi Julien, On Thu, Jan 06, 2022 at 04:04:34PM +0000, Julien Grall wrote: > Hi, > > On 06/01/2022 15:43, Oleksii Moisieiev wrote: > > On Thu, Jan 06, 2022 at 02:02:10PM +0000, Julien Grall wrote: > > > > > > > > > On 06/01/2022 13:53, Oleksii Moisieiev wrote: > > > > Hi Julien, > > > > > > Hi, > > > > > > > > > > > On Mon, Jan 03, 2022 at 01:14:17PM +0000, Julien Grall wrote: > > > > > Hi, > > > > > > > > > > On 24/12/2021 17:02, Oleksii Moisieiev wrote: > > > > > > On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: > > > > > > > On 20/12/2021 16:41, Oleksii Moisieiev wrote: > > > > > > > > > 2) What are the expected memory attribute for the regions? > > > > > > > > > > > > > > > > xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. > > > > > > > > > > > > > > I think you misunderstood my comment. Memory can be mapped with various type > > > > > > > (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will > > > > > > > the firmware expect? What will the guest OS usually? > > > > > > > > > > > > > > The reason I am asking is the attributes have to matched to avoid any > > > > > > > coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen > > > > > > > will configure the stage-2 to use Device nGnRnE. As the result, the result > > > > > > > memory access will be Device nGnRnE which is very strict. > > > > > > > > > > > > > > > > > > > Let me share with you the configuration example: > > > > > > scmi expects memory to be configured in the device-tree: > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0xXXXXXXX { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0xXXXXXX 0x0 0x1000>; > > > > > > }; > > > > > > > > > > > > where XXXXXX address I allocate in alloc_magic_pages function. > > > > > > > > > > The goal of alloc_magic_pages() is to allocate RAM. However, what you want > > > > > is a guest physical address and then map a part of the shared page. > > > > > > > > Do you mean that I can't use alloc_magic_pages to allocate shared > > > > memory region for SCMI? > > > Correct. alloc_magic_pages() will allocate a RAM page and then assign to the > > > guest. From your description, this is not what you want because you will > > > call XEN_DOMCTL_memory_mapping (and therefore replace the mapping). > > > > > > > Ok thanks, I will refactor this part in v2. > > > > > > > > > > > > > > > > I can see two options here: > > > > > 1) Hardcode the SCMI region in the memory map > > > > > 2) Create a new region in the memory map that can be used for reserving > > > > > memory for mapping. > > > > > > > > Could you please explain what do you mean under the "new region in the > > > > memory map"? > > > > > > I mean reserving some guest physical address that could be used for map host > > > physical address (e.g. SCMI region, GIC CPU interface...). > > > > > > So rather than hardcoding the address, we have something more flexible. > > > > > > > Ok, I will fix that in v2. > > Just for avoidance of doubt. I was clarify option 2 and not requesting to > implement. That said, if you want to implement option 2 I would be happy to > review it. > I'm thinking about the best way to reserve address for the domain. We have xen_pfn_t shared_info_pfn in struct xc_dom_image which is not used for Arm architecture. It can be set from shared_info_arm callback, defined in xg_dom_arm.c. I can use shared_info to store address of the SCMI and then use map_sci_page to call XEN_DOMCTL_memory_mapping. This will allow me to reuse existing functionality and do not allocate extra RAM. What do you think about that? -- Best regards, Oleksii. > > > > > > > > > > > > > > > > We still have plenty of space in the guest memory map. So the former is > > > > > probably going to be fine for now. > > > > > > > > > > > Then I get paddr of the scmi channel for this domain and use > > > > > > XEN_DOMCTL_memory_mapping to map scmi channel address to gfn. > > > > > > > Hope I wass able to answer your question. > > > > > > > > > > What you provided me is how the guest OS will locate the shared memory. This > > > > > still doesn't tell me which memory attribute will be used to map the page in > > > > > Stage-1 (guest page-tables). > > > > > > > > > > To find that out, you want to look at the driver and how the mapping is > > > > > done. The Linux driver (drivers/firmware/arm_scmi) is using devm_ioremap() > > > > > (see smc_chan_setup()). > > > > > > > > > > Under the hood, the function devm_ioremap() is using PROT_DEVICE_nGnRE > > > > > (arm64) which is one of the most restrictive memory attribute. > > > > > > > > > > This means the firmware should be able to deal with the most restrictive > > > > > attribute and therefore using XEN_DOMCTL_memory_mapping to map the shared > > > > > page in stage-2 should be fine. > > > > > > > > > > > > > I'm using vmap call to map channel memory (see smc_create_channel()). > > > > vmap call sets PAGE_HYPERVISOR flag which sets MT_NORMAL (0x7) flag. > > > > > > You want to use ioremap(). > > > > > > > I've used ioremap originally, but changed it to vmap because ioremap > > doesn't support memcpy. > > What if I use __vmap with MT_DEVICE_nGnRE flag? > > That's not going to help. Our implementation of memcpy() is using unaligned > access (which is forbidden on Device memory). > > You will need something similar to memcpy_toio() in Linux. I don't think we > have one today in Xen, so I would suggest to import the implementation from > Linux. > > > > > > > Considering that protocol is synchronous and only one agent per channel is > > > > expected - this works fine for now. > > > > But I agree that the same memory attributes should be used in xen and > > > > kernel. I fill fix that in v2. > > > > > > I am a bit confused. Are you mapping the full shared memory area in Xen? If > > > yes, why do you need to map the memory that is going to be shared with a > > > domain? > > > > > > > Xen should have an access to all agent channels because it should send > > SCMI_BASE_DISCOVER_AGENT to each channel and receive agent_id during > > scmi_probe call. > > Hmmm... Just to confirm, this will only happen during Xen boot? IOW, Xen > will never write to the channel when a domain is running? > > If yes, then I think it would be best to unmap the channel once they are > used. This would prevent all sort of issues (e.g. Xen mistakenly written in > them). > > Cheers, > > -- > Julien Grall
On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > Hi Stefano, > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > Hi Stefano, > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > resets and power-domains via SCMI; > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > --- > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > + > > > > > > > > endmenu > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > new file mode 100644 > > > > > > > > index 0000000000..9563067ddc > > > > > > > > --- /dev/null > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > +config SCMI_SMC > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > + default n > > > > > > > > + depends on SCI > > > > > > > > + ---help--- > > > > > > > > + > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > + for communication. > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > obj-y += sci.o > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > new file mode 100644 > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > --- /dev/null > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > +/* > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > + * > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > + * > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > + * > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > + * (at your option) any later version. > > > > > > > > + * > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > + * GNU General Public License for more details. > > > > > > > > + */ > > > > > > > > + > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > +#include <asm/smccc.h> > > > > > > > > +#include <asm/io.h> > > > > > > > > +#include <xen/bitops.h> > > > > > > > > +#include <xen/config.h> > > > > > > > > +#include <xen/sched.h> > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > +#include <xen/iocap.h> > > > > > > > > +#include <xen/init.h> > > > > > > > > +#include <xen/err.h> > > > > > > > > +#include <xen/lib.h> > > > > > > > > +#include <xen/list.h> > > > > > > > > +#include <xen/mm.h> > > > > > > > > +#include <xen/string.h> > > > > > > > > +#include <xen/time.h> > > > > > > > > +#include <xen/vmap.h> > > > > > > > > + > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > + > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > + > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > + > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > channels: > > > > > > > > > > > > reserved-memory { > > > > > > /* reserved region for scmi channels*/ > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > no-map; > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > }; > > > > > > }; > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > the current scmi channel: > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > }; > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > assigned to this agent. > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > compatible string, not as a node name, and it would need to be described > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > reserved-memory { > > > > > scp-shmem@0x53FF0000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > }; > > > > > scp-shmem@0x53FF1000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > }; > > > > > scp-shmem@0x53FF2000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > }; > > > > > ... > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > there were some if, checking if memory weren't reserved. > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > shmem page. > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > which has no compatible string and provides no-map property. > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > space for the domain. > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > and scmi_devid property to the > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > Those node and property are needed only for Xen and useless for > > > > non-virtualized systems. I can add this node and property description to > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > What do you think about it? > > > > > > Reply below > > > > > > [...] > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > activities to get "linux,scmi_mem" upstream under > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > Documentation/devicetree/bindings (probably > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > work on the Xen code that makes use of it. > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > Xen specific. In general, it would be best not to introduce Xen specific > > > properties into generic bindings. It is a problem both from a > > > specification perspective (because it has hard to handle Xen specific > > > cases in fully generic bindings, especially as those bindings are > > > maintained as part of the Linux kernel) and from a user perspective > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > looks like a generic property that should be needed for the Linux SCMI > > > driver too. Why the Linux driver doesn't need it? > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > On non-virtualized systems - there is no need of this call, because OS > > is the only one entity, running on the system. > > OK. Even if it is only required for virtualized systems, I think that > scmi_devid is important enough that should be part of the upstream > binding. I think it is worth starting an email thread on the LKML with > Rob Herring and the SCMI maintainers to discuss the addition of > scmi_devid to the binding. > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > list of device_ids to dom.cfg, such as: > > sci_devs = [ 0, 1, 15, 35 ]; > > > > Using this approach, we can remove scmi_devid from the device tree and > > just pass a list of scmi_devids to XEN using additional hypercall. > > We can probably make hypercall taking devid list as input parameter. > > This will take only 1 hypercall to setup sci permissions. > > But how would a user know which are the right SCMI IDs to add to the > sci_devs list? Would the user have to go and read the reference manual > of the platform to find the SCMI IDs and then write sci_devs by hand? > If that is the case, then I think that it would be better to add > scmi_devid to device tree. > > In general, I think this configuration should happen automatically > without user intervention. The user should just specify "enable SCMI" > and it should work. > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > way, I don't think we should need linux,scmi_mem. > > > > This requires further investigation. I will try to make implementation > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > reuslts with you. > > OK, thanks. Hi Stefano, As I did some investigation about using reserved-memory area linux,scmi_mem and now I need your advice. I see 2 possible implementations for now: 1) Add memory-region parameter to cpu_scp_shm node which points to the reserved memory region. So device-tree will look like this: reserved-memory { /* reserved region for scmi channels*/ scmi_memory: region@53FF0000{ no-map; reg = <0x0 0x53FF0000 0x0 0x10000>; }; }; cpu_scp_shm: scp-shmem@0x53FF0000 { compatible = "arm,scmi-shmem"; reg = <0x0 0x53FF0000 0x0 0x1000>; memory-region = <&scmi_memory>; }; So cpu_scp_shm node has a reference to scmi_memory region. This mean that xen can find reserved memory region without adding additional names to the device-tree bindings. memory-region parameter as a reference to reserved memory and region creation described in: https://github.com/torvalds/linux/blob/v5.15/Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt This approach I've implemented already and it works. 2) The second approach is the format you suggested: > > > > > reserved-memory { > > > > > scp-shmem@0x53FF0000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > }; > > > > > scp-shmem@0x53FF1000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > }; > > > > > scp-shmem@0x53FF2000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > }; > > > > > ... This approach has an advantage that xen ARM_SCI driver do not know about how channels are placed in the reserved memory, but introduces some disadvantages: a) We provide extra 14 (in our case) arm,scmi-shmem nodes which are not used in the device-tree. In current implementation I have separate scmi.dtsi file which introduces scmi support for both XEN-based and non-virtualized systems. Having 14 extra channels in the device-tree may be confusing. b) In case if we have all 15 channels, described in partial device-tree, we should not copy any node to the domain device-tree. I think it will be better to generate arm,scmi-shmem node in the Domain device-tree. The problem is that arm,scmi-smc node, which is using arm,scmi-shmem node can't be generated. I prefer it to be copied from the partial device-tree because it includes some platform specific configuration, such as func-id and list of the protocols (for example different platforms may require different list of the protocols). So in this case we will have 1 node copied and 1 node generated. I think even for dom0less we should use arm,scmi-smc node from the device-tree because protocol configuration and funcid is related to the platform. I prefer the second approach and will try to make it if it's OK to copy arm,scmi-smc node from partial Device-tree and generate arm,scmi-shmem node. What do you think about that? Also I wanted to mention that I'm not planning to make ARM_SCI support for dom0less in terms of this patch series bacause I can't test dom0less configuration for now. So let me know if some of my functionality breaks dom0less. -- Best regards Oleksii.
On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > Hi Stefano, > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > Hi Stefano, > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > --- > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > + > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > new file mode 100644 > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > --- /dev/null > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > +config SCMI_SMC > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > + default n > > > > > > > > > + depends on SCI > > > > > > > > > + ---help--- > > > > > > > > > + > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > + for communication. > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > obj-y += sci.o > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > new file mode 100644 > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > --- /dev/null > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > +/* > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > + * > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > + * > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > + * > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > + * (at your option) any later version. > > > > > > > > > + * > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > + */ > > > > > > > > > + > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > +#include <asm/io.h> > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > +#include <xen/config.h> > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > +#include <xen/init.h> > > > > > > > > > +#include <xen/err.h> > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > +#include <xen/list.h> > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > +#include <xen/string.h> > > > > > > > > > +#include <xen/time.h> > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > + > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > + > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > + > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > + > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > channels: > > > > > > > > > > > > > > reserved-memory { > > > > > > > /* reserved region for scmi channels*/ > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > no-map; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > }; > > > > > > > }; > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > the current scmi channel: > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > }; > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > assigned to this agent. > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > reserved-memory { > > > > > > scp-shmem@0x53FF0000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > }; > > > > > > scp-shmem@0x53FF1000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > }; > > > > > > scp-shmem@0x53FF2000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > }; > > > > > > ... > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > there were some if, checking if memory weren't reserved. > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > shmem page. > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > which has no compatible string and provides no-map property. > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > space for the domain. > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > and scmi_devid property to the > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > Those node and property are needed only for Xen and useless for > > > > > non-virtualized systems. I can add this node and property description to > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > What do you think about it? > > > > > > > > Reply below > > > > > > > > [...] > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > Documentation/devicetree/bindings (probably > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > properties into generic bindings. It is a problem both from a > > > > specification perspective (because it has hard to handle Xen specific > > > > cases in fully generic bindings, especially as those bindings are > > > > maintained as part of the Linux kernel) and from a user perspective > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > looks like a generic property that should be needed for the Linux SCMI > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > On non-virtualized systems - there is no need of this call, because OS > > > is the only one entity, running on the system. > > > > OK. Even if it is only required for virtualized systems, I think that > > scmi_devid is important enough that should be part of the upstream > > binding. I think it is worth starting an email thread on the LKML with > > Rob Herring and the SCMI maintainers to discuss the addition of > > scmi_devid to the binding. > > > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > list of device_ids to dom.cfg, such as: > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > We can probably make hypercall taking devid list as input parameter. > > > This will take only 1 hypercall to setup sci permissions. > > > > But how would a user know which are the right SCMI IDs to add to the > > sci_devs list? Would the user have to go and read the reference manual > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > If that is the case, then I think that it would be better to add > > scmi_devid to device tree. > > > > In general, I think this configuration should happen automatically > > without user intervention. The user should just specify "enable SCMI" > > and it should work. > > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > way, I don't think we should need linux,scmi_mem. > > > > > > This requires further investigation. I will try to make implementation > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > reuslts with you. > > > > OK, thanks. > > Hi Stefano, > > As I did some investigation about using reserved-memory area > linux,scmi_mem and now I need your advice. > > I see 2 possible implementations for now: > 1) Add memory-region parameter to cpu_scp_shm node which points to the > reserved memory region. > So device-tree will look like this: > > reserved-memory { > /* reserved region for scmi channels*/ > scmi_memory: region@53FF0000{ > no-map; > reg = <0x0 0x53FF0000 0x0 0x10000>; > }; > }; > cpu_scp_shm: scp-shmem@0x53FF0000 { > compatible = "arm,scmi-shmem"; > reg = <0x0 0x53FF0000 0x0 0x1000>; > memory-region = <&scmi_memory>; > }; > > So cpu_scp_shm node has a reference to scmi_memory region. This mean > that xen can find reserved memory region without adding additional names > to the device-tree bindings. > memory-region parameter as a reference to reserved memory and region > creation described in: > https://github.com/torvalds/linux/blob/v5.15/Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt > > This approach I've implemented already and it works. This approach would require a discussion with the upstream device tree maintainers. Likely, we would need to add a note about the usage of the "memory-region" property to arm,scmi.yaml. Also, I have the feeling that they would ask to add the "memory-region" property directly to the "arm,scmi-smc" node, as an alternative (or in addition) to the existing "shmem" property. That said, from my point of view this approach is also a viable option. I don't see any major problems. The main question (after reading everything else that you wrote below) is whether the "arm,scmi-smc" node in this case could be automatically generated. > 2) The second approach is the format you suggested: > > > > > > reserved-memory { > > > > > > scp-shmem@0x53FF0000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > }; > > > > > > scp-shmem@0x53FF1000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > }; > > > > > > scp-shmem@0x53FF2000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > }; > > > > > > ... > > This approach has an advantage that xen ARM_SCI driver do not know about > how channels are placed in the reserved memory, but introduces some > disadvantages: > a) We provide extra 14 (in our case) arm,scmi-shmem nodes which are not used > in the device-tree. In current implementation I have separate scmi.dtsi > file which introduces scmi support for both XEN-based and > non-virtualized systems. Having 14 extra channels in the device-tree may > be confusing. I can see that while it would be ideal for Xen to see all 14+1 channels in device tree (on the host device tree), we wouldn't want to expose all of them to the domains, not even to dom0. How many channels do we want dom0 to see by the way? For this discussion, I'll just assume for now that dom0 only sees 1 channel like the domUs. Now we have a problem: how do we go about "filtering" the "arm,scmi-shmem" device tree nodes? Which is also what you are asking below in point b). > b) In case if we have all 15 channels, described in partial device-tree, I think you meant "described in the host device tree", right? > we should not copy any node to the domain device-tree. I think it will > be better to generate arm,scmi-shmem node in the Domain device-tree. Yes, I think it makes sense for Xen to generate the "arm,scmi-shmem" device tree description for the DomU/Dom0 based on the channels allocated to the domain. > The problem is that arm,scmi-smc node, which is using arm,scmi-shmem > node can't be generated. I prefer it to be copied from the partial > device-tree because it includes some platform specific configuration, > such as func-id and list of the protocols (for example different > platforms may require different list of the protocols). So in this > case we will have 1 node copied and 1 node generated. > > I think even for dom0less we should use arm,scmi-smc node from the > device-tree because protocol configuration and funcid is related to the > platform. I am not sure I understood what you wrote. You are saying that the "arm,scmi-smc" node includes some platform specific configurations so it cannot be automatically generated by Xen (or by the tools) and instead it needs to be manually provided as part of the partial dtb for the domU. Is that correct? If so, I would like to understand the reasons behind it. Manual device tree editing is problematic. I looked for "func-id" in Documentation/devicetree/bindings/firmware/arm,scmi.yaml but couldn't find any results. Do you have an example of the platform specific configuration or protocol configuration that would make it difficult to automatically generate the "arm,scmi-smc" node for the domains? Also, is this a problem just for approach #2 or also for approach #1? If it is a problem only for approach #2, then let's just go with approach #1. > I prefer the second approach and will try to make it if it's OK to copy > arm,scmi-smc node from partial Device-tree and generate arm,scmi-shmem > node. > > What do you think about that? From a device tree specification perspective, I think both approaches are OK (with a minor comment on the first approach as I wrote above.) But from a Xen perspective I think it is important that we don't require the user to manually provide the SCMI configuration in the partial DTB. It would be better if we could generate it automatically from Xen or the tools (or even an independent script). Or copy the "arm,scmi-smc" node from the host device tree to the domU device tree without modifications. So if using approach #1 allows us to automatically generate the "arm,scmi-smc" node for the guest, then I think it's best for sure. > Also I wanted to mention that I'm not planning to make ARM_SCI support for > dom0less in terms of this patch series bacause I can't test > dom0less configuration for now. So let me know if some of my > functionality breaks dom0less. That's fine. I don't mean to scope-creep your patch series, which is extremely valuable as is. That said, I would be happy to provide you with a very simple dom0less configuration for your platform to enable you to test, or alternatively I could write a patch to add dom0less domU support if you are happy to help reviewing and testing it.
On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > On Thu, Jan 06, 2022 at 04:04:34PM +0000, Julien Grall wrote: > > On 06/01/2022 15:43, Oleksii Moisieiev wrote: > > > On Thu, Jan 06, 2022 at 02:02:10PM +0000, Julien Grall wrote: > > > > On 06/01/2022 13:53, Oleksii Moisieiev wrote: > > > > > Hi Julien, > > > > > > > > > > On Mon, Jan 03, 2022 at 01:14:17PM +0000, Julien Grall wrote: > > > > > > Hi, > > > > > > > > > > > > On 24/12/2021 17:02, Oleksii Moisieiev wrote: > > > > > > > On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: > > > > > > > > On 20/12/2021 16:41, Oleksii Moisieiev wrote: > > > > > > > > > > 2) What are the expected memory attribute for the regions? > > > > > > > > > > > > > > > > > > xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. > > > > > > > > > > > > > > > > I think you misunderstood my comment. Memory can be mapped with various type > > > > > > > > (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will > > > > > > > > the firmware expect? What will the guest OS usually? > > > > > > > > > > > > > > > > The reason I am asking is the attributes have to matched to avoid any > > > > > > > > coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen > > > > > > > > will configure the stage-2 to use Device nGnRnE. As the result, the result > > > > > > > > memory access will be Device nGnRnE which is very strict. > > > > > > > > :w > > > > > > > > > > > > > > Let me share with you the configuration example: > > > > > > > scmi expects memory to be configured in the device-tree: > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0xXXXXXXX { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0xXXXXXX 0x0 0x1000>; > > > > > > > }; > > > > > > > > > > > > > > where XXXXXX address I allocate in alloc_magic_pages function. > > > > > > > > > > > > The goal of alloc_magic_pages() is to allocate RAM. However, what you want > > > > > > is a guest physical address and then map a part of the shared page. > > > > > > > > > > Do you mean that I can't use alloc_magic_pages to allocate shared > > > > > memory region for SCMI? > > > > Correct. alloc_magic_pages() will allocate a RAM page and then assign to the > > > > guest. From your description, this is not what you want because you will > > > > call XEN_DOMCTL_memory_mapping (and therefore replace the mapping). > > > > > > > > > > Ok thanks, I will refactor this part in v2. > > > > > > > > > > > > > > > > > > > > I can see two options here: > > > > > > 1) Hardcode the SCMI region in the memory map > > > > > > 2) Create a new region in the memory map that can be used for reserving > > > > > > memory for mapping. > > > > > > > > > > Could you please explain what do you mean under the "new region in the > > > > > memory map"? > > > > > > > > I mean reserving some guest physical address that could be used for map host > > > > physical address (e.g. SCMI region, GIC CPU interface...). > > > > > > > > So rather than hardcoding the address, we have something more flexible. > > > > > > > > > > Ok, I will fix that in v2. > > > > Just for avoidance of doubt. I was clarify option 2 and not requesting to > > implement. That said, if you want to implement option 2 I would be happy to > > review it. > > > > I'm thinking about the best way to reserve address for the domain. > We have xen_pfn_t shared_info_pfn in struct xc_dom_image which is not > used for Arm architecture. It can be set from shared_info_arm callback, > defined in xg_dom_arm.c. > I can use shared_info to store address of the SCMI and then use map_sci_page to > call XEN_DOMCTL_memory_mapping. > > This will allow me to reuse existing functionality and do not allocate > extra RAM. > > What do you think about that? I cannot speak for Julien but I think he meant something else (Julien please correct me if I am wrong.) Exposing addresses via device tree is not a problem. Normally we pick a fixed address for guest resources, for instance GUEST_GICD_BASE, see xen/include/public/arch-arm.h. We could do that for SCMI as well and it is basically approach 1). However, it is a bit inflexible and could cause issues with things like direct-map (https://marc.info/?l=xen-devel&m=163997768108997). A more flexible way would be for the SCMI guest address to be dynamically generated somehow. I am not sure how Julien envisioned the address to be generated exactly. Thanks to Oleksandr's work we have a way to find large regions of "free" address space. It is currently used for grant-table mappings. Maybe we could use a subset of it for SCMI? It might be best to wait for Julien's answer as he might have a better idea.
Hi Stefano, On Wed, Jan 19, 2022 at 05:28:21PM -0800, Stefano Stabellini wrote: > On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > > Hi Stefano, > > > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > --- > > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > > + > > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > > new file mode 100644 > > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > > --- /dev/null > > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > > +config SCMI_SMC > > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > > + default n > > > > > > > > > > + depends on SCI > > > > > > > > > > + ---help--- > > > > > > > > > > + > > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > > + for communication. > > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > > obj-y += sci.o > > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > new file mode 100644 > > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > > --- /dev/null > > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > > +/* > > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > + * > > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > > + * > > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > > + * > > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > > + * (at your option) any later version. > > > > > > > > > > + * > > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > > + */ > > > > > > > > > > + > > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > > +#include <asm/io.h> > > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > > +#include <xen/config.h> > > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > > +#include <xen/init.h> > > > > > > > > > > +#include <xen/err.h> > > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > > +#include <xen/list.h> > > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > > +#include <xen/string.h> > > > > > > > > > > +#include <xen/time.h> > > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > > + > > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > > + > > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > > + > > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > > + > > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > > channels: > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > /* reserved region for scmi channels*/ > > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > > no-map; > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > > }; > > > > > > > > }; > > > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > > the current scmi channel: > > > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > }; > > > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > > assigned to this agent. > > > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > > > reserved-memory { > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > }; > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > }; > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > }; > > > > > > > ... > > > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > > there were some if, checking if memory weren't reserved. > > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > > shmem page. > > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > > which has no compatible string and provides no-map property. > > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > > space for the domain. > > > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > > and scmi_devid property to the > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > > Those node and property are needed only for Xen and useless for > > > > > > non-virtualized systems. I can add this node and property description to > > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > > What do you think about it? > > > > > > > > > > Reply below > > > > > > > > > > [...] > > > > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > > Documentation/devicetree/bindings (probably > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > > properties into generic bindings. It is a problem both from a > > > > > specification perspective (because it has hard to handle Xen specific > > > > > cases in fully generic bindings, especially as those bindings are > > > > > maintained as part of the Linux kernel) and from a user perspective > > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > > looks like a generic property that should be needed for the Linux SCMI > > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > > On non-virtualized systems - there is no need of this call, because OS > > > > is the only one entity, running on the system. > > > > > > OK. Even if it is only required for virtualized systems, I think that > > > scmi_devid is important enough that should be part of the upstream > > > binding. I think it is worth starting an email thread on the LKML with > > > Rob Herring and the SCMI maintainers to discuss the addition of > > > scmi_devid to the binding. > > > > > > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > > list of device_ids to dom.cfg, such as: > > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > > We can probably make hypercall taking devid list as input parameter. > > > > This will take only 1 hypercall to setup sci permissions. > > > > > > But how would a user know which are the right SCMI IDs to add to the > > > sci_devs list? Would the user have to go and read the reference manual > > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > > If that is the case, then I think that it would be better to add > > > scmi_devid to device tree. > > > > > > In general, I think this configuration should happen automatically > > > without user intervention. The user should just specify "enable SCMI" > > > and it should work. > > > > > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > > way, I don't think we should need linux,scmi_mem. > > > > > > > > This requires further investigation. I will try to make implementation > > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > > reuslts with you. > > > > > > OK, thanks. > > > > Hi Stefano, > > > > As I did some investigation about using reserved-memory area > > linux,scmi_mem and now I need your advice. > > > > I see 2 possible implementations for now: > > 1) Add memory-region parameter to cpu_scp_shm node which points to the > > reserved memory region. > > So device-tree will look like this: > > > > reserved-memory { > > /* reserved region for scmi channels*/ > > scmi_memory: region@53FF0000{ > > no-map; > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > }; > > }; > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > compatible = "arm,scmi-shmem"; > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > memory-region = <&scmi_memory>; > > }; > > > > So cpu_scp_shm node has a reference to scmi_memory region. This mean > > that xen can find reserved memory region without adding additional names > > to the device-tree bindings. > > memory-region parameter as a reference to reserved memory and region > > creation described in: > > https://urldefense.com/v3/__https://github.com/torvalds/linux/blob/v5.15/Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt__;!!GF_29dbcQIUBPA!k6x19x1gYF1CPlgAZj7std3ifqhq-9DXvuF0nwonNPUwMzZpYHYbrRJziJrgdFIOjyan$ [github[.]com] > > > > This approach I've implemented already and it works. > > This approach would require a discussion with the upstream device tree > maintainers. Likely, we would need to add a note about the usage of the > "memory-region" property to arm,scmi.yaml. > > Also, I have the feeling that they would ask to add the "memory-region" > property directly to the "arm,scmi-smc" node, as an alternative (or > in addition) to the existing "shmem" property. > > That said, from my point of view this approach is also a viable option. > I don't see any major problems. > > The main question (after reading everything else that you wrote below) > is whether the "arm,scmi-smc" node in this case could be automatically > generated. > arm,scmi-smc node can be generated in both cases. I think I'd leave it as backup in case if the second approach will not work. > > > 2) The second approach is the format you suggested: > > > > > > > reserved-memory { > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > }; > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > }; > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > }; > > > > > > > ... > > > > This approach has an advantage that xen ARM_SCI driver do not know about > > how channels are placed in the reserved memory, but introduces some > > disadvantages: > > a) We provide extra 14 (in our case) arm,scmi-shmem nodes which are not used > > in the device-tree. In current implementation I have separate scmi.dtsi > > file which introduces scmi support for both XEN-based and > > non-virtualized systems. Having 14 extra channels in the device-tree may > > be confusing. > > I can see that while it would be ideal for Xen to see all 14+1 channels > in device tree (on the host device tree), we wouldn't want to expose all > of them to the domains, not even to dom0. > > How many channels do we want dom0 to see by the way? For this > discussion, I'll just assume for now that dom0 only sees 1 channel like > the domUs. For dom0 we need only one channel. > > Now we have a problem: how do we go about "filtering" the > "arm,scmi-shmem" device tree nodes? Which is also what you are asking > below in point b). > Xen will not need to filter "arm,scmi-shmem" node. It will just create shmem node in Domain device-tree. I don't see any problem for xen configuration here. What bothers me here is that I set scmi configuration for platform dts, not for xen or domu dts files. So for example I have the following structure of the dts files for my platform (r8a77961-salvator-xs): * r8a77961-scmi.dtsi - this file includes all scmi related nodes and set scmi_devid for the devices, that should use scmi. * r8a77961-salvator-xs.dts - dts file which generates dtb for the platform. It includes r8a77961-scmi.dtsi so I populate scmi to platform dtb, which is used for system with no hypervisor. * r8a77961-salvator-xs-xen.dts - dts file for xen which includes r8a77961-salvator-xs.dts and inherits scmi configuration from it. * r8a77961-salvator-xs-domu.dts - dts file for DomU which includes r8a77961-salvator-xs.dts and inherits scmi configuration from it. In this case r8a77961-salvator-xs.dtb r8a77961-salvator-xs-xen.dtb r8a77961-salvator-xs-domu.dtb files will inherit 14+1 channel. I can give you a link to Merge request with this changes if you need it. For xen and domu dtb it is not a problem because all "arm,scmi-shmem" nodes will be omitted and new will be generated for the domains. What bothers me is that r8a77961-salvator-xs.dtb will have 14 unused channels. Just got an idea while writing this: I can create only one "arm,scmi-shmem" node in r8a77961-scmi.dtsi and add 14 more nodes, needed for xen explicitly in r8a77961-salvator-xs-xen.dts. Then we will have valid configurations for all cases. This can be a solution. What do you think? > > > b) In case if we have all 15 channels, described in partial device-tree, > > I think you meant "described in the host device tree", right? > Yeah that's what I've meant. > > > we should not copy any node to the domain device-tree. I think it will > > be better to generate arm,scmi-shmem node in the Domain device-tree. > > Yes, I think it makes sense for Xen to generate the "arm,scmi-shmem" > device tree description for the DomU/Dom0 based on the channels > allocated to the domain. > > > > The problem is that arm,scmi-smc node, which is using arm,scmi-shmem > > node can't be generated. I prefer it to be copied from the partial > > device-tree because it includes some platform specific configuration, > > such as func-id and list of the protocols (for example different > > platforms may require different list of the protocols). So in this > > case we will have 1 node copied and 1 node generated. > > > > I think even for dom0less we should use arm,scmi-smc node from the > > device-tree because protocol configuration and funcid is related to the > > platform. > > I am not sure I understood what you wrote. You are saying that the > "arm,scmi-smc" node includes some platform specific configurations so > it cannot be automatically generated by Xen (or by the tools) and > instead it needs to be manually provided as part of the partial dtb for > the domU. Is that correct? > > If so, I would like to understand the reasons behind it. Manual > device tree editing is problematic. > > I looked for "func-id" in > Documentation/devicetree/bindings/firmware/arm,scmi.yaml but couldn't > find any results. Do you have an example of the platform specific > configuration or protocol configuration that would make it difficult to > automatically generate the "arm,scmi-smc" node for the domains? Sorry, I used wrong term (used term from the specification), arm,smc-id of cause. > > Also, is this a problem just for approach #2 or also for approach #1? > If it is a problem only for approach #2, then let's just go with > approach #1. > We can't copy "arm,scmi-smc" in both approaches. The difference is that in the first approach we can copy both "arm,scmi-smc" and "arm,scmi-shmem" nodes while in the second approach we should copy "arm,scmi-smc", but we have to generate "arm,scmi-shmem" node. arm,scmi-smc node can't be generated because it includes properties and configurations that depends from platform and should be get from the device tree. Here is "arm,scmi-smc" node expample: firmware { scmi { compatible = "arm,scmi-smc" arm,smc-id = <0x82000002>; shmem = <&cpu_scp_shm>; #address-cells = <1>; #size-cells = <0>; scmi_power: protocol@11 { reg = <0x11>; #power-domain-cells = <1>; }; scmi_clock: protocol@14 { ... }; scmi_reset: protocol@16 { ... }; ... }; }; It has 3 configurable options: * arm,smc-id parameter, setting func_id for scmi protocol. This id can be different for different platforms. For example stm32mp1 architecture use different scm-id for different agents: https://github.com/ARM-software/arm-trusted-firmware/blob/0586c41b3f2d52aae847b7212e7b0c7e19197ea2/plat/st/stm32mp1/include/stm32mp1_smc.h#L39 * shmem which includes phandle to arm,scmi-shmem node. But this is not a problem and can be updated. * list of the protocol subnodes. This is also configurable parameter, not regs or names, but the number of the protocols. For example onle platform can use power-domains/clock/resets via scmi, when another will require volage-control and sensor-management to be added. Xen should know this parameters to be able to generate "arm,scmi-smc" node. Also we're currently discussing new scmi protocol with ARM: Pinctrl over SCMI. It should allow domains to access pinctrl subsystem, placed in Firmware through SCMI protocol. scmi_pinctrl node will look like this: firmware { scmi { ... scmi_pinctrl: protocol@18 { reg = <0x18>; #pinctrl-cells = <0>; i2c2_pins: i2c2 { groups = <74>; /* i2c2_a */ function = <15>; /* i2c2 */ }; irq0_pins: irq0 { groups = <81>; /* intc_ex_irq0 */ function = <19>; /* intc_ex */ }; avb_pins: avb { mux { /* avb_link, avb_mdio, avb_mii */ groups = <17>, <21>, <22>; function = <1>; /* avb */ }; pins_mdio { groups = <21>; /* avb_mdio */ drive-strength = <24>; }; pins_mii_tx { /* PIN_AVB_TX_CTL, PIN_AVB_TXC, PIN_AVB_TD0, PIN_AVB_TD1, PIN_AVB_TD2, PIN_AVB_TD3 */ pins = <242>, <240>, <236>, <237>, <238>, <239>; drive-strength = <12>; }; }; ... }; }; }; So "arm,scmi-smc" node will have even more platform specific settings. > > > I prefer the second approach and will try to make it if it's OK to copy > > arm,scmi-smc node from partial Device-tree and generate arm,scmi-shmem > > node. > > > > What do you think about that? > > From a device tree specification perspective, I think both approaches > are OK (with a minor comment on the first approach as I wrote above.) > > But from a Xen perspective I think it is important that we don't require > the user to manually provide the SCMI configuration in the partial DTB. > It would be better if we could generate it automatically from Xen or the > tools (or even an independent script). Or copy the "arm,scmi-smc" node > from the host device tree to the domU device tree without modifications. I think copy "arm,scmi-smc" node is the only option we have. I'm not sure what do you mean under "host device tree" if you mean Xen device-tree - then I think it will not cover the case with stm32mp1 I've mentioned above. I think it will be better to copy "arm,scmi-smc" node from Domu partial Device-tree to Domu device-tree. So AGENT0 smc-id will be set in xen device-tree and copied to dom0 and AGENT1 scm-is set in domu device-tree and copied to dom-u. Do you agree with my points? > > So if using approach #1 allows us to automatically generate the > "arm,scmi-smc" node for the guest, then I think it's best for sure. > Summarizing all written above I would focus on the second approach and put aside the first approach implementation. If you don't mind. > > > Also I wanted to mention that I'm not planning to make ARM_SCI support for > > dom0less in terms of this patch series bacause I can't test > > dom0less configuration for now. So let me know if some of my > > functionality breaks dom0less. > > That's fine. I don't mean to scope-creep your patch series, which is > extremely valuable as is. > > That said, I would be happy to provide you with a very simple dom0less > configuration for your platform to enable you to test, or alternatively > I could write a patch to add dom0less domU support if you are happy to > help reviewing and testing it. I was thinking about making dom0less support in the different patch-series because there are still questions to be discussed. For example, how arm,scmi-smc node will be generated for DomUs and how the case, when scmi configuration is different for DomU1 and DomU2 (as in case of stm32mp1 when smc-id is different) should be handled. What do you think about continue without dom0less support and discuss dom0less once we done with the main part? Oleksii.
On Wed, Jan 19, 2022 at 06:10:46PM -0800, Stefano Stabellini wrote: > On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > > On Thu, Jan 06, 2022 at 04:04:34PM +0000, Julien Grall wrote: > > > On 06/01/2022 15:43, Oleksii Moisieiev wrote: > > > > On Thu, Jan 06, 2022 at 02:02:10PM +0000, Julien Grall wrote: > > > > > On 06/01/2022 13:53, Oleksii Moisieiev wrote: > > > > > > Hi Julien, > > > > > > > > > > > > On Mon, Jan 03, 2022 at 01:14:17PM +0000, Julien Grall wrote: > > > > > > > Hi, > > > > > > > > > > > > > > On 24/12/2021 17:02, Oleksii Moisieiev wrote: > > > > > > > > On Fri, Dec 24, 2021 at 03:42:42PM +0100, Julien Grall wrote: > > > > > > > > > On 20/12/2021 16:41, Oleksii Moisieiev wrote: > > > > > > > > > > > 2) What are the expected memory attribute for the regions? > > > > > > > > > > > > > > > > > > > > xen uses iommu_permit_access to pass agent page to the guest. So guest can access the page directly. > > > > > > > > > > > > > > > > > > I think you misunderstood my comment. Memory can be mapped with various type > > > > > > > > > (e.g. Device, Memory) and attribute (cacheable, non-cacheable...). What will > > > > > > > > > the firmware expect? What will the guest OS usually? > > > > > > > > > > > > > > > > > > The reason I am asking is the attributes have to matched to avoid any > > > > > > > > > coherency issues. At the moment, if you use XEN_DOMCTL_memory_mapping, Xen > > > > > > > > > will configure the stage-2 to use Device nGnRnE. As the result, the result > > > > > > > > > memory access will be Device nGnRnE which is very strict. > > > > > > > > > :w > > > > > > > > > > > > > > > > Let me share with you the configuration example: > > > > > > > > scmi expects memory to be configured in the device-tree: > > > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0xXXXXXXX { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0xXXXXXX 0x0 0x1000>; > > > > > > > > }; > > > > > > > > > > > > > > > > where XXXXXX address I allocate in alloc_magic_pages function. > > > > > > > > > > > > > > The goal of alloc_magic_pages() is to allocate RAM. However, what you want > > > > > > > is a guest physical address and then map a part of the shared page. > > > > > > > > > > > > Do you mean that I can't use alloc_magic_pages to allocate shared > > > > > > memory region for SCMI? > > > > > Correct. alloc_magic_pages() will allocate a RAM page and then assign to the > > > > > guest. From your description, this is not what you want because you will > > > > > call XEN_DOMCTL_memory_mapping (and therefore replace the mapping). > > > > > > > > > > > > > Ok thanks, I will refactor this part in v2. > > > > > > > > > > > > > > > > > > > > > > > > I can see two options here: > > > > > > > 1) Hardcode the SCMI region in the memory map > > > > > > > 2) Create a new region in the memory map that can be used for reserving > > > > > > > memory for mapping. > > > > > > > > > > > > Could you please explain what do you mean under the "new region in the > > > > > > memory map"? > > > > > > > > > > I mean reserving some guest physical address that could be used for map host > > > > > physical address (e.g. SCMI region, GIC CPU interface...). > > > > > > > > > > So rather than hardcoding the address, we have something more flexible. > > > > > > > > > > > > > Ok, I will fix that in v2. > > > > > > Just for avoidance of doubt. I was clarify option 2 and not requesting to > > > implement. That said, if you want to implement option 2 I would be happy to > > > review it. > > > > > > > I'm thinking about the best way to reserve address for the domain. > > We have xen_pfn_t shared_info_pfn in struct xc_dom_image which is not > > used for Arm architecture. It can be set from shared_info_arm callback, > > defined in xg_dom_arm.c. > > I can use shared_info to store address of the SCMI and then use map_sci_page to > > call XEN_DOMCTL_memory_mapping. > > > > This will allow me to reuse existing functionality and do not allocate > > extra RAM. > > > > What do you think about that? > > I cannot speak for Julien but I think he meant something else (Julien > please correct me if I am wrong.) Exposing addresses via device tree is > not a problem. > > Normally we pick a fixed address for guest resources, for instance > GUEST_GICD_BASE, see xen/include/public/arch-arm.h. We could do that for > SCMI as well and it is basically approach 1). > > However, it is a bit inflexible and could cause issues with things like > direct-map (https://urldefense.com/v3/__https://marc.info/?l=xen-devel&m=163997768108997__;!!GF_29dbcQIUBPA!kvNsu9pjqIwZ42N2q6aSQhTT_zA3OCEDkr7DwmAiuldEMwj2UiFReaPI8XlxsG-HOZ6v$ [marc[.]info]). A more > flexible way would be for the SCMI guest address to be dynamically > generated somehow. > > I am not sure how Julien envisioned the address to be generated exactly. > > Thanks to Oleksandr's work we have a way to find large regions of "free" > address space. It is currently used for grant-table mappings. Maybe we > could use a subset of it for SCMI? It might be best to wait for Julien's > answer as he might have a better idea. Thank you for the answer. I think it will be best to reserve some space and generate address for SMCI. Hope Julien will advise how it can be done.
On Thu, 20 Jan 2022, Oleksii Moisieiev wrote: > On Wed, Jan 19, 2022 at 05:28:21PM -0800, Stefano Stabellini wrote: > > On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > > > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > Hi Stefano, > > > > > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > --- > > > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > > > + > > > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > > > --- /dev/null > > > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > > > +config SCMI_SMC > > > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > > > + default n > > > > > > > > > > > + depends on SCI > > > > > > > > > > > + ---help--- > > > > > > > > > > > + > > > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > > > + for communication. > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > > > obj-y += sci.o > > > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > > > --- /dev/null > > > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > > > +/* > > > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > + * > > > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > > > + * > > > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > > > + * > > > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > > > + * (at your option) any later version. > > > > > > > > > > > + * > > > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > > > + */ > > > > > > > > > > > + > > > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > > > +#include <asm/io.h> > > > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > > > +#include <xen/config.h> > > > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > > > +#include <xen/init.h> > > > > > > > > > > > +#include <xen/err.h> > > > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > > > +#include <xen/list.h> > > > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > > > +#include <xen/string.h> > > > > > > > > > > > +#include <xen/time.h> > > > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > > > + > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > > > + > > > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > > > + > > > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > > > + > > > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > > > channels: > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > /* reserved region for scmi channels*/ > > > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > > > no-map; > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > > > }; > > > > > > > > > }; > > > > > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > > > the current scmi channel: > > > > > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > }; > > > > > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > > > assigned to this agent. > > > > > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > }; > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > }; > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > }; > > > > > > > > ... > > > > > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > > > there were some if, checking if memory weren't reserved. > > > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > > > shmem page. > > > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > > > which has no compatible string and provides no-map property. > > > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > > > space for the domain. > > > > > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > > > and scmi_devid property to the > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > > > Those node and property are needed only for Xen and useless for > > > > > > > non-virtualized systems. I can add this node and property description to > > > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > > > What do you think about it? > > > > > > > > > > > > Reply below > > > > > > > > > > > > [...] > > > > > > > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > > > Documentation/devicetree/bindings (probably > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > > > properties into generic bindings. It is a problem both from a > > > > > > specification perspective (because it has hard to handle Xen specific > > > > > > cases in fully generic bindings, especially as those bindings are > > > > > > maintained as part of the Linux kernel) and from a user perspective > > > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > > > looks like a generic property that should be needed for the Linux SCMI > > > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > > > On non-virtualized systems - there is no need of this call, because OS > > > > > is the only one entity, running on the system. > > > > > > > > OK. Even if it is only required for virtualized systems, I think that > > > > scmi_devid is important enough that should be part of the upstream > > > > binding. I think it is worth starting an email thread on the LKML with > > > > Rob Herring and the SCMI maintainers to discuss the addition of > > > > scmi_devid to the binding. > > > > > > > > > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > > > list of device_ids to dom.cfg, such as: > > > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > > > We can probably make hypercall taking devid list as input parameter. > > > > > This will take only 1 hypercall to setup sci permissions. > > > > > > > > But how would a user know which are the right SCMI IDs to add to the > > > > sci_devs list? Would the user have to go and read the reference manual > > > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > > > If that is the case, then I think that it would be better to add > > > > scmi_devid to device tree. > > > > > > > > In general, I think this configuration should happen automatically > > > > without user intervention. The user should just specify "enable SCMI" > > > > and it should work. > > > > > > > > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > > > way, I don't think we should need linux,scmi_mem. > > > > > > > > > > This requires further investigation. I will try to make implementation > > > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > > > reuslts with you. > > > > > > > > OK, thanks. > > > > > > Hi Stefano, > > > > > > As I did some investigation about using reserved-memory area > > > linux,scmi_mem and now I need your advice. > > > > > > I see 2 possible implementations for now: > > > 1) Add memory-region parameter to cpu_scp_shm node which points to the > > > reserved memory region. > > > So device-tree will look like this: > > > > > > reserved-memory { > > > /* reserved region for scmi channels*/ > > > scmi_memory: region@53FF0000{ > > > no-map; > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > }; > > > }; > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > compatible = "arm,scmi-shmem"; > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > memory-region = <&scmi_memory>; > > > }; > > > > > > So cpu_scp_shm node has a reference to scmi_memory region. This mean > > > that xen can find reserved memory region without adding additional names > > > to the device-tree bindings. > > > memory-region parameter as a reference to reserved memory and region > > > creation described in: > > > https://urldefense.com/v3/__https://github.com/torvalds/linux/blob/v5.15/Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt__;!!GF_29dbcQIUBPA!k6x19x1gYF1CPlgAZj7std3ifqhq-9DXvuF0nwonNPUwMzZpYHYbrRJziJrgdFIOjyan$ [github[.]com] > > > > > > This approach I've implemented already and it works. > > > > This approach would require a discussion with the upstream device tree > > maintainers. Likely, we would need to add a note about the usage of the > > "memory-region" property to arm,scmi.yaml. > > > > Also, I have the feeling that they would ask to add the "memory-region" > > property directly to the "arm,scmi-smc" node, as an alternative (or > > in addition) to the existing "shmem" property. > > > > That said, from my point of view this approach is also a viable option. > > I don't see any major problems. > > > > The main question (after reading everything else that you wrote below) > > is whether the "arm,scmi-smc" node in this case could be automatically > > generated. > > > > arm,scmi-smc node can be generated in both cases. I think I'd leave it > as backup in case if the second approach will not work. > > > > > > 2) The second approach is the format you suggested: > > > > > > > > reserved-memory { > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > }; > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > }; > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > }; > > > > > > > > ... > > > > > > This approach has an advantage that xen ARM_SCI driver do not know about > > > how channels are placed in the reserved memory, but introduces some > > > disadvantages: > > > a) We provide extra 14 (in our case) arm,scmi-shmem nodes which are not used > > > in the device-tree. In current implementation I have separate scmi.dtsi > > > file which introduces scmi support for both XEN-based and > > > non-virtualized systems. Having 14 extra channels in the device-tree may > > > be confusing. > > > > I can see that while it would be ideal for Xen to see all 14+1 channels > > in device tree (on the host device tree), we wouldn't want to expose all > > of them to the domains, not even to dom0. > > > > How many channels do we want dom0 to see by the way? For this > > discussion, I'll just assume for now that dom0 only sees 1 channel like > > the domUs. > > For dom0 we need only one channel. > > > > > Now we have a problem: how do we go about "filtering" the > > "arm,scmi-shmem" device tree nodes? Which is also what you are asking > > below in point b). > > > > Xen will not need to filter "arm,scmi-shmem" node. It will just > create shmem node in Domain device-tree. I don't see any problem for xen > configuration here. > What bothers me here is that I set scmi configuration for platform dts, > not for xen or domu dts files. > So for example I have the following structure of the dts files for my > platform (r8a77961-salvator-xs): > * r8a77961-scmi.dtsi - this file includes all scmi related nodes and set > scmi_devid for the devices, that should use scmi. > * r8a77961-salvator-xs.dts - dts file which generates dtb for the platform. > It includes r8a77961-scmi.dtsi so I populate scmi to platform dtb, which > is used for system with no hypervisor. > * r8a77961-salvator-xs-xen.dts - dts file for xen which includes > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > * r8a77961-salvator-xs-domu.dts - dts file for DomU which includes > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > In this case r8a77961-salvator-xs.dtb r8a77961-salvator-xs-xen.dtb > r8a77961-salvator-xs-domu.dtb files will inherit 14+1 channel. > > I can give you a link to Merge request with this changes if you need it. > > For xen and domu dtb it is not a problem because all "arm,scmi-shmem" > nodes will be omitted and new will be generated for the domains. > > What bothers me is that r8a77961-salvator-xs.dtb will have 14 unused channels. > > Just got an idea while writing this: I can create only one > "arm,scmi-shmem" node in r8a77961-scmi.dtsi and add 14 more nodes, > needed for xen explicitly in r8a77961-salvator-xs-xen.dts. > > Then we will have valid configurations for all cases. > This can be a solution. What do you think? It is good that you brought this up because it helps me explain what I mean. And of course it is up to you where you place the nodes in the various dts files at your disposal. Either way it would work but I think they should belong to r8a77961-salvator-xs.dts. Generally the platform vendor (e.g. Xilinx) provides a device tree description of the platform to use including all the available resources and firmware interfaces. In your case it would be r8a77961-scmi.dtsi + r8a77961-salvator-xs.dts. This is what I call the "host device tree" below. Users should be able to boot a fully functional system using the host device tree pretty much "as is" to run Xen, Linux or any other software. Certainly the SCMI device tree description should be part of the host device tree, so in your case it would be r8a77961-salvator-xs.dts. And the description should include all 14+1 channels because this is the generic platform description -- we cannot know for sure how the users are going to use the system. This is why r8a77961-salvator-xs-xen.dts should be as small as possible or ideally inexistent. There shouldn't be a need for a special device tree modification to allow Xen to run. In reality, even at Xilinx we have something like r8a77961-salvator-xs-xen.dts, although it is really small. But I see that r8a77961-salvator-xs-xen.dts could be viewed as the device tree additions to run hypervisors and from that point of view it is more acceptable to place the 14 channels there. The biggest problem is r8a77961-salvator-xs-domu.dts: who is going to write it? And how? It wouldn't be provided by the platform vendor, so it is the user the one that has to find a way to write it. I know the user already has to write a partial DTB for device assignment, but any time the process is more complex than "copy the host device tree node for device XXX to the partial DTB" it is a problem. Errors are made and the system doesn't work. I think we don't want to make it even more difficult by having to manually produce the SCMI domU description too. The SCMI description for domU could be automatically generated by Xen, or libxl/xl. If that's an issue, then the SCMI description could be automatically generated by an external tool but I think it would make things more complex and harder to maintain. In short my point of view is: - r8a77961-scmi.dtsi + r8a77961-salvator-xs.dts should be as generic as possible so the SCMI nodes should have 14+1 channels - but putting the 14 channels in r8a77961-salvator-xs-xen.dts is still OKish - it is important that r8a77961-salvator-xs-domu.dts is automatically generated by Xen or libxl or another software tool > > > b) In case if we have all 15 channels, described in partial device-tree, > > > > I think you meant "described in the host device tree", right? > > > Yeah that's what I've meant. > > > > > we should not copy any node to the domain device-tree. I think it will > > > be better to generate arm,scmi-shmem node in the Domain device-tree. > > > > Yes, I think it makes sense for Xen to generate the "arm,scmi-shmem" > > device tree description for the DomU/Dom0 based on the channels > > allocated to the domain. > > > > > > > The problem is that arm,scmi-smc node, which is using arm,scmi-shmem > > > node can't be generated. I prefer it to be copied from the partial > > > device-tree because it includes some platform specific configuration, > > > such as func-id and list of the protocols (for example different > > > platforms may require different list of the protocols). So in this > > > case we will have 1 node copied and 1 node generated. > > > > > > I think even for dom0less we should use arm,scmi-smc node from the > > > device-tree because protocol configuration and funcid is related to the > > > platform. > > > > I am not sure I understood what you wrote. You are saying that the > > "arm,scmi-smc" node includes some platform specific configurations so > > it cannot be automatically generated by Xen (or by the tools) and > > instead it needs to be manually provided as part of the partial dtb for > > the domU. Is that correct? > > > > If so, I would like to understand the reasons behind it. Manual > > device tree editing is problematic. > > > > I looked for "func-id" in > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml but couldn't > > find any results. Do you have an example of the platform specific > > configuration or protocol configuration that would make it difficult to > > automatically generate the "arm,scmi-smc" node for the domains? > > Sorry, I used wrong term (used term from the specification), arm,smc-id > of cause. > > > > > Also, is this a problem just for approach #2 or also for approach #1? > > If it is a problem only for approach #2, then let's just go with > > approach #1. > > > > We can't copy "arm,scmi-smc" in both approaches. The difference is that > in the first approach we can copy both "arm,scmi-smc" and > "arm,scmi-shmem" nodes while in the second approach we should copy > "arm,scmi-smc", but we have to generate "arm,scmi-shmem" node. > > arm,scmi-smc node can't be generated because it includes properties and > configurations that depends from platform and should be get from the > device tree. > Here is "arm,scmi-smc" node expample: > firmware { > scmi { > compatible = "arm,scmi-smc" > arm,smc-id = <0x82000002>; > shmem = <&cpu_scp_shm>; > #address-cells = <1>; > #size-cells = <0>; > scmi_power: protocol@11 { > reg = <0x11>; > #power-domain-cells = <1>; > }; > > scmi_clock: protocol@14 { > ... > }; > > scmi_reset: protocol@16 { > ... > }; > ... > }; > }; > > It has 3 configurable options: > * arm,smc-id parameter, setting func_id for scmi protocol. This id can be > different for different platforms. > For example stm32mp1 architecture use different scm-id for different > agents: > https://github.com/ARM-software/arm-trusted-firmware/blob/0586c41b3f2d52aae847b7212e7b0c7e19197ea2/plat/st/stm32mp1/include/stm32mp1_smc.h#L39 > > * shmem which includes phandle to arm,scmi-shmem node. But this is not > a problem and can be updated. > > * list of the protocol subnodes. This is also configurable parameter, > not regs or names, but the number of the protocols. For example onle > platform can use power-domains/clock/resets via scmi, when another will > require volage-control and sensor-management to be added. > > Xen should know this parameters to be able to generate "arm,scmi-smc" node. > > Also we're currently discussing new scmi protocol with ARM: Pinctrl over > SCMI. > > It should allow domains to access pinctrl subsystem, placed in Firmware > through SCMI protocol. > scmi_pinctrl node will look like this: > > firmware { > scmi { > ... > scmi_pinctrl: protocol@18 { > reg = <0x18>; > #pinctrl-cells = <0>; > > i2c2_pins: i2c2 { > groups = <74>; /* i2c2_a */ > function = <15>; /* i2c2 */ > }; > > irq0_pins: irq0 { > groups = <81>; /* intc_ex_irq0 */ > function = <19>; /* intc_ex */ > }; > > avb_pins: avb { > mux { > /* avb_link, avb_mdio, avb_mii */ > groups = <17>, <21>, <22>; > function = <1>; /* avb */ > }; > > pins_mdio { > groups = <21>; /* avb_mdio */ > drive-strength = <24>; > }; > > pins_mii_tx { > /* PIN_AVB_TX_CTL, PIN_AVB_TXC, PIN_AVB_TD0, > PIN_AVB_TD1, PIN_AVB_TD2, PIN_AVB_TD3 */ > pins = <242>, <240>, <236>, <237>, <238>, <239>; > drive-strength = <12>; > }; > }; > ... > }; > }; > }; > > So "arm,scmi-smc" node will have even more platform specific settings. > > > > > > I prefer the second approach and will try to make it if it's OK to copy > > > arm,scmi-smc node from partial Device-tree and generate arm,scmi-shmem > > > node. > > > > > > What do you think about that? > > > > From a device tree specification perspective, I think both approaches > > are OK (with a minor comment on the first approach as I wrote above.) > > > > But from a Xen perspective I think it is important that we don't require > > the user to manually provide the SCMI configuration in the partial DTB. > > It would be better if we could generate it automatically from Xen or the > > tools (or even an independent script). Or copy the "arm,scmi-smc" node > > from the host device tree to the domU device tree without modifications. > > I think copy "arm,scmi-smc" node is the only option we have. > I'm not sure what do you mean under "host device tree" if you mean Xen > device-tree - then I think it will not cover the case with stm32mp1 I've > mentioned above. I think it will be better to copy "arm,scmi-smc" node > from Domu partial Device-tree to Domu device-tree. > So AGENT0 smc-id will be set in xen device-tree and copied to dom0 and > AGENT1 scm-is set in domu device-tree and copied to dom-u. > > Do you agree with my points? I think we are saying similar things, but we are getting mixed up with the terminology. Let's start from the basics :-) # Host device tree The device tree given to Xen at boot time. This is the device tree that Xen parses to discover what's available on the platform. In your case, it seems to include r8a77961-salvator-xs-xen.dts. # Partial DTB (Ignoring Dom0less) this is the small DTB that gets passed to xl with the "device_tree" option in the xl config file. It is copied verbatim to the domU device tree by xl/libxl. # Copy the "arm,scmi-smc" node from host device tree This means that the domU "arm,scmi-smc" node is an exact copy of the host device tree SCMI node. I don't think this is actually possible in most cases because the domU description is typically a bit different from the host description. For instance, the host description could include 14+1 channels while the domU description should only include 1 channel. # Copy the "arm,scmi-smc" node from the partial DTB This implies that somebody or something create an "arm,scmi-smc" node for the domU and placed it into the partial DTB. Then, Xen and/or xl/libxl will copy the node from the partial DTB to the DomU device tree. The main question in this case is: who is going to write the partial DTB? We dont want the user (i.e. a person) to have to manually write the SCMI description for the domU. It should be an automated tools that does it. At that point, it is easier if it is Xen or xl/libxl. Alternativaly, we could think of an external tool but I think it would make things more difficult to maintain. # Generate the "arm,scmi-smc" node for domUs When I write "generate the arm,scmi-smc node", I mean that Xen and libxl/xl will generate the "arm,scmi-smc" node for the domU. Thus, the node will not be copied from the partial DTB or from the device tree, instead, it should be created directly by Xen and/or libxl/xl. However, the domU "arm,scmi-smc" node could still be derived from the host device tree "arm,scmi-smc" node. In other words, Xen or xl/libxl would look at the host device tree "arm,scmi-smc" node, copy it to the domU device tree while making as many changes as necessary. The DomU "arm,scmi-smc" node doesn't have to be entirely fake and static. It could be dynamically created to match the host device tree description. I think this is the best option. # Conclusion I am suggesting that Xen and/or libxl automatically produce the "arm,scmi-smc" node for domUs based on the host device tree description and based on the channel mapped to the domU. This way, the user (a person) doesn't have to go and manually edit the domU partial DTB. > > So if using approach #1 allows us to automatically generate the > > "arm,scmi-smc" node for the guest, then I think it's best for sure. > > > > Summarizing all written above I would focus on the second approach > and put aside the first approach implementation. If you don't mind. Sure, that's fine by me > > > Also I wanted to mention that I'm not planning to make ARM_SCI support for > > > dom0less in terms of this patch series bacause I can't test > > > dom0less configuration for now. So let me know if some of my > > > functionality breaks dom0less. > > > > That's fine. I don't mean to scope-creep your patch series, which is > > extremely valuable as is. > > > > That said, I would be happy to provide you with a very simple dom0less > > configuration for your platform to enable you to test, or alternatively > > I could write a patch to add dom0less domU support if you are happy to > > help reviewing and testing it. > > I was thinking about making dom0less support in the different > patch-series because there are still questions to be discussed. > > For example, how arm,scmi-smc node will be generated for DomUs and how > the case, when scmi configuration is different for DomU1 and DomU2 (as > in case of stm32mp1 when smc-id is different) should be handled. > > What do you think about continue without dom0less support and discuss > dom0less once we done with the main part? That's OK, especially if you are happy to work on dom0less support later.
Hi Stefano, On Thu, Jan 20, 2022 at 02:29:41PM -0800, Stefano Stabellini wrote: > On Thu, 20 Jan 2022, Oleksii Moisieiev wrote: > > On Wed, Jan 19, 2022 at 05:28:21PM -0800, Stefano Stabellini wrote: > > > On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > > > > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > > > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > > --- > > > > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > > > > + > > > > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > > > > --- /dev/null > > > > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > > > > +config SCMI_SMC > > > > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > > > > + default n > > > > > > > > > > > > + depends on SCI > > > > > > > > > > > > + ---help--- > > > > > > > > > > > > + > > > > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > > > > + for communication. > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > > > > obj-y += sci.o > > > > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > > > > --- /dev/null > > > > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > > > > +/* > > > > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > + * > > > > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > > > > + * > > > > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > > > > + * > > > > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > > > > + * (at your option) any later version. > > > > > > > > > > > > + * > > > > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > > > > + */ > > > > > > > > > > > > + > > > > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > > > > +#include <asm/io.h> > > > > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > > > > +#include <xen/config.h> > > > > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > > > > +#include <xen/init.h> > > > > > > > > > > > > +#include <xen/err.h> > > > > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > > > > +#include <xen/list.h> > > > > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > > > > +#include <xen/string.h> > > > > > > > > > > > > +#include <xen/time.h> > > > > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > > > > + > > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > > > > + > > > > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > > > > + > > > > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > > > > + > > > > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > > > > channels: > > > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > > /* reserved region for scmi channels*/ > > > > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > > > > no-map; > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > > > > }; > > > > > > > > > > }; > > > > > > > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > > > > the current scmi channel: > > > > > > > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > }; > > > > > > > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > > > > assigned to this agent. > > > > > > > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > }; > > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > > }; > > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > > }; > > > > > > > > > ... > > > > > > > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > > > > there were some if, checking if memory weren't reserved. > > > > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > > > > shmem page. > > > > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > > > > which has no compatible string and provides no-map property. > > > > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > > > > space for the domain. > > > > > > > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > > > > and scmi_devid property to the > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > > > > Those node and property are needed only for Xen and useless for > > > > > > > > non-virtualized systems. I can add this node and property description to > > > > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > > > > What do you think about it? > > > > > > > > > > > > > > Reply below > > > > > > > > > > > > > > [...] > > > > > > > > > > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > > > > Documentation/devicetree/bindings (probably > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > > > > properties into generic bindings. It is a problem both from a > > > > > > > specification perspective (because it has hard to handle Xen specific > > > > > > > cases in fully generic bindings, especially as those bindings are > > > > > > > maintained as part of the Linux kernel) and from a user perspective > > > > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > > > > looks like a generic property that should be needed for the Linux SCMI > > > > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > > > > On non-virtualized systems - there is no need of this call, because OS > > > > > > is the only one entity, running on the system. > > > > > > > > > > OK. Even if it is only required for virtualized systems, I think that > > > > > scmi_devid is important enough that should be part of the upstream > > > > > binding. I think it is worth starting an email thread on the LKML with > > > > > Rob Herring and the SCMI maintainers to discuss the addition of > > > > > scmi_devid to the binding. > > > > > > > > > > > > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > > > > list of device_ids to dom.cfg, such as: > > > > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > > > > We can probably make hypercall taking devid list as input parameter. > > > > > > This will take only 1 hypercall to setup sci permissions. > > > > > > > > > > But how would a user know which are the right SCMI IDs to add to the > > > > > sci_devs list? Would the user have to go and read the reference manual > > > > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > > > > If that is the case, then I think that it would be better to add > > > > > scmi_devid to device tree. > > > > > > > > > > In general, I think this configuration should happen automatically > > > > > without user intervention. The user should just specify "enable SCMI" > > > > > and it should work. > > > > > > > > > > > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > > > > way, I don't think we should need linux,scmi_mem. > > > > > > > > > > > > This requires further investigation. I will try to make implementation > > > > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > > > > reuslts with you. > > > > > > > > > > OK, thanks. > > > > > > > > Hi Stefano, > > > > > > > > As I did some investigation about using reserved-memory area > > > > linux,scmi_mem and now I need your advice. > > > > > > > > I see 2 possible implementations for now: > > > > 1) Add memory-region parameter to cpu_scp_shm node which points to the > > > > reserved memory region. > > > > So device-tree will look like this: > > > > > > > > reserved-memory { > > > > /* reserved region for scmi channels*/ > > > > scmi_memory: region@53FF0000{ > > > > no-map; > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > }; > > > > }; > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > compatible = "arm,scmi-shmem"; > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > memory-region = <&scmi_memory>; > > > > }; > > > > > > > > So cpu_scp_shm node has a reference to scmi_memory region. This mean > > > > that xen can find reserved memory region without adding additional names > > > > to the device-tree bindings. > > > > memory-region parameter as a reference to reserved memory and region > > > > creation described in: > > > > https://urldefense.com/v3/__https://github.com/torvalds/linux/blob/v5.15/Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt__;!!GF_29dbcQIUBPA!k6x19x1gYF1CPlgAZj7std3ifqhq-9DXvuF0nwonNPUwMzZpYHYbrRJziJrgdFIOjyan$ [github[.]com] > > > > > > > > This approach I've implemented already and it works. > > > > > > This approach would require a discussion with the upstream device tree > > > maintainers. Likely, we would need to add a note about the usage of the > > > "memory-region" property to arm,scmi.yaml. > > > > > > Also, I have the feeling that they would ask to add the "memory-region" > > > property directly to the "arm,scmi-smc" node, as an alternative (or > > > in addition) to the existing "shmem" property. > > > > > > That said, from my point of view this approach is also a viable option. > > > I don't see any major problems. > > > > > > The main question (after reading everything else that you wrote below) > > > is whether the "arm,scmi-smc" node in this case could be automatically > > > generated. > > > > > > > arm,scmi-smc node can be generated in both cases. I think I'd leave it > > as backup in case if the second approach will not work. > > > > > > > > > 2) The second approach is the format you suggested: > > > > > > > > > reserved-memory { > > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > }; > > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > > }; > > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > > }; > > > > > > > > > ... > > > > > > > > This approach has an advantage that xen ARM_SCI driver do not know about > > > > how channels are placed in the reserved memory, but introduces some > > > > disadvantages: > > > > a) We provide extra 14 (in our case) arm,scmi-shmem nodes which are not used > > > > in the device-tree. In current implementation I have separate scmi.dtsi > > > > file which introduces scmi support for both XEN-based and > > > > non-virtualized systems. Having 14 extra channels in the device-tree may > > > > be confusing. > > > > > > I can see that while it would be ideal for Xen to see all 14+1 channels > > > in device tree (on the host device tree), we wouldn't want to expose all > > > of them to the domains, not even to dom0. > > > > > > How many channels do we want dom0 to see by the way? For this > > > discussion, I'll just assume for now that dom0 only sees 1 channel like > > > the domUs. > > > > For dom0 we need only one channel. > > > > > > > > Now we have a problem: how do we go about "filtering" the > > > "arm,scmi-shmem" device tree nodes? Which is also what you are asking > > > below in point b). > > > > > > > Xen will not need to filter "arm,scmi-shmem" node. It will just > > create shmem node in Domain device-tree. I don't see any problem for xen > > configuration here. > > What bothers me here is that I set scmi configuration for platform dts, > > not for xen or domu dts files. > > So for example I have the following structure of the dts files for my > > platform (r8a77961-salvator-xs): > > * r8a77961-scmi.dtsi - this file includes all scmi related nodes and set > > scmi_devid for the devices, that should use scmi. > > * r8a77961-salvator-xs.dts - dts file which generates dtb for the platform. > > It includes r8a77961-scmi.dtsi so I populate scmi to platform dtb, which > > is used for system with no hypervisor. > > * r8a77961-salvator-xs-xen.dts - dts file for xen which includes > > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > * r8a77961-salvator-xs-domu.dts - dts file for DomU which includes > > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > > > In this case r8a77961-salvator-xs.dtb r8a77961-salvator-xs-xen.dtb > > r8a77961-salvator-xs-domu.dtb files will inherit 14+1 channel. > > > > I can give you a link to Merge request with this changes if you need it. > > > > For xen and domu dtb it is not a problem because all "arm,scmi-shmem" > > nodes will be omitted and new will be generated for the domains. > > > > What bothers me is that r8a77961-salvator-xs.dtb will have 14 unused channels. > > > > Just got an idea while writing this: I can create only one > > "arm,scmi-shmem" node in r8a77961-scmi.dtsi and add 14 more nodes, > > needed for xen explicitly in r8a77961-salvator-xs-xen.dts. > > > > Then we will have valid configurations for all cases. > > This can be a solution. What do you think? > > It is good that you brought this up because it helps me explain what I > mean. And of course it is up to you where you place the nodes in the > various dts files at your disposal. Either way it would work but I think > they should belong to r8a77961-salvator-xs.dts. > > Generally the platform vendor (e.g. Xilinx) provides a device tree > description of the platform to use including all the available resources > and firmware interfaces. In your case it would be r8a77961-scmi.dtsi + > r8a77961-salvator-xs.dts. This is what I call the "host device tree" > below. Users should be able to boot a fully functional system using the > host device tree pretty much "as is" to run Xen, Linux or any other > software. > > Certainly the SCMI device tree description should be part of the host > device tree, so in your case it would be r8a77961-salvator-xs.dts. And > the description should include all 14+1 channels because this is the > generic platform description -- we cannot know for sure how the users > are going to use the system. > > This is why r8a77961-salvator-xs-xen.dts should be as small as possible > or ideally inexistent. There shouldn't be a need for a special device > tree modification to allow Xen to run. In reality, even at Xilinx we > have something like r8a77961-salvator-xs-xen.dts, although it is really > small. > > But I see that r8a77961-salvator-xs-xen.dts could be viewed as the > device tree additions to run hypervisors and from that point of view it > is more acceptable to place the 14 channels there. > > The biggest problem is r8a77961-salvator-xs-domu.dts: who is going to > write it? And how? It wouldn't be provided by the platform vendor, so it > is the user the one that has to find a way to write it. > > I know the user already has to write a partial DTB for device > assignment, but any time the process is more complex than "copy the host > device tree node for device XXX to the partial DTB" it is a problem. > Errors are made and the system doesn't work. > > I think we don't want to make it even more difficult by having to > manually produce the SCMI domU description too. The SCMI description for > domU could be automatically generated by Xen, or libxl/xl. If that's an > issue, then the SCMI description could be automatically generated by an > external tool but I think it would make things more complex and harder > to maintain. > > In short my point of view is: > - r8a77961-scmi.dtsi + r8a77961-salvator-xs.dts should be as generic as > possible so the SCMI nodes should have 14+1 channels > - but putting the 14 channels in r8a77961-salvator-xs-xen.dts is still > OKish > - it is important that r8a77961-salvator-xs-domu.dts is automatically > generated by Xen or libxl or another software tool > Thank you for the detailed response. I'll put all 14+1 channels to r8a77961-salvator-xs.dts then. I've described my thoughts about generation of the arm,scmi-smc node below. > > > > > b) In case if we have all 15 channels, described in partial device-tree, > > > > > > I think you meant "described in the host device tree", right? > > > > > Yeah that's what I've meant. > > > > > > > we should not copy any node to the domain device-tree. I think it will > > > > be better to generate arm,scmi-shmem node in the Domain device-tree. > > > > > > Yes, I think it makes sense for Xen to generate the "arm,scmi-shmem" > > > device tree description for the DomU/Dom0 based on the channels > > > allocated to the domain. > > > > > > > > > > The problem is that arm,scmi-smc node, which is using arm,scmi-shmem > > > > node can't be generated. I prefer it to be copied from the partial > > > > device-tree because it includes some platform specific configuration, > > > > such as func-id and list of the protocols (for example different > > > > platforms may require different list of the protocols). So in this > > > > case we will have 1 node copied and 1 node generated. > > > > > > > > I think even for dom0less we should use arm,scmi-smc node from the > > > > device-tree because protocol configuration and funcid is related to the > > > > platform. > > > > > > I am not sure I understood what you wrote. You are saying that the > > > "arm,scmi-smc" node includes some platform specific configurations so > > > it cannot be automatically generated by Xen (or by the tools) and > > > instead it needs to be manually provided as part of the partial dtb for > > > the domU. Is that correct? > > > > > > If so, I would like to understand the reasons behind it. Manual > > > device tree editing is problematic. > > > > > > I looked for "func-id" in > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml but couldn't > > > find any results. Do you have an example of the platform specific > > > configuration or protocol configuration that would make it difficult to > > > automatically generate the "arm,scmi-smc" node for the domains? > > > > Sorry, I used wrong term (used term from the specification), arm,smc-id > > of cause. > > > > > > > > Also, is this a problem just for approach #2 or also for approach #1? > > > If it is a problem only for approach #2, then let's just go with > > > approach #1. > > > > > > > We can't copy "arm,scmi-smc" in both approaches. The difference is that > > in the first approach we can copy both "arm,scmi-smc" and > > "arm,scmi-shmem" nodes while in the second approach we should copy > > "arm,scmi-smc", but we have to generate "arm,scmi-shmem" node. > > > > arm,scmi-smc node can't be generated because it includes properties and > > configurations that depends from platform and should be get from the > > device tree. > > Here is "arm,scmi-smc" node expample: > > firmware { > > scmi { > > compatible = "arm,scmi-smc" > > arm,smc-id = <0x82000002>; > > shmem = <&cpu_scp_shm>; > > #address-cells = <1>; > > #size-cells = <0>; > > scmi_power: protocol@11 { > > reg = <0x11>; > > #power-domain-cells = <1>; > > }; > > > > scmi_clock: protocol@14 { > > ... > > }; > > > > scmi_reset: protocol@16 { > > ... > > }; > > ... > > }; > > }; > > > > It has 3 configurable options: > > * arm,smc-id parameter, setting func_id for scmi protocol. This id can be > > different for different platforms. > > For example stm32mp1 architecture use different scm-id for different > > agents: > > https://urldefense.com/v3/__https://github.com/ARM-software/arm-trusted-firmware/blob/0586c41b3f2d52aae847b7212e7b0c7e19197ea2/plat/st/stm32mp1/include/stm32mp1_smc.h*L39__;Iw!!GF_29dbcQIUBPA!mTRUjtSg19iVaYo3Cgjop5ckPWqKsHVo1EZCEA1zCbod9KpNSXX291A8vAuNdTCr46MA$ [github[.]com] > > > > * shmem which includes phandle to arm,scmi-shmem node. But this is not > > a problem and can be updated. > > > > * list of the protocol subnodes. This is also configurable parameter, > > not regs or names, but the number of the protocols. For example onle > > platform can use power-domains/clock/resets via scmi, when another will > > require volage-control and sensor-management to be added. > > > > Xen should know this parameters to be able to generate "arm,scmi-smc" node. > > > > Also we're currently discussing new scmi protocol with ARM: Pinctrl over > > SCMI. > > > > It should allow domains to access pinctrl subsystem, placed in Firmware > > through SCMI protocol. > > scmi_pinctrl node will look like this: > > > > firmware { > > scmi { > > ... > > scmi_pinctrl: protocol@18 { > > reg = <0x18>; > > #pinctrl-cells = <0>; > > > > i2c2_pins: i2c2 { > > groups = <74>; /* i2c2_a */ > > function = <15>; /* i2c2 */ > > }; > > > > irq0_pins: irq0 { > > groups = <81>; /* intc_ex_irq0 */ > > function = <19>; /* intc_ex */ > > }; > > > > avb_pins: avb { > > mux { > > /* avb_link, avb_mdio, avb_mii */ > > groups = <17>, <21>, <22>; > > function = <1>; /* avb */ > > }; > > > > pins_mdio { > > groups = <21>; /* avb_mdio */ > > drive-strength = <24>; > > }; > > > > pins_mii_tx { > > /* PIN_AVB_TX_CTL, PIN_AVB_TXC, PIN_AVB_TD0, > > PIN_AVB_TD1, PIN_AVB_TD2, PIN_AVB_TD3 */ > > pins = <242>, <240>, <236>, <237>, <238>, <239>; > > drive-strength = <12>; > > }; > > }; > > ... > > }; > > }; > > }; > > > > So "arm,scmi-smc" node will have even more platform specific settings. > > > > > > > > > I prefer the second approach and will try to make it if it's OK to copy > > > > arm,scmi-smc node from partial Device-tree and generate arm,scmi-shmem > > > > node. > > > > > > > > What do you think about that? > > > > > > From a device tree specification perspective, I think both approaches > > > are OK (with a minor comment on the first approach as I wrote above.) > > > > > > But from a Xen perspective I think it is important that we don't require > > > the user to manually provide the SCMI configuration in the partial DTB. > > > It would be better if we could generate it automatically from Xen or the > > > tools (or even an independent script). Or copy the "arm,scmi-smc" node > > > from the host device tree to the domU device tree without modifications. > > > > I think copy "arm,scmi-smc" node is the only option we have. > > I'm not sure what do you mean under "host device tree" if you mean Xen > > device-tree - then I think it will not cover the case with stm32mp1 I've > > mentioned above. I think it will be better to copy "arm,scmi-smc" node > > from Domu partial Device-tree to Domu device-tree. > > So AGENT0 smc-id will be set in xen device-tree and copied to dom0 and > > AGENT1 scm-is set in domu device-tree and copied to dom-u. > > > > Do you agree with my points? > > I think we are saying similar things, but we are getting mixed up with > the terminology. Let's start from the basics :-) > > # Host device tree > The device tree given to Xen at boot time. This is the device tree that > Xen parses to discover what's available on the platform. In your case, > it seems to include r8a77961-salvator-xs-xen.dts. > > # Partial DTB > (Ignoring Dom0less) this is the small DTB that gets passed to xl with > the "device_tree" option in the xl config file. It is copied verbatim > to the domU device tree by xl/libxl. > > # Copy the "arm,scmi-smc" node from host device tree > This means that the domU "arm,scmi-smc" node is an exact copy of the > host device tree SCMI node. I don't think this is actually possible in > most cases because the domU description is typically a bit different > from the host description. For instance, the host description could > include 14+1 channels while the domU description should only include 1 > channel. > > # Copy the "arm,scmi-smc" node from the partial DTB > This implies that somebody or something create an "arm,scmi-smc" node > for the domU and placed it into the partial DTB. Then, Xen and/or > xl/libxl will copy the node from the partial DTB to the DomU device > tree. The main question in this case is: who is going to write the > partial DTB? We dont want the user (i.e. a person) to have to manually > write the SCMI description for the domU. It should be an automated tools > that does it. At that point, it is easier if it is Xen or xl/libxl. > Alternativaly, we could think of an external tool but I think it would > make things more difficult to maintain. > > # Generate the "arm,scmi-smc" node for domUs > When I write "generate the arm,scmi-smc node", I mean that Xen and > libxl/xl will generate the "arm,scmi-smc" node for the domU. Thus, the > node will not be copied from the partial DTB or from the device tree, > instead, it should be created directly by Xen and/or libxl/xl. > > However, the domU "arm,scmi-smc" node could still be derived from the > host device tree "arm,scmi-smc" node. In other words, Xen or xl/libxl > would look at the host device tree "arm,scmi-smc" node, copy it to the > domU device tree while making as many changes as necessary. > > The DomU "arm,scmi-smc" node doesn't have to be entirely fake and > static. It could be dynamically created to match the host device tree > description. I think this is the best option. > > > # Conclusion > I am suggesting that Xen and/or libxl automatically produce the > "arm,scmi-smc" node for domUs based on the host device tree description > and based on the channel mapped to the domU. This way, the user (a > person) doesn't have to go and manually edit the domU partial DTB. > That sounds reasonable. The problem is that arm,scmi-smc node can be copmlicated and include a lot of configuration. Also for different mediators this node can be different. As I inderstand, there is no mechanism for xl to access host device-tree right now. Correct me if I'm wrong. I see the following way we can generate arm,scmi-smc node for DomU: We say that if scmi-smc mediator is enabled - then Dom0 is configured to use SCMI. This means that Dom0 device-tree will have arm,scmi-smc node and it can be reached from the userspace. In this case xl can use infromation from /proc/device-tree/firmware/scmi to generate arm,scmi-smc node for DomU. But in this case xl should know the exact path of scmi node. Or we can generate some special node, called "shared" in Dom0 device-tree which will include copy of the arm,scmi-smc node, which can be used for domains. In this case xl can scan /proc/device-tree/shared node and find arm,scmi-smc copatible node and use it to generate arm,scmi-smc node for DomU. Also this can be used for another features in future. What do you think about this? > > > > So if using approach #1 allows us to automatically generate the > > > "arm,scmi-smc" node for the guest, then I think it's best for sure. > > > > > > > Summarizing all written above I would focus on the second approach > > and put aside the first approach implementation. If you don't mind. > > Sure, that's fine by me > > > > > > Also I wanted to mention that I'm not planning to make ARM_SCI support for > > > > dom0less in terms of this patch series bacause I can't test > > > > dom0less configuration for now. So let me know if some of my > > > > functionality breaks dom0less. > > > > > > That's fine. I don't mean to scope-creep your patch series, which is > > > extremely valuable as is. > > > > > > That said, I would be happy to provide you with a very simple dom0less > > > configuration for your platform to enable you to test, or alternatively > > > I could write a patch to add dom0less domU support if you are happy to > > > help reviewing and testing it. > > > > I was thinking about making dom0less support in the different > > patch-series because there are still questions to be discussed. > > > > For example, how arm,scmi-smc node will be generated for DomUs and how > > the case, when scmi configuration is different for DomU1 and DomU2 (as > > in case of stm32mp1 when smc-id is different) should be handled. > > > > What do you think about continue without dom0less support and discuss > > dom0less once we done with the main part? > > That's OK, especially if you are happy to work on dom0less support > later. Sure I will be happy to work on dom0less support later.
On Fri, 21 Jan 2022, Oleksii Moisieiev wrote: > On Thu, Jan 20, 2022 at 02:29:41PM -0800, Stefano Stabellini wrote: > > On Thu, 20 Jan 2022, Oleksii Moisieiev wrote: > > > On Wed, Jan 19, 2022 at 05:28:21PM -0800, Stefano Stabellini wrote: > > > > On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > > > > > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > > > > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > > > --- > > > > > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > > > > > + > > > > > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > > > > > --- /dev/null > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > > > > > +config SCMI_SMC > > > > > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > > > > > + default n > > > > > > > > > > > > > + depends on SCI > > > > > > > > > > > > > + ---help--- > > > > > > > > > > > > > + > > > > > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > > > > > + for communication. > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > > > > > obj-y += sci.o > > > > > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > > > > > --- /dev/null > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > > > > > +/* > > > > > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > + * > > > > > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > > > > > + * > > > > > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > > > > > + * > > > > > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > > > > > + * (at your option) any later version. > > > > > > > > > > > > > + * > > > > > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > > > > > + */ > > > > > > > > > > > > > + > > > > > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > > > > > +#include <asm/io.h> > > > > > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > > > > > +#include <xen/config.h> > > > > > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > > > > > +#include <xen/init.h> > > > > > > > > > > > > > +#include <xen/err.h> > > > > > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > > > > > +#include <xen/list.h> > > > > > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > > > > > +#include <xen/string.h> > > > > > > > > > > > > > +#include <xen/time.h> > > > > > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > > > > > + > > > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > > > > > + > > > > > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > > > > > + > > > > > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > > > > > + > > > > > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > > > > > channels: > > > > > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > > > /* reserved region for scmi channels*/ > > > > > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > > > > > no-map; > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > > > > > }; > > > > > > > > > > > }; > > > > > > > > > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > > > > > the current scmi channel: > > > > > > > > > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > > }; > > > > > > > > > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > > > > > assigned to this agent. > > > > > > > > > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > }; > > > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > > > }; > > > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > > > }; > > > > > > > > > > ... > > > > > > > > > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > > > > > there were some if, checking if memory weren't reserved. > > > > > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > > > > > shmem page. > > > > > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > > > > > which has no compatible string and provides no-map property. > > > > > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > > > > > space for the domain. > > > > > > > > > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > > > > > and scmi_devid property to the > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > > > > > Those node and property are needed only for Xen and useless for > > > > > > > > > non-virtualized systems. I can add this node and property description to > > > > > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > > > > > What do you think about it? > > > > > > > > > > > > > > > > Reply below > > > > > > > > > > > > > > > > [...] > > > > > > > > > > > > > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > > > > > Documentation/devicetree/bindings (probably > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > > > > > properties into generic bindings. It is a problem both from a > > > > > > > > specification perspective (because it has hard to handle Xen specific > > > > > > > > cases in fully generic bindings, especially as those bindings are > > > > > > > > maintained as part of the Linux kernel) and from a user perspective > > > > > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > > > > > looks like a generic property that should be needed for the Linux SCMI > > > > > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > > > > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > > > > > On non-virtualized systems - there is no need of this call, because OS > > > > > > > is the only one entity, running on the system. > > > > > > > > > > > > OK. Even if it is only required for virtualized systems, I think that > > > > > > scmi_devid is important enough that should be part of the upstream > > > > > > binding. I think it is worth starting an email thread on the LKML with > > > > > > Rob Herring and the SCMI maintainers to discuss the addition of > > > > > > scmi_devid to the binding. > > > > > > > > > > > > > > > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > > > > > list of device_ids to dom.cfg, such as: > > > > > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > > > > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > > > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > > > > > We can probably make hypercall taking devid list as input parameter. > > > > > > > This will take only 1 hypercall to setup sci permissions. > > > > > > > > > > > > But how would a user know which are the right SCMI IDs to add to the > > > > > > sci_devs list? Would the user have to go and read the reference manual > > > > > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > > > > > If that is the case, then I think that it would be better to add > > > > > > scmi_devid to device tree. > > > > > > > > > > > > In general, I think this configuration should happen automatically > > > > > > without user intervention. The user should just specify "enable SCMI" > > > > > > and it should work. > > > > > > > > > > > > > > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > > > > > way, I don't think we should need linux,scmi_mem. > > > > > > > > > > > > > > This requires further investigation. I will try to make implementation > > > > > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > > > > > reuslts with you. > > > > > > > > > > > > OK, thanks. > > > > > > > > > > Hi Stefano, > > > > > > > > > > As I did some investigation about using reserved-memory area > > > > > linux,scmi_mem and now I need your advice. > > > > > > > > > > I see 2 possible implementations for now: > > > > > 1) Add memory-region parameter to cpu_scp_shm node which points to the > > > > > reserved memory region. > > > > > So device-tree will look like this: > > > > > > > > > > reserved-memory { > > > > > /* reserved region for scmi channels*/ > > > > > scmi_memory: region@53FF0000{ > > > > > no-map; > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > }; > > > > > }; > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > compatible = "arm,scmi-shmem"; > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > memory-region = <&scmi_memory>; > > > > > }; > > > > > > > > > > So cpu_scp_shm node has a reference to scmi_memory region. This mean > > > > > that xen can find reserved memory region without adding additional names > > > > > to the device-tree bindings. > > > > > memory-region parameter as a reference to reserved memory and region > > > > > creation described in: > > > > > https://urldefense.com/v3/__https://github.com/torvalds/linux/blob/v5.15/Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt__;!!GF_29dbcQIUBPA!k6x19x1gYF1CPlgAZj7std3ifqhq-9DXvuF0nwonNPUwMzZpYHYbrRJziJrgdFIOjyan$ [github[.]com] > > > > > > > > > > This approach I've implemented already and it works. > > > > > > > > This approach would require a discussion with the upstream device tree > > > > maintainers. Likely, we would need to add a note about the usage of the > > > > "memory-region" property to arm,scmi.yaml. > > > > > > > > Also, I have the feeling that they would ask to add the "memory-region" > > > > property directly to the "arm,scmi-smc" node, as an alternative (or > > > > in addition) to the existing "shmem" property. > > > > > > > > That said, from my point of view this approach is also a viable option. > > > > I don't see any major problems. > > > > > > > > The main question (after reading everything else that you wrote below) > > > > is whether the "arm,scmi-smc" node in this case could be automatically > > > > generated. > > > > > > > > > > arm,scmi-smc node can be generated in both cases. I think I'd leave it > > > as backup in case if the second approach will not work. > > > > > > > > > > > > 2) The second approach is the format you suggested: > > > > > > > > > > reserved-memory { > > > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > }; > > > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > > > }; > > > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > > > }; > > > > > > > > > > ... > > > > > > > > > > This approach has an advantage that xen ARM_SCI driver do not know about > > > > > how channels are placed in the reserved memory, but introduces some > > > > > disadvantages: > > > > > a) We provide extra 14 (in our case) arm,scmi-shmem nodes which are not used > > > > > in the device-tree. In current implementation I have separate scmi.dtsi > > > > > file which introduces scmi support for both XEN-based and > > > > > non-virtualized systems. Having 14 extra channels in the device-tree may > > > > > be confusing. > > > > > > > > I can see that while it would be ideal for Xen to see all 14+1 channels > > > > in device tree (on the host device tree), we wouldn't want to expose all > > > > of them to the domains, not even to dom0. > > > > > > > > How many channels do we want dom0 to see by the way? For this > > > > discussion, I'll just assume for now that dom0 only sees 1 channel like > > > > the domUs. > > > > > > For dom0 we need only one channel. > > > > > > > > > > > Now we have a problem: how do we go about "filtering" the > > > > "arm,scmi-shmem" device tree nodes? Which is also what you are asking > > > > below in point b). > > > > > > > > > > Xen will not need to filter "arm,scmi-shmem" node. It will just > > > create shmem node in Domain device-tree. I don't see any problem for xen > > > configuration here. > > > What bothers me here is that I set scmi configuration for platform dts, > > > not for xen or domu dts files. > > > So for example I have the following structure of the dts files for my > > > platform (r8a77961-salvator-xs): > > > * r8a77961-scmi.dtsi - this file includes all scmi related nodes and set > > > scmi_devid for the devices, that should use scmi. > > > * r8a77961-salvator-xs.dts - dts file which generates dtb for the platform. > > > It includes r8a77961-scmi.dtsi so I populate scmi to platform dtb, which > > > is used for system with no hypervisor. > > > * r8a77961-salvator-xs-xen.dts - dts file for xen which includes > > > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > > * r8a77961-salvator-xs-domu.dts - dts file for DomU which includes > > > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > > > > > In this case r8a77961-salvator-xs.dtb r8a77961-salvator-xs-xen.dtb > > > r8a77961-salvator-xs-domu.dtb files will inherit 14+1 channel. > > > > > > I can give you a link to Merge request with this changes if you need it. > > > > > > For xen and domu dtb it is not a problem because all "arm,scmi-shmem" > > > nodes will be omitted and new will be generated for the domains. > > > > > > What bothers me is that r8a77961-salvator-xs.dtb will have 14 unused channels. > > > > > > Just got an idea while writing this: I can create only one > > > "arm,scmi-shmem" node in r8a77961-scmi.dtsi and add 14 more nodes, > > > needed for xen explicitly in r8a77961-salvator-xs-xen.dts. > > > > > > Then we will have valid configurations for all cases. > > > This can be a solution. What do you think? > > > > It is good that you brought this up because it helps me explain what I > > mean. And of course it is up to you where you place the nodes in the > > various dts files at your disposal. Either way it would work but I think > > they should belong to r8a77961-salvator-xs.dts. > > > > Generally the platform vendor (e.g. Xilinx) provides a device tree > > description of the platform to use including all the available resources > > and firmware interfaces. In your case it would be r8a77961-scmi.dtsi + > > r8a77961-salvator-xs.dts. This is what I call the "host device tree" > > below. Users should be able to boot a fully functional system using the > > host device tree pretty much "as is" to run Xen, Linux or any other > > software. > > > > Certainly the SCMI device tree description should be part of the host > > device tree, so in your case it would be r8a77961-salvator-xs.dts. And > > the description should include all 14+1 channels because this is the > > generic platform description -- we cannot know for sure how the users > > are going to use the system. > > > > This is why r8a77961-salvator-xs-xen.dts should be as small as possible > > or ideally inexistent. There shouldn't be a need for a special device > > tree modification to allow Xen to run. In reality, even at Xilinx we > > have something like r8a77961-salvator-xs-xen.dts, although it is really > > small. > > > > But I see that r8a77961-salvator-xs-xen.dts could be viewed as the > > device tree additions to run hypervisors and from that point of view it > > is more acceptable to place the 14 channels there. > > > > The biggest problem is r8a77961-salvator-xs-domu.dts: who is going to > > write it? And how? It wouldn't be provided by the platform vendor, so it > > is the user the one that has to find a way to write it. > > > > I know the user already has to write a partial DTB for device > > assignment, but any time the process is more complex than "copy the host > > device tree node for device XXX to the partial DTB" it is a problem. > > Errors are made and the system doesn't work. > > > > I think we don't want to make it even more difficult by having to > > manually produce the SCMI domU description too. The SCMI description for > > domU could be automatically generated by Xen, or libxl/xl. If that's an > > issue, then the SCMI description could be automatically generated by an > > external tool but I think it would make things more complex and harder > > to maintain. > > > > In short my point of view is: > > - r8a77961-scmi.dtsi + r8a77961-salvator-xs.dts should be as generic as > > possible so the SCMI nodes should have 14+1 channels > > - but putting the 14 channels in r8a77961-salvator-xs-xen.dts is still > > OKish > > - it is important that r8a77961-salvator-xs-domu.dts is automatically > > generated by Xen or libxl or another software tool > > > > Thank you for the detailed response. I'll put all 14+1 channels to > r8a77961-salvator-xs.dts then. > I've described my thoughts about generation of the arm,scmi-smc node below. > > > > > > > > b) In case if we have all 15 channels, described in partial device-tree, > > > > > > > > I think you meant "described in the host device tree", right? > > > > > > > Yeah that's what I've meant. > > > > > > > > > we should not copy any node to the domain device-tree. I think it will > > > > > be better to generate arm,scmi-shmem node in the Domain device-tree. > > > > > > > > Yes, I think it makes sense for Xen to generate the "arm,scmi-shmem" > > > > device tree description for the DomU/Dom0 based on the channels > > > > allocated to the domain. > > > > > > > > > > > > > The problem is that arm,scmi-smc node, which is using arm,scmi-shmem > > > > > node can't be generated. I prefer it to be copied from the partial > > > > > device-tree because it includes some platform specific configuration, > > > > > such as func-id and list of the protocols (for example different > > > > > platforms may require different list of the protocols). So in this > > > > > case we will have 1 node copied and 1 node generated. > > > > > > > > > > I think even for dom0less we should use arm,scmi-smc node from the > > > > > device-tree because protocol configuration and funcid is related to the > > > > > platform. > > > > > > > > I am not sure I understood what you wrote. You are saying that the > > > > "arm,scmi-smc" node includes some platform specific configurations so > > > > it cannot be automatically generated by Xen (or by the tools) and > > > > instead it needs to be manually provided as part of the partial dtb for > > > > the domU. Is that correct? > > > > > > > > If so, I would like to understand the reasons behind it. Manual > > > > device tree editing is problematic. > > > > > > > > I looked for "func-id" in > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml but couldn't > > > > find any results. Do you have an example of the platform specific > > > > configuration or protocol configuration that would make it difficult to > > > > automatically generate the "arm,scmi-smc" node for the domains? > > > > > > Sorry, I used wrong term (used term from the specification), arm,smc-id > > > of cause. > > > > > > > > > > > Also, is this a problem just for approach #2 or also for approach #1? > > > > If it is a problem only for approach #2, then let's just go with > > > > approach #1. > > > > > > > > > > We can't copy "arm,scmi-smc" in both approaches. The difference is that > > > in the first approach we can copy both "arm,scmi-smc" and > > > "arm,scmi-shmem" nodes while in the second approach we should copy > > > "arm,scmi-smc", but we have to generate "arm,scmi-shmem" node. > > > > > > arm,scmi-smc node can't be generated because it includes properties and > > > configurations that depends from platform and should be get from the > > > device tree. > > > Here is "arm,scmi-smc" node expample: > > > firmware { > > > scmi { > > > compatible = "arm,scmi-smc" > > > arm,smc-id = <0x82000002>; > > > shmem = <&cpu_scp_shm>; > > > #address-cells = <1>; > > > #size-cells = <0>; > > > scmi_power: protocol@11 { > > > reg = <0x11>; > > > #power-domain-cells = <1>; > > > }; > > > > > > scmi_clock: protocol@14 { > > > ... > > > }; > > > > > > scmi_reset: protocol@16 { > > > ... > > > }; > > > ... > > > }; > > > }; > > > > > > It has 3 configurable options: > > > * arm,smc-id parameter, setting func_id for scmi protocol. This id can be > > > different for different platforms. > > > For example stm32mp1 architecture use different scm-id for different > > > agents: > > > https://urldefense.com/v3/__https://github.com/ARM-software/arm-trusted-firmware/blob/0586c41b3f2d52aae847b7212e7b0c7e19197ea2/plat/st/stm32mp1/include/stm32mp1_smc.h*L39__;Iw!!GF_29dbcQIUBPA!mTRUjtSg19iVaYo3Cgjop5ckPWqKsHVo1EZCEA1zCbod9KpNSXX291A8vAuNdTCr46MA$ [github[.]com] > > > > > > * shmem which includes phandle to arm,scmi-shmem node. But this is not > > > a problem and can be updated. > > > > > > * list of the protocol subnodes. This is also configurable parameter, > > > not regs or names, but the number of the protocols. For example onle > > > platform can use power-domains/clock/resets via scmi, when another will > > > require volage-control and sensor-management to be added. > > > > > > Xen should know this parameters to be able to generate "arm,scmi-smc" node. > > > > > > Also we're currently discussing new scmi protocol with ARM: Pinctrl over > > > SCMI. > > > > > > It should allow domains to access pinctrl subsystem, placed in Firmware > > > through SCMI protocol. > > > scmi_pinctrl node will look like this: > > > > > > firmware { > > > scmi { > > > ... > > > scmi_pinctrl: protocol@18 { > > > reg = <0x18>; > > > #pinctrl-cells = <0>; > > > > > > i2c2_pins: i2c2 { > > > groups = <74>; /* i2c2_a */ > > > function = <15>; /* i2c2 */ > > > }; > > > > > > irq0_pins: irq0 { > > > groups = <81>; /* intc_ex_irq0 */ > > > function = <19>; /* intc_ex */ > > > }; > > > > > > avb_pins: avb { > > > mux { > > > /* avb_link, avb_mdio, avb_mii */ > > > groups = <17>, <21>, <22>; > > > function = <1>; /* avb */ > > > }; > > > > > > pins_mdio { > > > groups = <21>; /* avb_mdio */ > > > drive-strength = <24>; > > > }; > > > > > > pins_mii_tx { > > > /* PIN_AVB_TX_CTL, PIN_AVB_TXC, PIN_AVB_TD0, > > > PIN_AVB_TD1, PIN_AVB_TD2, PIN_AVB_TD3 */ > > > pins = <242>, <240>, <236>, <237>, <238>, <239>; > > > drive-strength = <12>; > > > }; > > > }; > > > ... > > > }; > > > }; > > > }; > > > > > > So "arm,scmi-smc" node will have even more platform specific settings. > > > > > > > > > > > > I prefer the second approach and will try to make it if it's OK to copy > > > > > arm,scmi-smc node from partial Device-tree and generate arm,scmi-shmem > > > > > node. > > > > > > > > > > What do you think about that? > > > > > > > > From a device tree specification perspective, I think both approaches > > > > are OK (with a minor comment on the first approach as I wrote above.) > > > > > > > > But from a Xen perspective I think it is important that we don't require > > > > the user to manually provide the SCMI configuration in the partial DTB. > > > > It would be better if we could generate it automatically from Xen or the > > > > tools (or even an independent script). Or copy the "arm,scmi-smc" node > > > > from the host device tree to the domU device tree without modifications. > > > > > > I think copy "arm,scmi-smc" node is the only option we have. > > > I'm not sure what do you mean under "host device tree" if you mean Xen > > > device-tree - then I think it will not cover the case with stm32mp1 I've > > > mentioned above. I think it will be better to copy "arm,scmi-smc" node > > > from Domu partial Device-tree to Domu device-tree. > > > So AGENT0 smc-id will be set in xen device-tree and copied to dom0 and > > > AGENT1 scm-is set in domu device-tree and copied to dom-u. > > > > > > Do you agree with my points? > > > > I think we are saying similar things, but we are getting mixed up with > > the terminology. Let's start from the basics :-) > > > > # Host device tree > > The device tree given to Xen at boot time. This is the device tree that > > Xen parses to discover what's available on the platform. In your case, > > it seems to include r8a77961-salvator-xs-xen.dts. > > > > # Partial DTB > > (Ignoring Dom0less) this is the small DTB that gets passed to xl with > > the "device_tree" option in the xl config file. It is copied verbatim > > to the domU device tree by xl/libxl. > > > > # Copy the "arm,scmi-smc" node from host device tree > > This means that the domU "arm,scmi-smc" node is an exact copy of the > > host device tree SCMI node. I don't think this is actually possible in > > most cases because the domU description is typically a bit different > > from the host description. For instance, the host description could > > include 14+1 channels while the domU description should only include 1 > > channel. > > > > # Copy the "arm,scmi-smc" node from the partial DTB > > This implies that somebody or something create an "arm,scmi-smc" node > > for the domU and placed it into the partial DTB. Then, Xen and/or > > xl/libxl will copy the node from the partial DTB to the DomU device > > tree. The main question in this case is: who is going to write the > > partial DTB? We dont want the user (i.e. a person) to have to manually > > write the SCMI description for the domU. It should be an automated tools > > that does it. At that point, it is easier if it is Xen or xl/libxl. > > Alternativaly, we could think of an external tool but I think it would > > make things more difficult to maintain. > > > > # Generate the "arm,scmi-smc" node for domUs > > When I write "generate the arm,scmi-smc node", I mean that Xen and > > libxl/xl will generate the "arm,scmi-smc" node for the domU. Thus, the > > node will not be copied from the partial DTB or from the device tree, > > instead, it should be created directly by Xen and/or libxl/xl. > > > > However, the domU "arm,scmi-smc" node could still be derived from the > > host device tree "arm,scmi-smc" node. In other words, Xen or xl/libxl > > would look at the host device tree "arm,scmi-smc" node, copy it to the > > domU device tree while making as many changes as necessary. > > > > The DomU "arm,scmi-smc" node doesn't have to be entirely fake and > > static. It could be dynamically created to match the host device tree > > description. I think this is the best option. > > > > > > # Conclusion > > I am suggesting that Xen and/or libxl automatically produce the > > "arm,scmi-smc" node for domUs based on the host device tree description > > and based on the channel mapped to the domU. This way, the user (a > > person) doesn't have to go and manually edit the domU partial DTB. > > > > That sounds reasonable. The problem is that arm,scmi-smc node can be > copmlicated and include a lot of configuration. Also for different > mediators this node can be different. > As I inderstand, there is no mechanism for xl to access host device-tree > right now. Correct me if I'm wrong. Yes, you are right. And the lack of a mechanism for xl to access the host device tree is a problem. > I see the following way we can generate arm,scmi-smc node for DomU: > We say that if scmi-smc mediator is enabled - then Dom0 is configured to > use SCMI. This means that Dom0 device-tree will have arm,scmi-smc node > and it can be reached from the userspace. So far so good > In this case xl can use infromation from /proc/device-tree/firmware/scmi > to generate arm,scmi-smc node for DomU. But in this case xl should know > the exact path of scmi node. > > Or we can generate some special node, called "shared" in Dom0 device-tree > which will include copy of the arm,scmi-smc node, which can be used for domains. > In this case xl can scan /proc/device-tree/shared node and find > arm,scmi-smc copatible node and use it to generate arm,scmi-smc node for > DomU. > Also this can be used for another features in future. > > What do you think about this? Basing the domU SCMI node generation on the dom0 SCMI node is not great, because it should be based on the host device tree rather than dom0 and the dom0 SCMI description will be different. Instead of copying just the host SCMI node somewhere in the dom0 device tree so that xl/libxl can access it via /proc/device-tree, I think it would be better to make the full host DTB available to xl/libxl. There are probably many ways to do this, but there are a couple I can think of on top of my head: - introduce a new hypercall to get the full host dtb from Xen The hypercall would pass an address and a size in guest physical memory and Xen would copy the host DTB to it. - introduce something like /proc/device-tree to hypfs See xen/common/hypfs.c
On Fri, Jan 21, 2022 at 12:49:55PM -0800, Stefano Stabellini wrote: > On Fri, 21 Jan 2022, Oleksii Moisieiev wrote: > > On Thu, Jan 20, 2022 at 02:29:41PM -0800, Stefano Stabellini wrote: > > > On Thu, 20 Jan 2022, Oleksii Moisieiev wrote: > > > > On Wed, Jan 19, 2022 at 05:28:21PM -0800, Stefano Stabellini wrote: > > > > > On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > > > > > > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > > > > > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > > > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > > > > --- > > > > > > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > > > > > > + > > > > > > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > > > > > > --- /dev/null > > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > > > > > > +config SCMI_SMC > > > > > > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > > > > > > + default n > > > > > > > > > > > > > > + depends on SCI > > > > > > > > > > > > > > + ---help--- > > > > > > > > > > > > > > + > > > > > > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > > > > > > + for communication. > > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > > > > > > obj-y += sci.o > > > > > > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > > > > > > --- /dev/null > > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > > > > > > +/* > > > > > > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > > > > > > + * (at your option) any later version. > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > > > > > > + */ > > > > > > > > > > > > > > + > > > > > > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > > > > > > +#include <asm/io.h> > > > > > > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > > > > > > +#include <xen/config.h> > > > > > > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > > > > > > +#include <xen/init.h> > > > > > > > > > > > > > > +#include <xen/err.h> > > > > > > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > > > > > > +#include <xen/list.h> > > > > > > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > > > > > > +#include <xen/string.h> > > > > > > > > > > > > > > +#include <xen/time.h> > > > > > > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > > > > > > + > > > > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > > > > > > + > > > > > > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > > > > > > + > > > > > > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > > > > > > + > > > > > > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > > > > > > channels: > > > > > > > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > > > > /* reserved region for scmi channels*/ > > > > > > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > > > > > > no-map; > > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > > > > > > }; > > > > > > > > > > > > }; > > > > > > > > > > > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > > > > > > the current scmi channel: > > > > > > > > > > > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > > > }; > > > > > > > > > > > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > > > > > > assigned to this agent. > > > > > > > > > > > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > > }; > > > > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > > > > }; > > > > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > > > > }; > > > > > > > > > > > ... > > > > > > > > > > > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > > > > > > there were some if, checking if memory weren't reserved. > > > > > > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > > > > > > shmem page. > > > > > > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > > > > > > which has no compatible string and provides no-map property. > > > > > > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > > > > > > space for the domain. > > > > > > > > > > > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > > > > > > and scmi_devid property to the > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > > > > > > Those node and property are needed only for Xen and useless for > > > > > > > > > > non-virtualized systems. I can add this node and property description to > > > > > > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > > > > > > What do you think about it? > > > > > > > > > > > > > > > > > > Reply below > > > > > > > > > > > > > > > > > > [...] > > > > > > > > > > > > > > > > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > > > > > > Documentation/devicetree/bindings (probably > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > > > > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > > > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > > > > > > properties into generic bindings. It is a problem both from a > > > > > > > > > specification perspective (because it has hard to handle Xen specific > > > > > > > > > cases in fully generic bindings, especially as those bindings are > > > > > > > > > maintained as part of the Linux kernel) and from a user perspective > > > > > > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > > > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > > > > > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > > > > > > looks like a generic property that should be needed for the Linux SCMI > > > > > > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > > > > > > > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > > > > > > On non-virtualized systems - there is no need of this call, because OS > > > > > > > > is the only one entity, running on the system. > > > > > > > > > > > > > > OK. Even if it is only required for virtualized systems, I think that > > > > > > > scmi_devid is important enough that should be part of the upstream > > > > > > > binding. I think it is worth starting an email thread on the LKML with > > > > > > > Rob Herring and the SCMI maintainers to discuss the addition of > > > > > > > scmi_devid to the binding. > > > > > > > > > > > > > > > > > > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > > > > > > list of device_ids to dom.cfg, such as: > > > > > > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > > > > > > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > > > > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > > > > > > We can probably make hypercall taking devid list as input parameter. > > > > > > > > This will take only 1 hypercall to setup sci permissions. > > > > > > > > > > > > > > But how would a user know which are the right SCMI IDs to add to the > > > > > > > sci_devs list? Would the user have to go and read the reference manual > > > > > > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > > > > > > If that is the case, then I think that it would be better to add > > > > > > > scmi_devid to device tree. > > > > > > > > > > > > > > In general, I think this configuration should happen automatically > > > > > > > without user intervention. The user should just specify "enable SCMI" > > > > > > > and it should work. > > > > > > > > > > > > > > > > > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > > > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > > > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > > > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > > > > > > way, I don't think we should need linux,scmi_mem. > > > > > > > > > > > > > > > > This requires further investigation. I will try to make implementation > > > > > > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > > > > > > reuslts with you. > > > > > > > > > > > > > > OK, thanks. > > > > > > > > > > > > Hi Stefano, > > > > > > > > > > > > As I did some investigation about using reserved-memory area > > > > > > linux,scmi_mem and now I need your advice. > > > > > > > > > > > > I see 2 possible implementations for now: > > > > > > 1) Add memory-region parameter to cpu_scp_shm node which points to the > > > > > > reserved memory region. > > > > > > So device-tree will look like this: > > > > > > > > > > > > reserved-memory { > > > > > > /* reserved region for scmi channels*/ > > > > > > scmi_memory: region@53FF0000{ > > > > > > no-map; > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > }; > > > > > > }; > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > compatible = "arm,scmi-shmem"; > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > memory-region = <&scmi_memory>; > > > > > > }; > > > > > > > > > > > > So cpu_scp_shm node has a reference to scmi_memory region. This mean > > > > > > that xen can find reserved memory region without adding additional names > > > > > > to the device-tree bindings. > > > > > > memory-region parameter as a reference to reserved memory and region > > > > > > creation described in: > > > > > > https://urldefense.com/v3/__https://github.com/torvalds/linux/blob/v5.15/Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt__;!!GF_29dbcQIUBPA!k6x19x1gYF1CPlgAZj7std3ifqhq-9DXvuF0nwonNPUwMzZpYHYbrRJziJrgdFIOjyan$ [github[.]com] > > > > > > > > > > > > This approach I've implemented already and it works. > > > > > > > > > > This approach would require a discussion with the upstream device tree > > > > > maintainers. Likely, we would need to add a note about the usage of the > > > > > "memory-region" property to arm,scmi.yaml. > > > > > > > > > > Also, I have the feeling that they would ask to add the "memory-region" > > > > > property directly to the "arm,scmi-smc" node, as an alternative (or > > > > > in addition) to the existing "shmem" property. > > > > > > > > > > That said, from my point of view this approach is also a viable option. > > > > > I don't see any major problems. > > > > > > > > > > The main question (after reading everything else that you wrote below) > > > > > is whether the "arm,scmi-smc" node in this case could be automatically > > > > > generated. > > > > > > > > > > > > > arm,scmi-smc node can be generated in both cases. I think I'd leave it > > > > as backup in case if the second approach will not work. > > > > > > > > > > > > > > > 2) The second approach is the format you suggested: > > > > > > > > > > > reserved-memory { > > > > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > > }; > > > > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > > > > }; > > > > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > > > > }; > > > > > > > > > > > ... > > > > > > > > > > > > This approach has an advantage that xen ARM_SCI driver do not know about > > > > > > how channels are placed in the reserved memory, but introduces some > > > > > > disadvantages: > > > > > > a) We provide extra 14 (in our case) arm,scmi-shmem nodes which are not used > > > > > > in the device-tree. In current implementation I have separate scmi.dtsi > > > > > > file which introduces scmi support for both XEN-based and > > > > > > non-virtualized systems. Having 14 extra channels in the device-tree may > > > > > > be confusing. > > > > > > > > > > I can see that while it would be ideal for Xen to see all 14+1 channels > > > > > in device tree (on the host device tree), we wouldn't want to expose all > > > > > of them to the domains, not even to dom0. > > > > > > > > > > How many channels do we want dom0 to see by the way? For this > > > > > discussion, I'll just assume for now that dom0 only sees 1 channel like > > > > > the domUs. > > > > > > > > For dom0 we need only one channel. > > > > > > > > > > > > > > Now we have a problem: how do we go about "filtering" the > > > > > "arm,scmi-shmem" device tree nodes? Which is also what you are asking > > > > > below in point b). > > > > > > > > > > > > > Xen will not need to filter "arm,scmi-shmem" node. It will just > > > > create shmem node in Domain device-tree. I don't see any problem for xen > > > > configuration here. > > > > What bothers me here is that I set scmi configuration for platform dts, > > > > not for xen or domu dts files. > > > > So for example I have the following structure of the dts files for my > > > > platform (r8a77961-salvator-xs): > > > > * r8a77961-scmi.dtsi - this file includes all scmi related nodes and set > > > > scmi_devid for the devices, that should use scmi. > > > > * r8a77961-salvator-xs.dts - dts file which generates dtb for the platform. > > > > It includes r8a77961-scmi.dtsi so I populate scmi to platform dtb, which > > > > is used for system with no hypervisor. > > > > * r8a77961-salvator-xs-xen.dts - dts file for xen which includes > > > > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > > > * r8a77961-salvator-xs-domu.dts - dts file for DomU which includes > > > > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > > > > > > > In this case r8a77961-salvator-xs.dtb r8a77961-salvator-xs-xen.dtb > > > > r8a77961-salvator-xs-domu.dtb files will inherit 14+1 channel. > > > > > > > > I can give you a link to Merge request with this changes if you need it. > > > > > > > > For xen and domu dtb it is not a problem because all "arm,scmi-shmem" > > > > nodes will be omitted and new will be generated for the domains. > > > > > > > > What bothers me is that r8a77961-salvator-xs.dtb will have 14 unused channels. > > > > > > > > Just got an idea while writing this: I can create only one > > > > "arm,scmi-shmem" node in r8a77961-scmi.dtsi and add 14 more nodes, > > > > needed for xen explicitly in r8a77961-salvator-xs-xen.dts. > > > > > > > > Then we will have valid configurations for all cases. > > > > This can be a solution. What do you think? > > > > > > It is good that you brought this up because it helps me explain what I > > > mean. And of course it is up to you where you place the nodes in the > > > various dts files at your disposal. Either way it would work but I think > > > they should belong to r8a77961-salvator-xs.dts. > > > > > > Generally the platform vendor (e.g. Xilinx) provides a device tree > > > description of the platform to use including all the available resources > > > and firmware interfaces. In your case it would be r8a77961-scmi.dtsi + > > > r8a77961-salvator-xs.dts. This is what I call the "host device tree" > > > below. Users should be able to boot a fully functional system using the > > > host device tree pretty much "as is" to run Xen, Linux or any other > > > software. > > > > > > Certainly the SCMI device tree description should be part of the host > > > device tree, so in your case it would be r8a77961-salvator-xs.dts. And > > > the description should include all 14+1 channels because this is the > > > generic platform description -- we cannot know for sure how the users > > > are going to use the system. > > > > > > This is why r8a77961-salvator-xs-xen.dts should be as small as possible > > > or ideally inexistent. There shouldn't be a need for a special device > > > tree modification to allow Xen to run. In reality, even at Xilinx we > > > have something like r8a77961-salvator-xs-xen.dts, although it is really > > > small. > > > > > > But I see that r8a77961-salvator-xs-xen.dts could be viewed as the > > > device tree additions to run hypervisors and from that point of view it > > > is more acceptable to place the 14 channels there. > > > > > > The biggest problem is r8a77961-salvator-xs-domu.dts: who is going to > > > write it? And how? It wouldn't be provided by the platform vendor, so it > > > is the user the one that has to find a way to write it. > > > > > > I know the user already has to write a partial DTB for device > > > assignment, but any time the process is more complex than "copy the host > > > device tree node for device XXX to the partial DTB" it is a problem. > > > Errors are made and the system doesn't work. > > > > > > I think we don't want to make it even more difficult by having to > > > manually produce the SCMI domU description too. The SCMI description for > > > domU could be automatically generated by Xen, or libxl/xl. If that's an > > > issue, then the SCMI description could be automatically generated by an > > > external tool but I think it would make things more complex and harder > > > to maintain. > > > > > > In short my point of view is: > > > - r8a77961-scmi.dtsi + r8a77961-salvator-xs.dts should be as generic as > > > possible so the SCMI nodes should have 14+1 channels > > > - but putting the 14 channels in r8a77961-salvator-xs-xen.dts is still > > > OKish > > > - it is important that r8a77961-salvator-xs-domu.dts is automatically > > > generated by Xen or libxl or another software tool > > > > > > > Thank you for the detailed response. I'll put all 14+1 channels to > > r8a77961-salvator-xs.dts then. > > I've described my thoughts about generation of the arm,scmi-smc node below. > > > > > > > > > > > b) In case if we have all 15 channels, described in partial device-tree, > > > > > > > > > > I think you meant "described in the host device tree", right? > > > > > > > > > Yeah that's what I've meant. > > > > > > > > > > > we should not copy any node to the domain device-tree. I think it will > > > > > > be better to generate arm,scmi-shmem node in the Domain device-tree. > > > > > > > > > > Yes, I think it makes sense for Xen to generate the "arm,scmi-shmem" > > > > > device tree description for the DomU/Dom0 based on the channels > > > > > allocated to the domain. > > > > > > > > > > > > > > > > The problem is that arm,scmi-smc node, which is using arm,scmi-shmem > > > > > > node can't be generated. I prefer it to be copied from the partial > > > > > > device-tree because it includes some platform specific configuration, > > > > > > such as func-id and list of the protocols (for example different > > > > > > platforms may require different list of the protocols). So in this > > > > > > case we will have 1 node copied and 1 node generated. > > > > > > > > > > > > I think even for dom0less we should use arm,scmi-smc node from the > > > > > > device-tree because protocol configuration and funcid is related to the > > > > > > platform. > > > > > > > > > > I am not sure I understood what you wrote. You are saying that the > > > > > "arm,scmi-smc" node includes some platform specific configurations so > > > > > it cannot be automatically generated by Xen (or by the tools) and > > > > > instead it needs to be manually provided as part of the partial dtb for > > > > > the domU. Is that correct? > > > > > > > > > > If so, I would like to understand the reasons behind it. Manual > > > > > device tree editing is problematic. > > > > > > > > > > I looked for "func-id" in > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml but couldn't > > > > > find any results. Do you have an example of the platform specific > > > > > configuration or protocol configuration that would make it difficult to > > > > > automatically generate the "arm,scmi-smc" node for the domains? > > > > > > > > Sorry, I used wrong term (used term from the specification), arm,smc-id > > > > of cause. > > > > > > > > > > > > > > Also, is this a problem just for approach #2 or also for approach #1? > > > > > If it is a problem only for approach #2, then let's just go with > > > > > approach #1. > > > > > > > > > > > > > We can't copy "arm,scmi-smc" in both approaches. The difference is that > > > > in the first approach we can copy both "arm,scmi-smc" and > > > > "arm,scmi-shmem" nodes while in the second approach we should copy > > > > "arm,scmi-smc", but we have to generate "arm,scmi-shmem" node. > > > > > > > > arm,scmi-smc node can't be generated because it includes properties and > > > > configurations that depends from platform and should be get from the > > > > device tree. > > > > Here is "arm,scmi-smc" node expample: > > > > firmware { > > > > scmi { > > > > compatible = "arm,scmi-smc" > > > > arm,smc-id = <0x82000002>; > > > > shmem = <&cpu_scp_shm>; > > > > #address-cells = <1>; > > > > #size-cells = <0>; > > > > scmi_power: protocol@11 { > > > > reg = <0x11>; > > > > #power-domain-cells = <1>; > > > > }; > > > > > > > > scmi_clock: protocol@14 { > > > > ... > > > > }; > > > > > > > > scmi_reset: protocol@16 { > > > > ... > > > > }; > > > > ... > > > > }; > > > > }; > > > > > > > > It has 3 configurable options: > > > > * arm,smc-id parameter, setting func_id for scmi protocol. This id can be > > > > different for different platforms. > > > > For example stm32mp1 architecture use different scm-id for different > > > > agents: > > > > https://urldefense.com/v3/__https://github.com/ARM-software/arm-trusted-firmware/blob/0586c41b3f2d52aae847b7212e7b0c7e19197ea2/plat/st/stm32mp1/include/stm32mp1_smc.h*L39__;Iw!!GF_29dbcQIUBPA!mTRUjtSg19iVaYo3Cgjop5ckPWqKsHVo1EZCEA1zCbod9KpNSXX291A8vAuNdTCr46MA$ [github[.]com] > > > > > > > > * shmem which includes phandle to arm,scmi-shmem node. But this is not > > > > a problem and can be updated. > > > > > > > > * list of the protocol subnodes. This is also configurable parameter, > > > > not regs or names, but the number of the protocols. For example onle > > > > platform can use power-domains/clock/resets via scmi, when another will > > > > require volage-control and sensor-management to be added. > > > > > > > > Xen should know this parameters to be able to generate "arm,scmi-smc" node. > > > > > > > > Also we're currently discussing new scmi protocol with ARM: Pinctrl over > > > > SCMI. > > > > > > > > It should allow domains to access pinctrl subsystem, placed in Firmware > > > > through SCMI protocol. > > > > scmi_pinctrl node will look like this: > > > > > > > > firmware { > > > > scmi { > > > > ... > > > > scmi_pinctrl: protocol@18 { > > > > reg = <0x18>; > > > > #pinctrl-cells = <0>; > > > > > > > > i2c2_pins: i2c2 { > > > > groups = <74>; /* i2c2_a */ > > > > function = <15>; /* i2c2 */ > > > > }; > > > > > > > > irq0_pins: irq0 { > > > > groups = <81>; /* intc_ex_irq0 */ > > > > function = <19>; /* intc_ex */ > > > > }; > > > > > > > > avb_pins: avb { > > > > mux { > > > > /* avb_link, avb_mdio, avb_mii */ > > > > groups = <17>, <21>, <22>; > > > > function = <1>; /* avb */ > > > > }; > > > > > > > > pins_mdio { > > > > groups = <21>; /* avb_mdio */ > > > > drive-strength = <24>; > > > > }; > > > > > > > > pins_mii_tx { > > > > /* PIN_AVB_TX_CTL, PIN_AVB_TXC, PIN_AVB_TD0, > > > > PIN_AVB_TD1, PIN_AVB_TD2, PIN_AVB_TD3 */ > > > > pins = <242>, <240>, <236>, <237>, <238>, <239>; > > > > drive-strength = <12>; > > > > }; > > > > }; > > > > ... > > > > }; > > > > }; > > > > }; > > > > > > > > So "arm,scmi-smc" node will have even more platform specific settings. > > > > > > > > > > > > > > > I prefer the second approach and will try to make it if it's OK to copy > > > > > > arm,scmi-smc node from partial Device-tree and generate arm,scmi-shmem > > > > > > node. > > > > > > > > > > > > What do you think about that? > > > > > > > > > > From a device tree specification perspective, I think both approaches > > > > > are OK (with a minor comment on the first approach as I wrote above.) > > > > > > > > > > But from a Xen perspective I think it is important that we don't require > > > > > the user to manually provide the SCMI configuration in the partial DTB. > > > > > It would be better if we could generate it automatically from Xen or the > > > > > tools (or even an independent script). Or copy the "arm,scmi-smc" node > > > > > from the host device tree to the domU device tree without modifications. > > > > > > > > I think copy "arm,scmi-smc" node is the only option we have. > > > > I'm not sure what do you mean under "host device tree" if you mean Xen > > > > device-tree - then I think it will not cover the case with stm32mp1 I've > > > > mentioned above. I think it will be better to copy "arm,scmi-smc" node > > > > from Domu partial Device-tree to Domu device-tree. > > > > So AGENT0 smc-id will be set in xen device-tree and copied to dom0 and > > > > AGENT1 scm-is set in domu device-tree and copied to dom-u. > > > > > > > > Do you agree with my points? > > > > > > I think we are saying similar things, but we are getting mixed up with > > > the terminology. Let's start from the basics :-) > > > > > > # Host device tree > > > The device tree given to Xen at boot time. This is the device tree that > > > Xen parses to discover what's available on the platform. In your case, > > > it seems to include r8a77961-salvator-xs-xen.dts. > > > > > > # Partial DTB > > > (Ignoring Dom0less) this is the small DTB that gets passed to xl with > > > the "device_tree" option in the xl config file. It is copied verbatim > > > to the domU device tree by xl/libxl. > > > > > > # Copy the "arm,scmi-smc" node from host device tree > > > This means that the domU "arm,scmi-smc" node is an exact copy of the > > > host device tree SCMI node. I don't think this is actually possible in > > > most cases because the domU description is typically a bit different > > > from the host description. For instance, the host description could > > > include 14+1 channels while the domU description should only include 1 > > > channel. > > > > > > # Copy the "arm,scmi-smc" node from the partial DTB > > > This implies that somebody or something create an "arm,scmi-smc" node > > > for the domU and placed it into the partial DTB. Then, Xen and/or > > > xl/libxl will copy the node from the partial DTB to the DomU device > > > tree. The main question in this case is: who is going to write the > > > partial DTB? We dont want the user (i.e. a person) to have to manually > > > write the SCMI description for the domU. It should be an automated tools > > > that does it. At that point, it is easier if it is Xen or xl/libxl. > > > Alternativaly, we could think of an external tool but I think it would > > > make things more difficult to maintain. > > > > > > # Generate the "arm,scmi-smc" node for domUs > > > When I write "generate the arm,scmi-smc node", I mean that Xen and > > > libxl/xl will generate the "arm,scmi-smc" node for the domU. Thus, the > > > node will not be copied from the partial DTB or from the device tree, > > > instead, it should be created directly by Xen and/or libxl/xl. > > > > > > However, the domU "arm,scmi-smc" node could still be derived from the > > > host device tree "arm,scmi-smc" node. In other words, Xen or xl/libxl > > > would look at the host device tree "arm,scmi-smc" node, copy it to the > > > domU device tree while making as many changes as necessary. > > > > > > The DomU "arm,scmi-smc" node doesn't have to be entirely fake and > > > static. It could be dynamically created to match the host device tree > > > description. I think this is the best option. > > > > > > > > > # Conclusion > > > I am suggesting that Xen and/or libxl automatically produce the > > > "arm,scmi-smc" node for domUs based on the host device tree description > > > and based on the channel mapped to the domU. This way, the user (a > > > person) doesn't have to go and manually edit the domU partial DTB. > > > > > > > That sounds reasonable. The problem is that arm,scmi-smc node can be > > copmlicated and include a lot of configuration. Also for different > > mediators this node can be different. > > As I inderstand, there is no mechanism for xl to access host device-tree > > right now. Correct me if I'm wrong. > > Yes, you are right. And the lack of a mechanism for xl to access the > host device tree is a problem. > > > > I see the following way we can generate arm,scmi-smc node for DomU: > > We say that if scmi-smc mediator is enabled - then Dom0 is configured to > > use SCMI. This means that Dom0 device-tree will have arm,scmi-smc node > > and it can be reached from the userspace. > > So far so good > > > > In this case xl can use infromation from /proc/device-tree/firmware/scmi > > to generate arm,scmi-smc node for DomU. But in this case xl should know > > the exact path of scmi node. > > > > Or we can generate some special node, called "shared" in Dom0 device-tree > > which will include copy of the arm,scmi-smc node, which can be used for domains. > > In this case xl can scan /proc/device-tree/shared node and find > > arm,scmi-smc copatible node and use it to generate arm,scmi-smc node for > > DomU. > > Also this can be used for another features in future. > > > > What do you think about this? > > Basing the domU SCMI node generation on the dom0 SCMI node is not great, > because it should be based on the host device tree rather than dom0 and > the dom0 SCMI description will be different. > > Instead of copying just the host SCMI node somewhere in the dom0 device > tree so that xl/libxl can access it via /proc/device-tree, I think it > would be better to make the full host DTB available to xl/libxl. > > There are probably many ways to do this, but there are a couple I can > think of on top of my head: > > - introduce a new hypercall to get the full host dtb from Xen > The hypercall would pass an address and a size in guest physical memory > and Xen would copy the host DTB to it. > > - introduce something like /proc/device-tree to hypfs > See xen/common/hypfs.c Hi Stefano, I've found the discussion, indirectly related to our topic [1], in which hypervisor <-> toolchain interface topic was raised. Where Andrii Anisov proposed to introduce another libxl type (f.e. File) which will read device tree file and operate with a binary blob. His idea is to introduce new Domctl hypecall, which is able to send device-tree blob from toolstack to the hypervisor. I was thinking about using his approach, but in opposite direction. Such as introduce a set of DOMCTL, which gives ability for xl user to request Host Device-tree nodes. I see something like that: XEN_DOMCTL_host_node_by_path - gets node path as input and returns blob with node; XEN_DOMCTL_find_host_compatible_node - gets compatible string and "skip" number for the case if there is more than 1 compatible node and returns blob with node. Advandages for my sight are: - I hope Andrii still has some code, which can be reused during the implementation; - Xen can determine if the requesting node can be exposed to the guest, which is additional layer of validation; - no need to transfer all device-tree to the toolstack. As for the hypfs this sounds like a separate feature to me. I'm not sure if we need to provide the entire copy of the device-tree. Also there could be some problems with this solution. For example, xl tool have to scan whole hypfs tree to find node by phandle or by compatible string. Looking forward for your thoughts about that. Also I would be happy if Julien could share his vision. [1] https://xen.markmail.org/search/?q=from%3AAndrii+Anisov+device-tree#query:from%3AAndrii%20Anisov%20device-tree+page:2+mid:tvgyp7fzpiyyso3j+state:results Oleksii
On Mon, 24 Jan 2022, Oleksii Moisieiev wrote: > On Fri, Jan 21, 2022 at 12:49:55PM -0800, Stefano Stabellini wrote: > > On Fri, 21 Jan 2022, Oleksii Moisieiev wrote: > > > On Thu, Jan 20, 2022 at 02:29:41PM -0800, Stefano Stabellini wrote: > > > > On Thu, 20 Jan 2022, Oleksii Moisieiev wrote: > > > > > On Wed, Jan 19, 2022 at 05:28:21PM -0800, Stefano Stabellini wrote: > > > > > > On Wed, 19 Jan 2022, Oleksii Moisieiev wrote: > > > > > > > On Wed, Dec 22, 2021 at 06:23:24PM -0800, Stefano Stabellini wrote: > > > > > > > > On Wed, 22 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > On Tue, Dec 21, 2021 at 01:22:50PM -0800, Stefano Stabellini wrote: > > > > > > > > > > On Tue, 21 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > > > > > > > On Mon, Dec 20, 2021 at 04:52:01PM -0800, Stefano Stabellini wrote: > > > > > > > > > > > > On Mon, 20 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > > > > > > > > > > > > > On Fri, Dec 17, 2021 at 06:14:55PM -0800, Stefano Stabellini wrote: > > > > > > > > > > > > > > On Tue, 14 Dec 2021, Oleksii Moisieiev wrote: > > > > > > > > > > > > > > > This is the implementation of SCI interface, called SCMI-SMC driver, > > > > > > > > > > > > > > > which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). > > > > > > > > > > > > > > > This allows devices from the Domains to work with clocks, resets and > > > > > > > > > > > > > > > power-domains without access to CPG. > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > The following features are implemented: > > > > > > > > > > > > > > > - request SCMI channels from ATF and pass channels to Domains; > > > > > > > > > > > > > > > - set device permissions for Domains based on the Domain partial > > > > > > > > > > > > > > > device-tree. Devices with permissions are able to work with clocks, > > > > > > > > > > > > > > > resets and power-domains via SCMI; > > > > > > > > > > > > > > > - redirect scmi messages from Domains to ATF. > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > > > > > --- > > > > > > > > > > > > > > > xen/arch/arm/Kconfig | 2 + > > > > > > > > > > > > > > > xen/arch/arm/sci/Kconfig | 10 + > > > > > > > > > > > > > > > xen/arch/arm/sci/Makefile | 1 + > > > > > > > > > > > > > > > xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ > > > > > > > > > > > > > > > xen/include/public/arch-arm.h | 1 + > > > > > > > > > > > > > > > 5 files changed, 809 insertions(+) > > > > > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > > > create mode 100644 xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig > > > > > > > > > > > > > > > index 186e1db389..02d96c6cfc 100644 > > > > > > > > > > > > > > > --- a/xen/arch/arm/Kconfig > > > > > > > > > > > > > > > +++ b/xen/arch/arm/Kconfig > > > > > > > > > > > > > > > @@ -114,6 +114,8 @@ config SCI > > > > > > > > > > > > > > > support. It allows guests to control system resourcess via one of > > > > > > > > > > > > > > > SCI mediators implemented in XEN. > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > +source "arch/arm/sci/Kconfig" > > > > > > > > > > > > > > > + > > > > > > > > > > > > > > > endmenu > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > menu "ARM errata workaround via the alternative framework" > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > > > > > index 0000000000..9563067ddc > > > > > > > > > > > > > > > --- /dev/null > > > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/Kconfig > > > > > > > > > > > > > > > @@ -0,0 +1,10 @@ > > > > > > > > > > > > > > > +config SCMI_SMC > > > > > > > > > > > > > > > + bool "Enable SCMI-SMC mediator driver" > > > > > > > > > > > > > > > + default n > > > > > > > > > > > > > > > + depends on SCI > > > > > > > > > > > > > > > + ---help--- > > > > > > > > > > > > > > > + > > > > > > > > > > > > > > > + Enables mediator in XEN to pass SCMI requests from Domains to ATF. > > > > > > > > > > > > > > > + This feature allows drivers from Domains to work with System > > > > > > > > > > > > > > > + Controllers (such as power,resets,clock etc.). SCP is used as transport > > > > > > > > > > > > > > > + for communication. > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > > > index 837dc7492b..67f2611872 100644 > > > > > > > > > > > > > > > --- a/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/Makefile > > > > > > > > > > > > > > > @@ -1 +1,2 @@ > > > > > > > > > > > > > > > obj-y += sci.o > > > > > > > > > > > > > > > +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o > > > > > > > > > > > > > > > diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > new file mode 100644 > > > > > > > > > > > > > > > index 0000000000..2eb01ea82d > > > > > > > > > > > > > > > --- /dev/null > > > > > > > > > > > > > > > +++ b/xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > @@ -0,0 +1,795 @@ > > > > > > > > > > > > > > > +/* > > > > > > > > > > > > > > > + * xen/arch/arm/sci/scmi_smc.c > > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > > + * SCMI mediator driver, using SCP as transport. > > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > > + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> > > > > > > > > > > > > > > > + * Copyright (C) 2021, EPAM Systems. > > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > > + * This program is free software; you can redistribute it and/or modify > > > > > > > > > > > > > > > + * it under the terms of the GNU General Public License as published by > > > > > > > > > > > > > > > + * the Free Software Foundation; either version 2 of the License, or > > > > > > > > > > > > > > > + * (at your option) any later version. > > > > > > > > > > > > > > > + * > > > > > > > > > > > > > > > + * This program is distributed in the hope that it will be useful, > > > > > > > > > > > > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > > > > > > > > > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > > > > > > > > > > > > > > > + * GNU General Public License for more details. > > > > > > > > > > > > > > > + */ > > > > > > > > > > > > > > > + > > > > > > > > > > > > > > > +#include <asm/sci/sci.h> > > > > > > > > > > > > > > > +#include <asm/smccc.h> > > > > > > > > > > > > > > > +#include <asm/io.h> > > > > > > > > > > > > > > > +#include <xen/bitops.h> > > > > > > > > > > > > > > > +#include <xen/config.h> > > > > > > > > > > > > > > > +#include <xen/sched.h> > > > > > > > > > > > > > > > +#include <xen/device_tree.h> > > > > > > > > > > > > > > > +#include <xen/iocap.h> > > > > > > > > > > > > > > > +#include <xen/init.h> > > > > > > > > > > > > > > > +#include <xen/err.h> > > > > > > > > > > > > > > > +#include <xen/lib.h> > > > > > > > > > > > > > > > +#include <xen/list.h> > > > > > > > > > > > > > > > +#include <xen/mm.h> > > > > > > > > > > > > > > > +#include <xen/string.h> > > > > > > > > > > > > > > > +#include <xen/time.h> > > > > > > > > > > > > > > > +#include <xen/vmap.h> > > > > > > > > > > > > > > > + > > > > > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL 0x10 > > > > > > > > > > > > > > > +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 > > > > > > > > > > > > > > > +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 > > > > > > > > > > > > > > > +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB > > > > > > > > > > > > > > > +#define SCMI_BASE_DISCOVER_AGENT 0x7 > > > > > > > > > > > > > > > + > > > > > > > > > > > > > > > +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ > > > > > > > > > > > > > > > +#define SCMI_SUCCESS 0 > > > > > > > > > > > > > > > +#define SCMI_NOT_SUPPORTED (-1) > > > > > > > > > > > > > > > +#define SCMI_INVALID_PARAMETERS (-2) > > > > > > > > > > > > > > > +#define SCMI_DENIED (-3) > > > > > > > > > > > > > > > +#define SCMI_NOT_FOUND (-4) > > > > > > > > > > > > > > > +#define SCMI_OUT_OF_RANGE (-5) > > > > > > > > > > > > > > > +#define SCMI_BUSY (-6) > > > > > > > > > > > > > > > +#define SCMI_COMMS_ERROR (-7) > > > > > > > > > > > > > > > +#define SCMI_GENERIC_ERROR (-8) > > > > > > > > > > > > > > > +#define SCMI_HARDWARE_ERROR (-9) > > > > > > > > > > > > > > > +#define SCMI_PROTOCOL_ERROR (-10) > > > > > > > > > > > > > > > + > > > > > > > > > > > > > > > +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") > > > > > > > > > > > > > > > + > > > > > > > > > > > > > > > +#define SCMI_SMC_ID "arm,smc-id" > > > > > > > > > > > > > > > +#define SCMI_SHARED_MEMORY "linux,scmi_mem" > > > > > > > > > > > > > > > > > > > > > > > > > > > > I could find the following SCMI binding in Linux, which describes > > > > > > > > > > > > > > the arm,scmi-smc compatible and the arm,smc-id property: > > > > > > > > > > > > > > > > > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml > > > > > > > > > > > > > > > > > > > > > > > > > > > > However, linux,scmi_mem is not described. Aren't you supposed to read > > > > > > > > > > > > > > the "shmem" property instead? And the compatible string used for this > > > > > > > > > > > > > > seems to be "arm,scmi-shmem". > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > We use linux,scmi_mem node to reserve memory, needed for all > > > > > > > > > > > > > channels: > > > > > > > > > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > > > > > /* reserved region for scmi channels*/ > > > > > > > > > > > > > scmi_memory: linux,scmi_mem@53FF0000 { > > > > > > > > > > > > > no-map; > > > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > > > > > > > }; > > > > > > > > > > > > > }; > > > > > > > > > > > > > > > > > > > > > > > > > > arm,scmi-shmem node used in shmem property defines only 1 page needed to > > > > > > > > > > > > > the current scmi channel: > > > > > > > > > > > > > > > > > > > > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > > > > }; > > > > > > > > > > > > > > > > > > > > > > > > > > For each Domain reg points to unigue page from linux,scmi_mem region, > > > > > > > > > > > > > assigned to this agent. > > > > > > > > > > > > > > > > > > > > > > > > If we were to use "linux,scmi_mem" we would have to introduce it as a > > > > > > > > > > > > compatible string, not as a node name, and it would need to be described > > > > > > > > > > > > in Documentation/devicetree/bindings/firmware/arm,scmi.yaml. > > > > > > > > > > > > > > > > > > > > > > > > But from your description I don't think it is necessary. We can just use > > > > > > > > > > > > "arm,scmi-shmem" to describe all the required regions: > > > > > > > > > > > > > > > > > > > > > > > > reserved-memory { > > > > > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > > > }; > > > > > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > > > > > }; > > > > > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > > > > > }; > > > > > > > > > > > > ... > > > > > > > > > > > > > > > > > > > > > > > > In other words, if all the individual channel pages are described as > > > > > > > > > > > > "arm,scmi-shmem", why do we also need a single larger region as > > > > > > > > > > > > "linux,scmi_mem"? > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > That was my first implementation. But I've met a problem with > > > > > > > > > > > scmi driver in kernel. I don't remember the exact place, but I remember > > > > > > > > > > > there were some if, checking if memory weren't reserved. > > > > > > > > > > > That's why I ended up splitting nodes reserved memory region and actual > > > > > > > > > > > shmem page. > > > > > > > > > > > For linux,scmi_mem node I took format from /reserved-memory/linux,lossy_decompress@54000000, > > > > > > > > > > > which has no compatible string and provides no-map property. > > > > > > > > > > > linux,scmi_shmem node is needed to prevent xen from allocating this > > > > > > > > > > > space for the domain. > > > > > > > > > > > > > > > > > > > > > > Very interesting question about should I introduce linux,scmi_mem node > > > > > > > > > > > and scmi_devid property to the > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml? > > > > > > > > > > > Those node and property are needed only for Xen and useless for > > > > > > > > > > > non-virtualized systems. I can add this node and property description to > > > > > > > > > > > arm,scmi.yaml, but leave a note that this is Xen specific params. > > > > > > > > > > > What do you think about it? > > > > > > > > > > > > > > > > > > > > Reply below > > > > > > > > > > > > > > > > > > > > [...] > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > In general we can't use properties that are not part of the device tree > > > > > > > > > > > > spec, either https://urldefense.com/v3/__https://www.devicetree.org/specifications/__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXCSnJPN$ [devicetree[.]org] or > > > > > > > > > > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/devicetree/bindings__;!!GF_29dbcQIUBPA!kNodtgmOQBc1iO76_6vTK-O1SoLxee_ChowYQiQYC595rMOsrnmof2zmk7BnhXloYUaj$ [git[.]kernel[.]org] > > > > > > > > > > > > > > > > > > > > > > > > "linux,scmi_mem" is currently absent. Are you aware of any upstreaming > > > > > > > > > > > > activities to get "linux,scmi_mem" upstream under > > > > > > > > > > > > Documentation/devicetree/bindings in Linux? > > > > > > > > > > > > > > > > > > > > > > > > If "linux,scmi_mem" is going upstream in Linux, then we could use it. > > > > > > > > > > > > Otherwise, first "linux,scmi_mem" needs to be added somewhere under > > > > > > > > > > > > Documentation/devicetree/bindings (probably > > > > > > > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml), then we can > > > > > > > > > > > > work on the Xen code that makes use of it. > > > > > > > > > > > > > > > > > > > > > > > > Does it make sense? > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > Yes I agree. I think linux,scmi_mem and scmi_devid should be upstreamed. > > > > > > > > > > > I will add those properties to arm,scmi.yaml, mark them as related to XEN and send patch. > > > > > > > > > > > > > > > > > > > > I didn't realize that linux,scmi_mem and scmi_devid are supposed to be > > > > > > > > > > Xen specific. In general, it would be best not to introduce Xen specific > > > > > > > > > > properties into generic bindings. It is a problem both from a > > > > > > > > > > specification perspective (because it has hard to handle Xen specific > > > > > > > > > > cases in fully generic bindings, especially as those bindings are > > > > > > > > > > maintained as part of the Linux kernel) and from a user perspective > > > > > > > > > > (because now the user has to deal with a Xen-specific dtb, or has to > > > > > > > > > > modify the host dtb to add Xen-specific information by hand.) > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > Let me start from scmi_devid. Why would scmi_devid be Xen-specific? It > > > > > > > > > > looks like a generic property that should be needed for the Linux SCMI > > > > > > > > > > driver too. Why the Linux driver doesn't need it? > > > > > > > > > > > > > > > > > > > > > > > > > > > > scmi_devid used during domain build. It passed as input parameter for SCMI_BASE_SET_DEVICE_PERMISSIONS message. > > > > > > > > > On non-virtualized systems - there is no need of this call, because OS > > > > > > > > > is the only one entity, running on the system. > > > > > > > > > > > > > > > > OK. Even if it is only required for virtualized systems, I think that > > > > > > > > scmi_devid is important enough that should be part of the upstream > > > > > > > > binding. I think it is worth starting an email thread on the LKML with > > > > > > > > Rob Herring and the SCMI maintainers to discuss the addition of > > > > > > > > scmi_devid to the binding. > > > > > > > > > > > > > > > > > > > > > > > > > I've chatted with Volodymyr_Babchuk and he gave a great idea to add a > > > > > > > > > list of device_ids to dom.cfg, such as: > > > > > > > > > sci_devs = [ 0, 1, 15, 35 ]; > > > > > > > > > > > > > > > > > > Using this approach, we can remove scmi_devid from the device tree and > > > > > > > > > just pass a list of scmi_devids to XEN using additional hypercall. > > > > > > > > > We can probably make hypercall taking devid list as input parameter. > > > > > > > > > This will take only 1 hypercall to setup sci permissions. > > > > > > > > > > > > > > > > But how would a user know which are the right SCMI IDs to add to the > > > > > > > > sci_devs list? Would the user have to go and read the reference manual > > > > > > > > of the platform to find the SCMI IDs and then write sci_devs by hand? > > > > > > > > If that is the case, then I think that it would be better to add > > > > > > > > scmi_devid to device tree. > > > > > > > > > > > > > > > > In general, I think this configuration should happen automatically > > > > > > > > without user intervention. The user should just specify "enable SCMI" > > > > > > > > and it should work. > > > > > > > > > > > > > > > > > > > > > > > > > > In regards to linux,scmi_mem, I think it would be best to do without it > > > > > > > > > > and fix the Linux SCMI driver if we need to do so. Xen should be able to > > > > > > > > > > parse the native "arm,scmi-shmem" nodes and Linux (dom0 or domU) should > > > > > > > > > > be able to parse the "arm,scmi-shmem" nodes generated by Xen. Either > > > > > > > > > > way, I don't think we should need linux,scmi_mem. > > > > > > > > > > > > > > > > > > This requires further investigation. I will try to make implementation > > > > > > > > > without linux,scmi_mem, using only arm,scmi-shmem nodes and share > > > > > > > > > reuslts with you. > > > > > > > > > > > > > > > > OK, thanks. > > > > > > > > > > > > > > Hi Stefano, > > > > > > > > > > > > > > As I did some investigation about using reserved-memory area > > > > > > > linux,scmi_mem and now I need your advice. > > > > > > > > > > > > > > I see 2 possible implementations for now: > > > > > > > 1) Add memory-region parameter to cpu_scp_shm node which points to the > > > > > > > reserved memory region. > > > > > > > So device-tree will look like this: > > > > > > > > > > > > > > reserved-memory { > > > > > > > /* reserved region for scmi channels*/ > > > > > > > scmi_memory: region@53FF0000{ > > > > > > > no-map; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > > > > > }; > > > > > > > }; > > > > > > > cpu_scp_shm: scp-shmem@0x53FF0000 { > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > memory-region = <&scmi_memory>; > > > > > > > }; > > > > > > > > > > > > > > So cpu_scp_shm node has a reference to scmi_memory region. This mean > > > > > > > that xen can find reserved memory region without adding additional names > > > > > > > to the device-tree bindings. > > > > > > > memory-region parameter as a reference to reserved memory and region > > > > > > > creation described in: > > > > > > > https://urldefense.com/v3/__https://github.com/torvalds/linux/blob/v5.15/Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt__;!!GF_29dbcQIUBPA!k6x19x1gYF1CPlgAZj7std3ifqhq-9DXvuF0nwonNPUwMzZpYHYbrRJziJrgdFIOjyan$ [github[.]com] > > > > > > > > > > > > > > This approach I've implemented already and it works. > > > > > > > > > > > > This approach would require a discussion with the upstream device tree > > > > > > maintainers. Likely, we would need to add a note about the usage of the > > > > > > "memory-region" property to arm,scmi.yaml. > > > > > > > > > > > > Also, I have the feeling that they would ask to add the "memory-region" > > > > > > property directly to the "arm,scmi-smc" node, as an alternative (or > > > > > > in addition) to the existing "shmem" property. > > > > > > > > > > > > That said, from my point of view this approach is also a viable option. > > > > > > I don't see any major problems. > > > > > > > > > > > > The main question (after reading everything else that you wrote below) > > > > > > is whether the "arm,scmi-smc" node in this case could be automatically > > > > > > generated. > > > > > > > > > > > > > > > > arm,scmi-smc node can be generated in both cases. I think I'd leave it > > > > > as backup in case if the second approach will not work. > > > > > > > > > > > > > > > > > > 2) The second approach is the format you suggested: > > > > > > > > > > > > reserved-memory { > > > > > > > > > > > > scp-shmem@0x53FF0000 { > > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > > reg = <0x0 0x53FF0000 0x0 0x1000>; > > > > > > > > > > > > }; > > > > > > > > > > > > scp-shmem@0x53FF1000 { > > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > > reg = <0x0 0x53FF1000 0x0 0x1000>; > > > > > > > > > > > > }; > > > > > > > > > > > > scp-shmem@0x53FF2000 { > > > > > > > > > > > > compatible = "arm,scmi-shmem"; > > > > > > > > > > > > reg = <0x0 0x53FF2000 0x0 0x1000>; > > > > > > > > > > > > }; > > > > > > > > > > > > ... > > > > > > > > > > > > > > This approach has an advantage that xen ARM_SCI driver do not know about > > > > > > > how channels are placed in the reserved memory, but introduces some > > > > > > > disadvantages: > > > > > > > a) We provide extra 14 (in our case) arm,scmi-shmem nodes which are not used > > > > > > > in the device-tree. In current implementation I have separate scmi.dtsi > > > > > > > file which introduces scmi support for both XEN-based and > > > > > > > non-virtualized systems. Having 14 extra channels in the device-tree may > > > > > > > be confusing. > > > > > > > > > > > > I can see that while it would be ideal for Xen to see all 14+1 channels > > > > > > in device tree (on the host device tree), we wouldn't want to expose all > > > > > > of them to the domains, not even to dom0. > > > > > > > > > > > > How many channels do we want dom0 to see by the way? For this > > > > > > discussion, I'll just assume for now that dom0 only sees 1 channel like > > > > > > the domUs. > > > > > > > > > > For dom0 we need only one channel. > > > > > > > > > > > > > > > > > Now we have a problem: how do we go about "filtering" the > > > > > > "arm,scmi-shmem" device tree nodes? Which is also what you are asking > > > > > > below in point b). > > > > > > > > > > > > > > > > Xen will not need to filter "arm,scmi-shmem" node. It will just > > > > > create shmem node in Domain device-tree. I don't see any problem for xen > > > > > configuration here. > > > > > What bothers me here is that I set scmi configuration for platform dts, > > > > > not for xen or domu dts files. > > > > > So for example I have the following structure of the dts files for my > > > > > platform (r8a77961-salvator-xs): > > > > > * r8a77961-scmi.dtsi - this file includes all scmi related nodes and set > > > > > scmi_devid for the devices, that should use scmi. > > > > > * r8a77961-salvator-xs.dts - dts file which generates dtb for the platform. > > > > > It includes r8a77961-scmi.dtsi so I populate scmi to platform dtb, which > > > > > is used for system with no hypervisor. > > > > > * r8a77961-salvator-xs-xen.dts - dts file for xen which includes > > > > > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > > > > * r8a77961-salvator-xs-domu.dts - dts file for DomU which includes > > > > > r8a77961-salvator-xs.dts and inherits scmi configuration from it. > > > > > > > > > > In this case r8a77961-salvator-xs.dtb r8a77961-salvator-xs-xen.dtb > > > > > r8a77961-salvator-xs-domu.dtb files will inherit 14+1 channel. > > > > > > > > > > I can give you a link to Merge request with this changes if you need it. > > > > > > > > > > For xen and domu dtb it is not a problem because all "arm,scmi-shmem" > > > > > nodes will be omitted and new will be generated for the domains. > > > > > > > > > > What bothers me is that r8a77961-salvator-xs.dtb will have 14 unused channels. > > > > > > > > > > Just got an idea while writing this: I can create only one > > > > > "arm,scmi-shmem" node in r8a77961-scmi.dtsi and add 14 more nodes, > > > > > needed for xen explicitly in r8a77961-salvator-xs-xen.dts. > > > > > > > > > > Then we will have valid configurations for all cases. > > > > > This can be a solution. What do you think? > > > > > > > > It is good that you brought this up because it helps me explain what I > > > > mean. And of course it is up to you where you place the nodes in the > > > > various dts files at your disposal. Either way it would work but I think > > > > they should belong to r8a77961-salvator-xs.dts. > > > > > > > > Generally the platform vendor (e.g. Xilinx) provides a device tree > > > > description of the platform to use including all the available resources > > > > and firmware interfaces. In your case it would be r8a77961-scmi.dtsi + > > > > r8a77961-salvator-xs.dts. This is what I call the "host device tree" > > > > below. Users should be able to boot a fully functional system using the > > > > host device tree pretty much "as is" to run Xen, Linux or any other > > > > software. > > > > > > > > Certainly the SCMI device tree description should be part of the host > > > > device tree, so in your case it would be r8a77961-salvator-xs.dts. And > > > > the description should include all 14+1 channels because this is the > > > > generic platform description -- we cannot know for sure how the users > > > > are going to use the system. > > > > > > > > This is why r8a77961-salvator-xs-xen.dts should be as small as possible > > > > or ideally inexistent. There shouldn't be a need for a special device > > > > tree modification to allow Xen to run. In reality, even at Xilinx we > > > > have something like r8a77961-salvator-xs-xen.dts, although it is really > > > > small. > > > > > > > > But I see that r8a77961-salvator-xs-xen.dts could be viewed as the > > > > device tree additions to run hypervisors and from that point of view it > > > > is more acceptable to place the 14 channels there. > > > > > > > > The biggest problem is r8a77961-salvator-xs-domu.dts: who is going to > > > > write it? And how? It wouldn't be provided by the platform vendor, so it > > > > is the user the one that has to find a way to write it. > > > > > > > > I know the user already has to write a partial DTB for device > > > > assignment, but any time the process is more complex than "copy the host > > > > device tree node for device XXX to the partial DTB" it is a problem. > > > > Errors are made and the system doesn't work. > > > > > > > > I think we don't want to make it even more difficult by having to > > > > manually produce the SCMI domU description too. The SCMI description for > > > > domU could be automatically generated by Xen, or libxl/xl. If that's an > > > > issue, then the SCMI description could be automatically generated by an > > > > external tool but I think it would make things more complex and harder > > > > to maintain. > > > > > > > > In short my point of view is: > > > > - r8a77961-scmi.dtsi + r8a77961-salvator-xs.dts should be as generic as > > > > possible so the SCMI nodes should have 14+1 channels > > > > - but putting the 14 channels in r8a77961-salvator-xs-xen.dts is still > > > > OKish > > > > - it is important that r8a77961-salvator-xs-domu.dts is automatically > > > > generated by Xen or libxl or another software tool > > > > > > > > > > Thank you for the detailed response. I'll put all 14+1 channels to > > > r8a77961-salvator-xs.dts then. > > > I've described my thoughts about generation of the arm,scmi-smc node below. > > > > > > > > > > > > > > b) In case if we have all 15 channels, described in partial device-tree, > > > > > > > > > > > > I think you meant "described in the host device tree", right? > > > > > > > > > > > Yeah that's what I've meant. > > > > > > > > > > > > > we should not copy any node to the domain device-tree. I think it will > > > > > > > be better to generate arm,scmi-shmem node in the Domain device-tree. > > > > > > > > > > > > Yes, I think it makes sense for Xen to generate the "arm,scmi-shmem" > > > > > > device tree description for the DomU/Dom0 based on the channels > > > > > > allocated to the domain. > > > > > > > > > > > > > > > > > > > The problem is that arm,scmi-smc node, which is using arm,scmi-shmem > > > > > > > node can't be generated. I prefer it to be copied from the partial > > > > > > > device-tree because it includes some platform specific configuration, > > > > > > > such as func-id and list of the protocols (for example different > > > > > > > platforms may require different list of the protocols). So in this > > > > > > > case we will have 1 node copied and 1 node generated. > > > > > > > > > > > > > > I think even for dom0less we should use arm,scmi-smc node from the > > > > > > > device-tree because protocol configuration and funcid is related to the > > > > > > > platform. > > > > > > > > > > > > I am not sure I understood what you wrote. You are saying that the > > > > > > "arm,scmi-smc" node includes some platform specific configurations so > > > > > > it cannot be automatically generated by Xen (or by the tools) and > > > > > > instead it needs to be manually provided as part of the partial dtb for > > > > > > the domU. Is that correct? > > > > > > > > > > > > If so, I would like to understand the reasons behind it. Manual > > > > > > device tree editing is problematic. > > > > > > > > > > > > I looked for "func-id" in > > > > > > Documentation/devicetree/bindings/firmware/arm,scmi.yaml but couldn't > > > > > > find any results. Do you have an example of the platform specific > > > > > > configuration or protocol configuration that would make it difficult to > > > > > > automatically generate the "arm,scmi-smc" node for the domains? > > > > > > > > > > Sorry, I used wrong term (used term from the specification), arm,smc-id > > > > > of cause. > > > > > > > > > > > > > > > > > Also, is this a problem just for approach #2 or also for approach #1? > > > > > > If it is a problem only for approach #2, then let's just go with > > > > > > approach #1. > > > > > > > > > > > > > > > > We can't copy "arm,scmi-smc" in both approaches. The difference is that > > > > > in the first approach we can copy both "arm,scmi-smc" and > > > > > "arm,scmi-shmem" nodes while in the second approach we should copy > > > > > "arm,scmi-smc", but we have to generate "arm,scmi-shmem" node. > > > > > > > > > > arm,scmi-smc node can't be generated because it includes properties and > > > > > configurations that depends from platform and should be get from the > > > > > device tree. > > > > > Here is "arm,scmi-smc" node expample: > > > > > firmware { > > > > > scmi { > > > > > compatible = "arm,scmi-smc" > > > > > arm,smc-id = <0x82000002>; > > > > > shmem = <&cpu_scp_shm>; > > > > > #address-cells = <1>; > > > > > #size-cells = <0>; > > > > > scmi_power: protocol@11 { > > > > > reg = <0x11>; > > > > > #power-domain-cells = <1>; > > > > > }; > > > > > > > > > > scmi_clock: protocol@14 { > > > > > ... > > > > > }; > > > > > > > > > > scmi_reset: protocol@16 { > > > > > ... > > > > > }; > > > > > ... > > > > > }; > > > > > }; > > > > > > > > > > It has 3 configurable options: > > > > > * arm,smc-id parameter, setting func_id for scmi protocol. This id can be > > > > > different for different platforms. > > > > > For example stm32mp1 architecture use different scm-id for different > > > > > agents: > > > > > https://urldefense.com/v3/__https://github.com/ARM-software/arm-trusted-firmware/blob/0586c41b3f2d52aae847b7212e7b0c7e19197ea2/plat/st/stm32mp1/include/stm32mp1_smc.h*L39__;Iw!!GF_29dbcQIUBPA!mTRUjtSg19iVaYo3Cgjop5ckPWqKsHVo1EZCEA1zCbod9KpNSXX291A8vAuNdTCr46MA$ [github[.]com] > > > > > > > > > > * shmem which includes phandle to arm,scmi-shmem node. But this is not > > > > > a problem and can be updated. > > > > > > > > > > * list of the protocol subnodes. This is also configurable parameter, > > > > > not regs or names, but the number of the protocols. For example onle > > > > > platform can use power-domains/clock/resets via scmi, when another will > > > > > require volage-control and sensor-management to be added. > > > > > > > > > > Xen should know this parameters to be able to generate "arm,scmi-smc" node. > > > > > > > > > > Also we're currently discussing new scmi protocol with ARM: Pinctrl over > > > > > SCMI. > > > > > > > > > > It should allow domains to access pinctrl subsystem, placed in Firmware > > > > > through SCMI protocol. > > > > > scmi_pinctrl node will look like this: > > > > > > > > > > firmware { > > > > > scmi { > > > > > ... > > > > > scmi_pinctrl: protocol@18 { > > > > > reg = <0x18>; > > > > > #pinctrl-cells = <0>; > > > > > > > > > > i2c2_pins: i2c2 { > > > > > groups = <74>; /* i2c2_a */ > > > > > function = <15>; /* i2c2 */ > > > > > }; > > > > > > > > > > irq0_pins: irq0 { > > > > > groups = <81>; /* intc_ex_irq0 */ > > > > > function = <19>; /* intc_ex */ > > > > > }; > > > > > > > > > > avb_pins: avb { > > > > > mux { > > > > > /* avb_link, avb_mdio, avb_mii */ > > > > > groups = <17>, <21>, <22>; > > > > > function = <1>; /* avb */ > > > > > }; > > > > > > > > > > pins_mdio { > > > > > groups = <21>; /* avb_mdio */ > > > > > drive-strength = <24>; > > > > > }; > > > > > > > > > > pins_mii_tx { > > > > > /* PIN_AVB_TX_CTL, PIN_AVB_TXC, PIN_AVB_TD0, > > > > > PIN_AVB_TD1, PIN_AVB_TD2, PIN_AVB_TD3 */ > > > > > pins = <242>, <240>, <236>, <237>, <238>, <239>; > > > > > drive-strength = <12>; > > > > > }; > > > > > }; > > > > > ... > > > > > }; > > > > > }; > > > > > }; > > > > > > > > > > So "arm,scmi-smc" node will have even more platform specific settings. > > > > > > > > > > > > > > > > > > I prefer the second approach and will try to make it if it's OK to copy > > > > > > > arm,scmi-smc node from partial Device-tree and generate arm,scmi-shmem > > > > > > > node. > > > > > > > > > > > > > > What do you think about that? > > > > > > > > > > > > From a device tree specification perspective, I think both approaches > > > > > > are OK (with a minor comment on the first approach as I wrote above.) > > > > > > > > > > > > But from a Xen perspective I think it is important that we don't require > > > > > > the user to manually provide the SCMI configuration in the partial DTB. > > > > > > It would be better if we could generate it automatically from Xen or the > > > > > > tools (or even an independent script). Or copy the "arm,scmi-smc" node > > > > > > from the host device tree to the domU device tree without modifications. > > > > > > > > > > I think copy "arm,scmi-smc" node is the only option we have. > > > > > I'm not sure what do you mean under "host device tree" if you mean Xen > > > > > device-tree - then I think it will not cover the case with stm32mp1 I've > > > > > mentioned above. I think it will be better to copy "arm,scmi-smc" node > > > > > from Domu partial Device-tree to Domu device-tree. > > > > > So AGENT0 smc-id will be set in xen device-tree and copied to dom0 and > > > > > AGENT1 scm-is set in domu device-tree and copied to dom-u. > > > > > > > > > > Do you agree with my points? > > > > > > > > I think we are saying similar things, but we are getting mixed up with > > > > the terminology. Let's start from the basics :-) > > > > > > > > # Host device tree > > > > The device tree given to Xen at boot time. This is the device tree that > > > > Xen parses to discover what's available on the platform. In your case, > > > > it seems to include r8a77961-salvator-xs-xen.dts. > > > > > > > > # Partial DTB > > > > (Ignoring Dom0less) this is the small DTB that gets passed to xl with > > > > the "device_tree" option in the xl config file. It is copied verbatim > > > > to the domU device tree by xl/libxl. > > > > > > > > # Copy the "arm,scmi-smc" node from host device tree > > > > This means that the domU "arm,scmi-smc" node is an exact copy of the > > > > host device tree SCMI node. I don't think this is actually possible in > > > > most cases because the domU description is typically a bit different > > > > from the host description. For instance, the host description could > > > > include 14+1 channels while the domU description should only include 1 > > > > channel. > > > > > > > > # Copy the "arm,scmi-smc" node from the partial DTB > > > > This implies that somebody or something create an "arm,scmi-smc" node > > > > for the domU and placed it into the partial DTB. Then, Xen and/or > > > > xl/libxl will copy the node from the partial DTB to the DomU device > > > > tree. The main question in this case is: who is going to write the > > > > partial DTB? We dont want the user (i.e. a person) to have to manually > > > > write the SCMI description for the domU. It should be an automated tools > > > > that does it. At that point, it is easier if it is Xen or xl/libxl. > > > > Alternativaly, we could think of an external tool but I think it would > > > > make things more difficult to maintain. > > > > > > > > # Generate the "arm,scmi-smc" node for domUs > > > > When I write "generate the arm,scmi-smc node", I mean that Xen and > > > > libxl/xl will generate the "arm,scmi-smc" node for the domU. Thus, the > > > > node will not be copied from the partial DTB or from the device tree, > > > > instead, it should be created directly by Xen and/or libxl/xl. > > > > > > > > However, the domU "arm,scmi-smc" node could still be derived from the > > > > host device tree "arm,scmi-smc" node. In other words, Xen or xl/libxl > > > > would look at the host device tree "arm,scmi-smc" node, copy it to the > > > > domU device tree while making as many changes as necessary. > > > > > > > > The DomU "arm,scmi-smc" node doesn't have to be entirely fake and > > > > static. It could be dynamically created to match the host device tree > > > > description. I think this is the best option. > > > > > > > > > > > > # Conclusion > > > > I am suggesting that Xen and/or libxl automatically produce the > > > > "arm,scmi-smc" node for domUs based on the host device tree description > > > > and based on the channel mapped to the domU. This way, the user (a > > > > person) doesn't have to go and manually edit the domU partial DTB. > > > > > > > > > > That sounds reasonable. The problem is that arm,scmi-smc node can be > > > copmlicated and include a lot of configuration. Also for different > > > mediators this node can be different. > > > As I inderstand, there is no mechanism for xl to access host device-tree > > > right now. Correct me if I'm wrong. > > > > Yes, you are right. And the lack of a mechanism for xl to access the > > host device tree is a problem. > > > > > > > I see the following way we can generate arm,scmi-smc node for DomU: > > > We say that if scmi-smc mediator is enabled - then Dom0 is configured to > > > use SCMI. This means that Dom0 device-tree will have arm,scmi-smc node > > > and it can be reached from the userspace. > > > > So far so good > > > > > > > In this case xl can use infromation from /proc/device-tree/firmware/scmi > > > to generate arm,scmi-smc node for DomU. But in this case xl should know > > > the exact path of scmi node. > > > > > > Or we can generate some special node, called "shared" in Dom0 device-tree > > > which will include copy of the arm,scmi-smc node, which can be used for domains. > > > In this case xl can scan /proc/device-tree/shared node and find > > > arm,scmi-smc copatible node and use it to generate arm,scmi-smc node for > > > DomU. > > > Also this can be used for another features in future. > > > > > > What do you think about this? > > > > Basing the domU SCMI node generation on the dom0 SCMI node is not great, > > because it should be based on the host device tree rather than dom0 and > > the dom0 SCMI description will be different. > > > > Instead of copying just the host SCMI node somewhere in the dom0 device > > tree so that xl/libxl can access it via /proc/device-tree, I think it > > would be better to make the full host DTB available to xl/libxl. > > > > There are probably many ways to do this, but there are a couple I can > > think of on top of my head: > > > > - introduce a new hypercall to get the full host dtb from Xen > > The hypercall would pass an address and a size in guest physical memory > > and Xen would copy the host DTB to it. > > > > - introduce something like /proc/device-tree to hypfs > > See xen/common/hypfs.c > > > Hi Stefano, > > I've found the discussion, indirectly related to our topic [1], in which > hypervisor <-> toolchain interface topic was raised. > > Where Andrii Anisov proposed to introduce another libxl type (f.e. File) which > will read device tree file and operate with a binary blob. > > His idea is to introduce new Domctl hypecall, which is able to send device-tree > blob from toolstack to the hypervisor. > I was thinking about using his approach, but in opposite direction. > Such as introduce a set of DOMCTL, which gives ability for xl user to request > Host Device-tree nodes. > I see something like that: > XEN_DOMCTL_host_node_by_path - gets node path as input and returns blob with node; > XEN_DOMCTL_find_host_compatible_node - gets compatible string and "skip" number > for the case if there is more than 1 compatible node and returns blob with node. > > Advandages for my sight are: > - I hope Andrii still has some code, which can be reused during the implementation; > - Xen can determine if the requesting node can be exposed to the guest, which > is additional layer of validation; > - no need to transfer all device-tree to the toolstack. > > As for the hypfs this sounds like a separate feature to me. I'm not sure if we > need to provide the entire copy of the device-tree. Also there could be some > problems with this solution. > For example, xl tool have to scan whole hypfs tree to find node by phandle or > by compatible string. > > Looking forward for your thoughts about that. Also I would be happy if Julien > could share his vision. > > [1] https://xen.markmail.org/search/?q=from%3AAndrii+Anisov+device-tree#query:from%3AAndrii%20Anisov%20device-tree+page:2+mid:tvgyp7fzpiyyso3j+state:results It looks like XEN_DOMCTL_host_node_by_path and XEN_DOMCTL_find_host_compatible_node would also solve the problem but I think that a single hypercall that retrieves the entire host DTB would be easier to implement and more robust in the long term. hypfs has the advantage that it would create an interface more similar to the one people are already used to on Linux systems (/proc/device-tree). xl/libxl would have to scan the whole hypfs tree, which intuitively I think it would be slower. Also the feature might be harder to implement but I am not sure. I don't have a strong preference and this is not a stable interface (we don't have to be extra paranoid about forward and backward compatibility). So I am fine either way. Let's see what the others think as well.
Hi, On 24/01/2022 19:06, Stefano Stabellini wrote: > It looks like XEN_DOMCTL_host_node_by_path and > XEN_DOMCTL_find_host_compatible_node would also solve the problem but I > think that a single hypercall that retrieves the entire host DTB would > be easier to implement DOMCTL should only be used to handle per-domain information. If we want to create a new sub-hypercall of either __HYPERVISOR_platform_op or __HYPERVISOR_sysctl_op (not sure which one). AFAICT, both are versioned. > and more robust in the long term. > > hypfs has the advantage that it would create an interface more similar > to the one people are already used to on Linux systems > (/proc/device-tree). xl/libxl would have to scan the whole hypfs tree, > which intuitively I think it would be slower. Even if you have the binary blob, you would still have to scan the device-tree. That said, it is probably going to be potentially a bit faster because you have less hypercall. However, here this is a trade-off between memory use and speed. If you want speed, then you may have to transfer up to 2MB every time. So the question is do we care more about speed or memory usage? > Also the feature might be > harder to implement but I am not sure. > > I don't have a strong preference and this is not a stable interface (we > don't have to be extra paranoid about forward and backward > compatibility). So I am fine either way. Let's see what the others think > as well. My preference would be to use hypfs as this is cleaner than exposing a blob. However, are we sure we can simply copy the content of the host Device-Tree to the guest Device-Tree for SCMI? For instance, I know that for device passthrough there are some property that needs to be altered for some devices. Hence, why it is not present. Although, I vaguely recalled to have written a PoC, not sure if it was posted on the ML. Cheeers,
On Mon, 24 Jan 2022, Julien Grall wrote: > On 24/01/2022 19:06, Stefano Stabellini wrote: > > It looks like XEN_DOMCTL_host_node_by_path and > > XEN_DOMCTL_find_host_compatible_node would also solve the problem but I > > think that a single hypercall that retrieves the entire host DTB would > > be easier to implement > > DOMCTL should only be used to handle per-domain information. If we want to > create a new sub-hypercall of either __HYPERVISOR_platform_op or > __HYPERVISOR_sysctl_op (not sure which one). > > AFAICT, both are versioned. > > > and more robust in the long term. > > > hypfs has the advantage that it would create an interface more similar > > to the one people are already used to on Linux systems > > (/proc/device-tree). xl/libxl would have to scan the whole hypfs tree, > > which intuitively I think it would be slower. > > Even if you have the binary blob, you would still have to scan the > device-tree. That said, it is probably going to be potentially a bit faster > because you have less hypercall. > > However, here this is a trade-off between memory use and speed. If you want > speed, then you may have to transfer up to 2MB every time. So the question is > do we care more about speed or memory usage? > > > Also the feature might be > > harder to implement but I am not sure. > > > > I don't have a strong preference and this is not a stable interface (we > > don't have to be extra paranoid about forward and backward > > compatibility). So I am fine either way. Let's see what the others think > > as well. > > My preference would be to use hypfs as this is cleaner than exposing a blob. That's also fine by me. Probably the hypfs implementation shouldn't be much more difficult than something like XEN_DOMCTL_host_node_by_path/XEN_DOMCTL_find_host_compatible_node. > However, are we sure we can simply copy the content of the host Device-Tree to > the guest Device-Tree for SCMI? For instance, I know that for device > passthrough there are some property that needs to be altered for some devices. > Hence, why it is not present. Although, I vaguely recalled to have written a > PoC, not sure if it was posted on the ML. The SCMI node cannot be copied "as is" from host to guest. It needs a couple of changes but they seem feasible as they are limited to the channels exposed to the guest. (The generic device passthrough case is a lot more difficult.)
On Mon, Jan 24, 2022 at 02:14:43PM -0800, Stefano Stabellini wrote: > On Mon, 24 Jan 2022, Julien Grall wrote: > > On 24/01/2022 19:06, Stefano Stabellini wrote: > > > It looks like XEN_DOMCTL_host_node_by_path and > > > XEN_DOMCTL_find_host_compatible_node would also solve the problem but I > > > think that a single hypercall that retrieves the entire host DTB would > > > be easier to implement > > > > DOMCTL should only be used to handle per-domain information. If we want to > > create a new sub-hypercall of either __HYPERVISOR_platform_op or > > __HYPERVISOR_sysctl_op (not sure which one). > > > > AFAICT, both are versioned. > > > > > and more robust in the long term. > > > > hypfs has the advantage that it would create an interface more similar > > > to the one people are already used to on Linux systems > > > (/proc/device-tree). xl/libxl would have to scan the whole hypfs tree, > > > which intuitively I think it would be slower. > > > > Even if you have the binary blob, you would still have to scan the > > device-tree. That said, it is probably going to be potentially a bit faster > > because you have less hypercall. > > > > However, here this is a trade-off between memory use and speed. If you want > > speed, then you may have to transfer up to 2MB every time. So the question is > > do we care more about speed or memory usage? > > > > > Also the feature might be > > > harder to implement but I am not sure. > > > > > > I don't have a strong preference and this is not a stable interface (we > > > don't have to be extra paranoid about forward and backward > > > compatibility). So I am fine either way. Let's see what the others think > > > as well. > > > > My preference would be to use hypfs as this is cleaner than exposing a blob. > > That's also fine by me. Probably the hypfs implementation shouldn't be > much more difficult than something like > XEN_DOMCTL_host_node_by_path/XEN_DOMCTL_find_host_compatible_node. > > > > However, are we sure we can simply copy the content of the host Device-Tree to > > the guest Device-Tree for SCMI? For instance, I know that for device > > passthrough there are some property that needs to be altered for some devices. > > Hence, why it is not present. Although, I vaguely recalled to have written a > > PoC, not sure if it was posted on the ML. > > The SCMI node cannot be copied "as is" from host to guest. It needs a > couple of changes but they seem feasible as they are limited to the > channels exposed to the guest. (The generic device passthrough case is a > lot more difficult.) Hi Stefano, What I'm thinking about is do we actually need to create SCMI node in DomU device-tree? I have this question is because we don't need SCMI node to be present in DomU device-tree if it has no passed-through devices, which are using scmi. So if we don't have passed-through devices or do not provide DomU partial device-tree in config, then there is no need to create SCMI node. For now I see the following possible domu configurations: 1) If DomU has a lot of passed-through devices and it's easier to inherit host device-tree and disable not passed-through devices. Partial device tree will looks like this: #include "r8a77961-salvator-xs.dts" //include host device tree / { soc { ... } }; // Disable non passed-through devices &hscif { status = "disabled"; }; In this case DomU partial device-tree will inherit arm,scmi-smc and arm,scmi-shmem nodes and all clock/reset/power-domains which are using scmi. All this nodes can be copied to DomU device-tree from partial device-tree. 2) DomU has few passed-through devices, so it's easier to add the device nodes to the passthrough node of DomU partial device-tree. DomU partial device-tree will look like this: { scmi_shmem: scp-shmem@0x53FF0000 { compatible = "arm,scmi-shmem"; reg = <0x0 0x53FF0000 0x0 0x10000>; }; scmi { arm,smc-id = <....>; compatible = "arm,scmi-smc"; shmem = <&scmi_shmem>; scmi_clock: protocol@14 { ... }; scmi_reset: protocol@16 { ... }; }; passthrough { hscif0: serial@e6540000 { compatible = "renesas,hscif-r8a77961"; scmi_devid = <5>; clocks = <&scmi_clock 5>; resets = <&scmi_reset 5>; ... }; }; }; As you can see in this case we have to manually copy arm,scmi-shmem and arm,scmi-smc nodes with hscif0 node or the device-tree compilation will fail. We can use 0x53FF0000, provided in arm,scmi-shmem node and map domain channel to this address and copy scmi related nodes to the DomU device-tree. This is useful when we need to expose only certain protocols to the DomU. Also it's easy to modify DomU scmi node, as we need for stm32mp1 for example when different smc-id should be set for DomU. 3) DomU doesn't have any passthrough nodes, which are using scmi. In this case we don't want SCMI nodes to be in the DomU device-tree. I see only one use-case when we may need scmi nodes to be generated by xl in DomU device-tree: Xen generates psci node to handle cpu_on and cpu_off. According to the Section 4.3.2.5 of the DEN0056C [1]: > For these power domains, this protocol can be used to implement PSCI CPU_SUSPEND, CPU_ON, CPU_FREEZE, CPU_DEFAULT_SUSPEND and CPU_OFF functions. So in theory psci node can use scmi to control cpu state. But this is not our use-case because we don't want to give DomU ability to stop physical CPU. Xen can't intercept and handle CPU_ON and CPU_OFF requests when mailbox transport is used for SCMI communication. [1] "SCMI Specification DEN0056C," [Online]. Available: https://developer.arm.com/documentation/den0056/latest Best regards, Oleksii.
On Tue, 25 Jan 2022, Oleksii Moisieiev wrote: > On Mon, Jan 24, 2022 at 02:14:43PM -0800, Stefano Stabellini wrote: > > On Mon, 24 Jan 2022, Julien Grall wrote: > > > On 24/01/2022 19:06, Stefano Stabellini wrote: > > > > It looks like XEN_DOMCTL_host_node_by_path and > > > > XEN_DOMCTL_find_host_compatible_node would also solve the problem but I > > > > think that a single hypercall that retrieves the entire host DTB would > > > > be easier to implement > > > > > > DOMCTL should only be used to handle per-domain information. If we want to > > > create a new sub-hypercall of either __HYPERVISOR_platform_op or > > > __HYPERVISOR_sysctl_op (not sure which one). > > > > > > AFAICT, both are versioned. > > > > > > > and more robust in the long term. > > > > > hypfs has the advantage that it would create an interface more similar > > > > to the one people are already used to on Linux systems > > > > (/proc/device-tree). xl/libxl would have to scan the whole hypfs tree, > > > > which intuitively I think it would be slower. > > > > > > Even if you have the binary blob, you would still have to scan the > > > device-tree. That said, it is probably going to be potentially a bit faster > > > because you have less hypercall. > > > > > > However, here this is a trade-off between memory use and speed. If you want > > > speed, then you may have to transfer up to 2MB every time. So the question is > > > do we care more about speed or memory usage? > > > > > > > Also the feature might be > > > > harder to implement but I am not sure. > > > > > > > > I don't have a strong preference and this is not a stable interface (we > > > > don't have to be extra paranoid about forward and backward > > > > compatibility). So I am fine either way. Let's see what the others think > > > > as well. > > > > > > My preference would be to use hypfs as this is cleaner than exposing a blob. > > > > That's also fine by me. Probably the hypfs implementation shouldn't be > > much more difficult than something like > > XEN_DOMCTL_host_node_by_path/XEN_DOMCTL_find_host_compatible_node. > > > > > > > However, are we sure we can simply copy the content of the host Device-Tree to > > > the guest Device-Tree for SCMI? For instance, I know that for device > > > passthrough there are some property that needs to be altered for some devices. > > > Hence, why it is not present. Although, I vaguely recalled to have written a > > > PoC, not sure if it was posted on the ML. > > > > The SCMI node cannot be copied "as is" from host to guest. It needs a > > couple of changes but they seem feasible as they are limited to the > > channels exposed to the guest. (The generic device passthrough case is a > > lot more difficult.) > > > Hi Stefano, > > What I'm thinking about is do we actually need to create SCMI node in DomU device-tree? > I have this question is because we don't need SCMI node to be present in DomU > device-tree if it has no passed-through devices, which are using scmi. > So if we don't have passed-through devices or do not provide DomU partial device-tree > in config, then there is no need to create SCMI node. > > For now I see the following possible domu configurations: > 1) If DomU has a lot of passed-through devices and it's easier to inherit > host device-tree and disable not passed-through devices. > Partial device tree will looks like this: > > #include "r8a77961-salvator-xs.dts" //include host device tree > > / > { > soc { > ... > } > > }; > > // Disable non passed-through devices > &hscif { > status = "disabled"; > }; > > In this case DomU partial device-tree will inherit arm,scmi-smc and > arm,scmi-shmem nodes and all clock/reset/power-domains which are using scmi. > All this nodes can be copied to DomU device-tree from partial device-tree. This is an almost dom0 configuration. For this kind of use-cases, I think it is enough to handle dom0 automatically correctly. I wouldn't ask for anything more than that. > 2) DomU has few passed-through devices, so it's easier to add the device nodes > to the passthrough node of DomU partial device-tree. > DomU partial device-tree will look like this: > { > scmi_shmem: scp-shmem@0x53FF0000 { > compatible = "arm,scmi-shmem"; > reg = <0x0 0x53FF0000 0x0 0x10000>; > }; > scmi { > arm,smc-id = <....>; > compatible = "arm,scmi-smc"; > shmem = <&scmi_shmem>; > scmi_clock: protocol@14 { > ... > }; > scmi_reset: protocol@16 { > ... > }; > }; > passthrough { > hscif0: serial@e6540000 { > compatible = "renesas,hscif-r8a77961"; > scmi_devid = <5>; > clocks = <&scmi_clock 5>; > resets = <&scmi_reset 5>; > ... > }; > }; > }; > > As you can see in this case we have to manually copy arm,scmi-shmem and > arm,scmi-smc nodes with hscif0 node or the device-tree compilation will fail. > We can use 0x53FF0000, provided in arm,scmi-shmem node and map domain channel > to this address and copy scmi related nodes to the DomU device-tree. > This is useful when we need to expose only certain protocols to the DomU. > Also it's easy to modify DomU scmi node, as we need for stm32mp1 for example > when different smc-id should be set for DomU. I think this is the most interesting case that should be automated and not require manual intervention. Let me explain why. Currently we require partial device trees to be manually written because there is no easy way to automatically generate them. (I have some ideas on how to automatically generate partial device trees but that is a separate discussion.) Unfortunately, it has become increasingly clear that it is too difficult for users (even advanced users!) to come up with the appropriate partial device trees. Thus, I am reluctant to introduce more things that rely on the user having to manually specify partial device tree information. This is why I would like the SCMI nodes to be automatically added for domUs. Of course, if a user provides the scmi and scmi_shmem nodes in the partial device tree we could just use them. But ideally we should also be able to automatically generated them based on the host device tree nodes, so that the user only needs to provide serial@e6540000 (in your example, scmi_devid would need to be populated too) and the rest would be done automatically as we do today for the gic and vuart nodes. At the same time I don't want to scope-creep this patch series too much and I don't mean to ask you to add a huge new infrastructure to Xen and the Xen tools just to get SCMI support in. I would rather have a not-great automatic generation of the domU SCMI nodes than nothing (e.g. using your suggested XEN_DOMCTL_host_node_by_path and XEN_DOMCTL_find_host_compatible_node hypercalls althought they would need to be platform_op as Julien suggested). It looks like the generation of scmi_shmem and scmi should be easy enough that could be handled without difficulty in xl/libxl. But if that turns out to be too difficult, we could have a small independent bash/python script that from the host device tree generates the partial device tree with the SCMI nodes. From Xen point of view we are would still be using the partial device tree, but the partial device tree itself would be generated instead of manually written. As this workflow requires a separate tool I think it is a worse option than the one above. Still better than nothing though. > 3) DomU doesn't have any passthrough nodes, which are using scmi. > In this case we don't want SCMI nodes to be in the DomU device-tree. > > I see only one use-case when we may need scmi nodes to be generated by xl in > DomU device-tree: > Xen generates psci node to handle cpu_on and cpu_off. > According to the Section 4.3.2.5 of the DEN0056C [1]: > > For these power domains, this protocol can be used to implement PSCI CPU_SUSPEND, CPU_ON, CPU_FREEZE, CPU_DEFAULT_SUSPEND and CPU_OFF functions. > > So in theory psci node can use scmi to control cpu state. But this is not our > use-case because we don't want to give DomU ability to stop physical CPU. > Xen can't intercept and handle CPU_ON and CPU_OFF requests when mailbox transport > is used for SCMI communication. > > [1] "SCMI Specification DEN0056C," [Online]. Available: https://developer.arm.com/documentation/den0056/latest I agree with you on this one; I am not worried about this case.
On Tue, Jan 25, 2022 at 01:19:46PM -0800, Stefano Stabellini wrote: > On Tue, 25 Jan 2022, Oleksii Moisieiev wrote: > > On Mon, Jan 24, 2022 at 02:14:43PM -0800, Stefano Stabellini wrote: > > > On Mon, 24 Jan 2022, Julien Grall wrote: > > > > On 24/01/2022 19:06, Stefano Stabellini wrote: > > > > > It looks like XEN_DOMCTL_host_node_by_path and > > > > > XEN_DOMCTL_find_host_compatible_node would also solve the problem but I > > > > > think that a single hypercall that retrieves the entire host DTB would > > > > > be easier to implement > > > > > > > > DOMCTL should only be used to handle per-domain information. If we want to > > > > create a new sub-hypercall of either __HYPERVISOR_platform_op or > > > > __HYPERVISOR_sysctl_op (not sure which one). > > > > > > > > AFAICT, both are versioned. > > > > > > > > > and more robust in the long term. > > > > > > hypfs has the advantage that it would create an interface more similar > > > > > to the one people are already used to on Linux systems > > > > > (/proc/device-tree). xl/libxl would have to scan the whole hypfs tree, > > > > > which intuitively I think it would be slower. > > > > > > > > Even if you have the binary blob, you would still have to scan the > > > > device-tree. That said, it is probably going to be potentially a bit faster > > > > because you have less hypercall. > > > > > > > > However, here this is a trade-off between memory use and speed. If you want > > > > speed, then you may have to transfer up to 2MB every time. So the question is > > > > do we care more about speed or memory usage? > > > > > > > > > Also the feature might be > > > > > harder to implement but I am not sure. > > > > > > > > > > I don't have a strong preference and this is not a stable interface (we > > > > > don't have to be extra paranoid about forward and backward > > > > > compatibility). So I am fine either way. Let's see what the others think > > > > > as well. > > > > > > > > My preference would be to use hypfs as this is cleaner than exposing a blob. > > > > > > That's also fine by me. Probably the hypfs implementation shouldn't be > > > much more difficult than something like > > > XEN_DOMCTL_host_node_by_path/XEN_DOMCTL_find_host_compatible_node. > > > > > > > > > > However, are we sure we can simply copy the content of the host Device-Tree to > > > > the guest Device-Tree for SCMI? For instance, I know that for device > > > > passthrough there are some property that needs to be altered for some devices. > > > > Hence, why it is not present. Although, I vaguely recalled to have written a > > > > PoC, not sure if it was posted on the ML. > > > > > > The SCMI node cannot be copied "as is" from host to guest. It needs a > > > couple of changes but they seem feasible as they are limited to the > > > channels exposed to the guest. (The generic device passthrough case is a > > > lot more difficult.) > > > > > > Hi Stefano, > > > > What I'm thinking about is do we actually need to create SCMI node in DomU device-tree? > > I have this question is because we don't need SCMI node to be present in DomU > > device-tree if it has no passed-through devices, which are using scmi. > > So if we don't have passed-through devices or do not provide DomU partial device-tree > > in config, then there is no need to create SCMI node. > > > > For now I see the following possible domu configurations: > > 1) If DomU has a lot of passed-through devices and it's easier to inherit > > host device-tree and disable not passed-through devices. > > Partial device tree will looks like this: > > > > #include "r8a77961-salvator-xs.dts" //include host device tree > > > > / > > { > > soc { > > ... > > } > > > > }; > > > > // Disable non passed-through devices > > &hscif { > > status = "disabled"; > > }; > > > > In this case DomU partial device-tree will inherit arm,scmi-smc and > > arm,scmi-shmem nodes and all clock/reset/power-domains which are using scmi. > > All this nodes can be copied to DomU device-tree from partial device-tree. > > This is an almost dom0 configuration. For this kind of use-cases, I > think it is enough to handle dom0 automatically correctly. I wouldn't > ask for anything more than that. > > > > 2) DomU has few passed-through devices, so it's easier to add the device nodes > > to the passthrough node of DomU partial device-tree. > > DomU partial device-tree will look like this: > > { > > scmi_shmem: scp-shmem@0x53FF0000 { > > compatible = "arm,scmi-shmem"; > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > }; > > scmi { > > arm,smc-id = <....>; > > compatible = "arm,scmi-smc"; > > shmem = <&scmi_shmem>; > > scmi_clock: protocol@14 { > > ... > > }; > > scmi_reset: protocol@16 { > > ... > > }; > > }; > > passthrough { > > hscif0: serial@e6540000 { > > compatible = "renesas,hscif-r8a77961"; > > scmi_devid = <5>; > > clocks = <&scmi_clock 5>; > > resets = <&scmi_reset 5>; > > ... > > }; > > }; > > }; > > > > As you can see in this case we have to manually copy arm,scmi-shmem and > > arm,scmi-smc nodes with hscif0 node or the device-tree compilation will fail. > > We can use 0x53FF0000, provided in arm,scmi-shmem node and map domain channel > > to this address and copy scmi related nodes to the DomU device-tree. > > This is useful when we need to expose only certain protocols to the DomU. > > Also it's easy to modify DomU scmi node, as we need for stm32mp1 for example > > when different smc-id should be set for DomU. > > I think this is the most interesting case that should be automated and > not require manual intervention. Let me explain why. > > Currently we require partial device trees to be manually written because > there is no easy way to automatically generate them. (I have some ideas > on how to automatically generate partial device trees but that is a > separate discussion.) > > Unfortunately, it has become increasingly clear that it is too difficult > for users (even advanced users!) to come up with the appropriate partial > device trees. Thus, I am reluctant to introduce more things that rely on > the user having to manually specify partial device tree information. > This is why I would like the SCMI nodes to be automatically added for > domUs. > > Of course, if a user provides the scmi and scmi_shmem nodes in the > partial device tree we could just use them. But ideally we should also > be able to automatically generated them based on the host device tree > nodes, so that the user only needs to provide serial@e6540000 (in your > example, scmi_devid would need to be populated too) and the rest would > be done automatically as we do today for the gic and vuart nodes. > > At the same time I don't want to scope-creep this patch series too much > and I don't mean to ask you to add a huge new infrastructure to Xen and > the Xen tools just to get SCMI support in. I would rather have a > not-great automatic generation of the domU SCMI nodes than nothing (e.g. > using your suggested XEN_DOMCTL_host_node_by_path and > XEN_DOMCTL_find_host_compatible_node hypercalls althought they would > need to be platform_op as Julien suggested). > > It looks like the generation of scmi_shmem and scmi should be easy > enough that could be handled without difficulty in xl/libxl. But if that > turns out to be too difficult, we could have a small independent > bash/python script that from the host device tree generates the partial > device tree with the SCMI nodes. From Xen point of view we are would > still be using the partial device tree, but the partial device tree > itself would be generated instead of manually written. As this workflow > requires a separate tool I think it is a worse option than the one > above. Still better than nothing though. > Hi Stefano, Thank you for the detail answer. I went through hypfs and will try to export host device_tree. Then xl will be able to use hypfs data to generate arm,scmi-smc node if arm,scmi-smc node was not provided in DomU partial device-tree. Unfortunately, some changes should be done to hypfs because it seems not ready to handle nested dynamic dirs. I'll see if I can update hypfs without breaking the original functionality. If not, I will have to create all hypfs dir structure on start. For now I'm working on making dynamically created hypfs tree structure, based on host device-tree. Best regards, Oleksii. > > > 3) DomU doesn't have any passthrough nodes, which are using scmi. > > In this case we don't want SCMI nodes to be in the DomU device-tree. > > > > I see only one use-case when we may need scmi nodes to be generated by xl in > > DomU device-tree: > > Xen generates psci node to handle cpu_on and cpu_off. > > According to the Section 4.3.2.5 of the DEN0056C [1]: > > > For these power domains, this protocol can be used to implement PSCI CPU_SUSPEND, CPU_ON, CPU_FREEZE, CPU_DEFAULT_SUSPEND and CPU_OFF functions. > > > > So in theory psci node can use scmi to control cpu state. But this is not our > > use-case because we don't want to give DomU ability to stop physical CPU. > > Xen can't intercept and handle CPU_ON and CPU_OFF requests when mailbox transport > > is used for SCMI communication. > > > > [1] "SCMI Specification DEN0056C," [Online]. Available: https://urldefense.com/v3/__https://developer.arm.com/documentation/den0056/latest__;!!GF_29dbcQIUBPA!k5oB4BpbIN-hU5jrWvNy9FLXi3Kavu3qTr5lESjK8NnlS261E0Nuqg2_pUQWxb2hDdLa$ [developer[.]arm[.]com] > > I agree with you on this one; I am not worried about this case.
On Thu, 27 Jan 2022, Oleksii Moisieiev wrote: > On Tue, Jan 25, 2022 at 01:19:46PM -0800, Stefano Stabellini wrote: > > On Tue, 25 Jan 2022, Oleksii Moisieiev wrote: > > > On Mon, Jan 24, 2022 at 02:14:43PM -0800, Stefano Stabellini wrote: > > > > On Mon, 24 Jan 2022, Julien Grall wrote: > > > > > On 24/01/2022 19:06, Stefano Stabellini wrote: > > > > > > It looks like XEN_DOMCTL_host_node_by_path and > > > > > > XEN_DOMCTL_find_host_compatible_node would also solve the problem but I > > > > > > think that a single hypercall that retrieves the entire host DTB would > > > > > > be easier to implement > > > > > > > > > > DOMCTL should only be used to handle per-domain information. If we want to > > > > > create a new sub-hypercall of either __HYPERVISOR_platform_op or > > > > > __HYPERVISOR_sysctl_op (not sure which one). > > > > > > > > > > AFAICT, both are versioned. > > > > > > > > > > > and more robust in the long term. > > > > > > > hypfs has the advantage that it would create an interface more similar > > > > > > to the one people are already used to on Linux systems > > > > > > (/proc/device-tree). xl/libxl would have to scan the whole hypfs tree, > > > > > > which intuitively I think it would be slower. > > > > > > > > > > Even if you have the binary blob, you would still have to scan the > > > > > device-tree. That said, it is probably going to be potentially a bit faster > > > > > because you have less hypercall. > > > > > > > > > > However, here this is a trade-off between memory use and speed. If you want > > > > > speed, then you may have to transfer up to 2MB every time. So the question is > > > > > do we care more about speed or memory usage? > > > > > > > > > > > Also the feature might be > > > > > > harder to implement but I am not sure. > > > > > > > > > > > > I don't have a strong preference and this is not a stable interface (we > > > > > > don't have to be extra paranoid about forward and backward > > > > > > compatibility). So I am fine either way. Let's see what the others think > > > > > > as well. > > > > > > > > > > My preference would be to use hypfs as this is cleaner than exposing a blob. > > > > > > > > That's also fine by me. Probably the hypfs implementation shouldn't be > > > > much more difficult than something like > > > > XEN_DOMCTL_host_node_by_path/XEN_DOMCTL_find_host_compatible_node. > > > > > > > > > > > > > However, are we sure we can simply copy the content of the host Device-Tree to > > > > > the guest Device-Tree for SCMI? For instance, I know that for device > > > > > passthrough there are some property that needs to be altered for some devices. > > > > > Hence, why it is not present. Although, I vaguely recalled to have written a > > > > > PoC, not sure if it was posted on the ML. > > > > > > > > The SCMI node cannot be copied "as is" from host to guest. It needs a > > > > couple of changes but they seem feasible as they are limited to the > > > > channels exposed to the guest. (The generic device passthrough case is a > > > > lot more difficult.) > > > > > > > > > Hi Stefano, > > > > > > What I'm thinking about is do we actually need to create SCMI node in DomU device-tree? > > > I have this question is because we don't need SCMI node to be present in DomU > > > device-tree if it has no passed-through devices, which are using scmi. > > > So if we don't have passed-through devices or do not provide DomU partial device-tree > > > in config, then there is no need to create SCMI node. > > > > > > For now I see the following possible domu configurations: > > > 1) If DomU has a lot of passed-through devices and it's easier to inherit > > > host device-tree and disable not passed-through devices. > > > Partial device tree will looks like this: > > > > > > #include "r8a77961-salvator-xs.dts" //include host device tree > > > > > > / > > > { > > > soc { > > > ... > > > } > > > > > > }; > > > > > > // Disable non passed-through devices > > > &hscif { > > > status = "disabled"; > > > }; > > > > > > In this case DomU partial device-tree will inherit arm,scmi-smc and > > > arm,scmi-shmem nodes and all clock/reset/power-domains which are using scmi. > > > All this nodes can be copied to DomU device-tree from partial device-tree. > > > > This is an almost dom0 configuration. For this kind of use-cases, I > > think it is enough to handle dom0 automatically correctly. I wouldn't > > ask for anything more than that. > > > > > > > 2) DomU has few passed-through devices, so it's easier to add the device nodes > > > to the passthrough node of DomU partial device-tree. > > > DomU partial device-tree will look like this: > > > { > > > scmi_shmem: scp-shmem@0x53FF0000 { > > > compatible = "arm,scmi-shmem"; > > > reg = <0x0 0x53FF0000 0x0 0x10000>; > > > }; > > > scmi { > > > arm,smc-id = <....>; > > > compatible = "arm,scmi-smc"; > > > shmem = <&scmi_shmem>; > > > scmi_clock: protocol@14 { > > > ... > > > }; > > > scmi_reset: protocol@16 { > > > ... > > > }; > > > }; > > > passthrough { > > > hscif0: serial@e6540000 { > > > compatible = "renesas,hscif-r8a77961"; > > > scmi_devid = <5>; > > > clocks = <&scmi_clock 5>; > > > resets = <&scmi_reset 5>; > > > ... > > > }; > > > }; > > > }; > > > > > > As you can see in this case we have to manually copy arm,scmi-shmem and > > > arm,scmi-smc nodes with hscif0 node or the device-tree compilation will fail. > > > We can use 0x53FF0000, provided in arm,scmi-shmem node and map domain channel > > > to this address and copy scmi related nodes to the DomU device-tree. > > > This is useful when we need to expose only certain protocols to the DomU. > > > Also it's easy to modify DomU scmi node, as we need for stm32mp1 for example > > > when different smc-id should be set for DomU. > > > > I think this is the most interesting case that should be automated and > > not require manual intervention. Let me explain why. > > > > Currently we require partial device trees to be manually written because > > there is no easy way to automatically generate them. (I have some ideas > > on how to automatically generate partial device trees but that is a > > separate discussion.) > > > > Unfortunately, it has become increasingly clear that it is too difficult > > for users (even advanced users!) to come up with the appropriate partial > > device trees. Thus, I am reluctant to introduce more things that rely on > > the user having to manually specify partial device tree information. > > This is why I would like the SCMI nodes to be automatically added for > > domUs. > > > > Of course, if a user provides the scmi and scmi_shmem nodes in the > > partial device tree we could just use them. But ideally we should also > > be able to automatically generated them based on the host device tree > > nodes, so that the user only needs to provide serial@e6540000 (in your > > example, scmi_devid would need to be populated too) and the rest would > > be done automatically as we do today for the gic and vuart nodes. > > > > At the same time I don't want to scope-creep this patch series too much > > and I don't mean to ask you to add a huge new infrastructure to Xen and > > the Xen tools just to get SCMI support in. I would rather have a > > not-great automatic generation of the domU SCMI nodes than nothing (e.g. > > using your suggested XEN_DOMCTL_host_node_by_path and > > XEN_DOMCTL_find_host_compatible_node hypercalls althought they would > > need to be platform_op as Julien suggested). > > > > It looks like the generation of scmi_shmem and scmi should be easy > > enough that could be handled without difficulty in xl/libxl. But if that > > turns out to be too difficult, we could have a small independent > > bash/python script that from the host device tree generates the partial > > device tree with the SCMI nodes. From Xen point of view we are would > > still be using the partial device tree, but the partial device tree > > itself would be generated instead of manually written. As this workflow > > requires a separate tool I think it is a worse option than the one > > above. Still better than nothing though. > > > Hi Stefano, > > Thank you for the detail answer. I went through hypfs and will try to > export host device_tree. Then xl will be able to use hypfs data to > generate arm,scmi-smc node if arm,scmi-smc node was not provided in > DomU partial device-tree. Unfortunately, some changes should be done > to hypfs because it seems not ready to handle nested dynamic dirs. > > I'll see if I can update hypfs without breaking the original > functionality. If not, I will have to create all hypfs dir structure on > start. For now I'm working on making dynamically created hypfs tree > structure, based on host device-tree. That's fantastic, thank you Oleksii! I think it is going to be super userful!
diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig index 186e1db389..02d96c6cfc 100644 --- a/xen/arch/arm/Kconfig +++ b/xen/arch/arm/Kconfig @@ -114,6 +114,8 @@ config SCI support. It allows guests to control system resourcess via one of SCI mediators implemented in XEN. +source "arch/arm/sci/Kconfig" + endmenu menu "ARM errata workaround via the alternative framework" diff --git a/xen/arch/arm/sci/Kconfig b/xen/arch/arm/sci/Kconfig new file mode 100644 index 0000000000..9563067ddc --- /dev/null +++ b/xen/arch/arm/sci/Kconfig @@ -0,0 +1,10 @@ +config SCMI_SMC + bool "Enable SCMI-SMC mediator driver" + default n + depends on SCI + ---help--- + + Enables mediator in XEN to pass SCMI requests from Domains to ATF. + This feature allows drivers from Domains to work with System + Controllers (such as power,resets,clock etc.). SCP is used as transport + for communication. diff --git a/xen/arch/arm/sci/Makefile b/xen/arch/arm/sci/Makefile index 837dc7492b..67f2611872 100644 --- a/xen/arch/arm/sci/Makefile +++ b/xen/arch/arm/sci/Makefile @@ -1 +1,2 @@ obj-y += sci.o +obj-$(CONFIG_SCMI_SMC) += scmi_smc.o diff --git a/xen/arch/arm/sci/scmi_smc.c b/xen/arch/arm/sci/scmi_smc.c new file mode 100644 index 0000000000..2eb01ea82d --- /dev/null +++ b/xen/arch/arm/sci/scmi_smc.c @@ -0,0 +1,795 @@ +/* + * xen/arch/arm/sci/scmi_smc.c + * + * SCMI mediator driver, using SCP as transport. + * + * Oleksii Moisieiev <oleksii_moisieiev@epam.com> + * Copyright (C) 2021, EPAM Systems. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <asm/sci/sci.h> +#include <asm/smccc.h> +#include <asm/io.h> +#include <xen/bitops.h> +#include <xen/config.h> +#include <xen/sched.h> +#include <xen/device_tree.h> +#include <xen/iocap.h> +#include <xen/init.h> +#include <xen/err.h> +#include <xen/lib.h> +#include <xen/list.h> +#include <xen/mm.h> +#include <xen/string.h> +#include <xen/time.h> +#include <xen/vmap.h> + +#define SCMI_BASE_PROTOCOL 0x10 +#define SCMI_BASE_PROTOCOL_ATTIBUTES 0x1 +#define SCMI_BASE_SET_DEVICE_PERMISSIONS 0x9 +#define SCMI_BASE_RESET_AGENT_CONFIGURATION 0xB +#define SCMI_BASE_DISCOVER_AGENT 0x7 + +/* SCMI return codes. See section 4.1.4 of SCMI spec (DEN0056C) */ +#define SCMI_SUCCESS 0 +#define SCMI_NOT_SUPPORTED (-1) +#define SCMI_INVALID_PARAMETERS (-2) +#define SCMI_DENIED (-3) +#define SCMI_NOT_FOUND (-4) +#define SCMI_OUT_OF_RANGE (-5) +#define SCMI_BUSY (-6) +#define SCMI_COMMS_ERROR (-7) +#define SCMI_GENERIC_ERROR (-8) +#define SCMI_HARDWARE_ERROR (-9) +#define SCMI_PROTOCOL_ERROR (-10) + +#define DT_MATCH_SCMI_SMC DT_MATCH_COMPATIBLE("arm,scmi-smc") + +#define SCMI_SMC_ID "arm,smc-id" +#define SCMI_SHARED_MEMORY "linux,scmi_mem" +#define SCMI_SHMEM "shmem" + +#define HYP_CHANNEL 0x0 + +#define HDR_ID GENMASK(7,0) +#define HDR_TYPE GENMASK(9, 8) +#define HDR_PROTO GENMASK(17, 10) + +/* SCMI protocol, refer to section 4.2.2.2 (DEN0056C) */ +#define MSG_N_AGENTS_MASK GENMASK(15, 8) + +#define FIELD_GET(_mask, _reg)\ + ((typeof(_mask))(((_reg) & (_mask)) >> (ffs64(_mask) - 1))) +#define FIELD_PREP(_mask, _val)\ + (((typeof(_mask))(_val) << (ffs64(_mask) - 1)) & (_mask)) + +typedef struct scmi_msg_header { + uint8_t id; + uint8_t type; + uint8_t protocol; +} scmi_msg_header_t; + +typedef struct scmi_perms_tx { + uint32_t agent_id; + uint32_t device_id; + uint32_t flags; +} scmi_perms_tx_t; + +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE BIT(0, UL) +#define SCMI_SHMEM_CHAN_STAT_CHANNEL_ERROR BIT(1, UL) + +#define SCMI_ALLOW_ACCESS BIT(0, UL) + +struct scmi_shared_mem { + uint32_t reserved; + uint32_t channel_status; + uint32_t reserved1[2]; + uint32_t flags; + uint32_t length; + uint32_t msg_header; + uint8_t msg_payload[]; +}; + +struct scmi_channel { + int chan_id; + int agent_id; + uint32_t func_id; + int domain_id; + uint64_t paddr; + struct scmi_shared_mem *shmem; + spinlock_t lock; + struct list_head list; +}; + +struct scmi_data { + struct list_head channel_list; + spinlock_t channel_list_lock; + bool initialized; + u64 shmem_addr, shmem_size; +}; + +static struct scmi_data scmi_data; + +/* + * pack_scmi_header() - packs and returns 32-bit header + * + * @hdr: pointer to header containing all the information on message id, + * protocol id and type id. + * + * Return: 32-bit packed message header to be sent to the platform. + */ +static inline uint32_t pack_scmi_header(scmi_msg_header_t *hdr) +{ + return FIELD_PREP(HDR_ID, hdr->id) | + FIELD_PREP(HDR_TYPE, hdr->type) | + FIELD_PREP(HDR_PROTO, hdr->protocol); +} + +/* + * unpack_scmi_header() - unpacks and records message and protocol id + * + * @msg_hdr: 32-bit packed message header sent from the platform + * @hdr: pointer to header to fetch message and protocol id. + */ +static inline void unpack_scmi_header(uint32_t msg_hdr, scmi_msg_header_t *hdr) +{ + hdr->id = FIELD_GET(HDR_ID, msg_hdr); + hdr->type = FIELD_GET(HDR_TYPE, msg_hdr); + hdr->protocol = FIELD_GET(HDR_PROTO, msg_hdr); +} + +static inline int channel_is_free(struct scmi_channel *chan_info) +{ + return ( chan_info->shmem->channel_status + & SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE ) ? 0 : -EBUSY; +} + +static int send_smc_message(struct scmi_channel *chan_info, + scmi_msg_header_t *hdr, void *data, int len) +{ + struct arm_smccc_res resp; + int ret; + + printk(XENLOG_DEBUG "scmi: status =%d len=%d\n", + chan_info->shmem->channel_status, len); + printk(XENLOG_DEBUG "scmi: header id = %d type = %d, proto = %d\n", + hdr->id, hdr->type, hdr->protocol); + + ret = channel_is_free(chan_info); + if ( IS_ERR_VALUE(ret) ) + return ret; + + chan_info->shmem->channel_status = 0x0; + /* Writing 0x0 right now, but SCMI_SHMEM_FLAG_INTR_ENABLED can be set */ + chan_info->shmem->flags = 0x0; + chan_info->shmem->length = sizeof(chan_info->shmem->msg_header) + len; + chan_info->shmem->msg_header = pack_scmi_header(hdr); + + printk(XENLOG_DEBUG "scmi: Writing to shmem address %p\n", + chan_info->shmem); + if ( len > 0 && data ) + memcpy((void *)(chan_info->shmem->msg_payload), data, len); + + arm_smccc_smc(chan_info->func_id, 0, 0, 0, 0, 0, 0, chan_info->chan_id, + &resp); + + printk(XENLOG_DEBUG "scmi: scmccc_smc response %d\n", (int)(resp.a0)); + + if ( resp.a0 ) + return -EOPNOTSUPP; + + return 0; +} + +static int check_scmi_status(int scmi_status) +{ + if ( scmi_status == SCMI_SUCCESS ) + return 0; + + printk(XENLOG_DEBUG "scmi: Error received: %d\n", scmi_status); + + switch ( scmi_status ) + { + case SCMI_NOT_SUPPORTED: + return -EOPNOTSUPP; + case SCMI_INVALID_PARAMETERS: + return -EINVAL; + case SCMI_DENIED: + return -EACCES; + case SCMI_NOT_FOUND: + return -ENOENT; + case SCMI_OUT_OF_RANGE: + return -ERANGE; + case SCMI_BUSY: + return -EBUSY; + case SCMI_COMMS_ERROR: + return -ENOTCONN; + case SCMI_GENERIC_ERROR: + return -EIO; + case SCMI_HARDWARE_ERROR: + return -ENXIO; + case SCMI_PROTOCOL_ERROR: + return -EBADMSG; + } + + return -EINVAL; +} + +static int get_smc_response(struct scmi_channel *chan_info, + scmi_msg_header_t *hdr, void *data, int len) +{ + int recv_len; + int ret; + + printk(XENLOG_DEBUG "scmi: get smc responce msgid %d\n", hdr->id); + + ret = channel_is_free(chan_info); + if ( IS_ERR_VALUE(ret) ) + return ret; + + recv_len = chan_info->shmem->length - sizeof(chan_info->shmem->msg_header); + + if ( recv_len < 0 ) + { + printk(XENLOG_ERR + "scmi: Wrong size of smc message. Data may be invalid\n"); + return -EINVAL; + } + + if ( recv_len > len ) + { + printk(XENLOG_ERR + "scmi: Not enough buffer for message %d, expecting %d\n", + recv_len, len); + return -EINVAL; + } + + unpack_scmi_header(chan_info->shmem->msg_header, hdr); + + if ( recv_len > 0 ) + { + memcpy(data, chan_info->shmem->msg_payload, recv_len); + } + + return 0; +} + +static int do_smc_xfer(struct scmi_channel *channel, scmi_msg_header_t *hdr, void *tx_data, int tx_size, + void *rx_data, int rx_size) +{ + int ret = 0; + + if ( !hdr ) + return -EINVAL; + + spin_lock(&channel->lock); + + ret = send_smc_message(channel, hdr, tx_data, tx_size); + if ( ret ) + goto clean; + + ret = get_smc_response(channel, hdr, rx_data, rx_size); +clean: + spin_unlock(&channel->lock); + + return ret; +} + +static struct scmi_channel *get_channel_by_id(uint8_t chan_id) +{ + struct scmi_channel *curr; + bool found = false; + + spin_lock(&scmi_data.channel_list_lock); + list_for_each_entry(curr, &scmi_data.channel_list, list) + if ( curr->chan_id == chan_id ) + { + found = true; + break; + } + + spin_unlock(&scmi_data.channel_list_lock); + if ( found ) + return curr; + + return NULL; +} + +static struct scmi_channel *get_channel_by_domain(uint8_t domain_id) +{ + struct scmi_channel *curr; + bool found = false; + + spin_lock(&scmi_data.channel_list_lock); + list_for_each_entry(curr, &scmi_data.channel_list, list) + if ( curr->domain_id == domain_id ) + { + found = true; + break; + } + + spin_unlock(&scmi_data.channel_list_lock); + if ( found ) + return curr; + + return NULL; +} + +static struct scmi_channel *aquire_scmi_channel(int domain_id) +{ + struct scmi_channel *curr; + bool found = false; + + ASSERT(domain_id != DOMID_INVALID && domain_id >= 0); + + spin_lock(&scmi_data.channel_list_lock); + list_for_each_entry(curr, &scmi_data.channel_list, list) + if ( (curr->domain_id == DOMID_INVALID) + && (curr->chan_id != HYP_CHANNEL) ) + { + curr->domain_id = domain_id; + found = true; + break; + } + + spin_unlock(&scmi_data.channel_list_lock); + if ( found ) + return curr; + + return NULL; +} + +static void relinquish_scmi_channel(struct scmi_channel *channel) +{ + spin_lock(&scmi_data.channel_list_lock); + ASSERT(channel != NULL); + channel->domain_id = DOMID_INVALID; + spin_unlock(&scmi_data.channel_list_lock); +} + +static struct scmi_channel *smc_create_channel(uint8_t chan_id, + uint32_t func_id, uint64_t addr) +{ + struct scmi_channel *channel; + mfn_t mfn; + + channel = get_channel_by_id(chan_id); + if ( channel ) + return ERR_PTR(EEXIST); + + channel = xmalloc(struct scmi_channel); + if ( !channel ) + return ERR_PTR(ENOMEM); + + channel->chan_id = chan_id; + channel->func_id = func_id; + channel->domain_id = DOMID_INVALID; + mfn = maddr_to_mfn(addr); + channel->shmem = vmap(&mfn, 1); + if ( !channel->shmem ) + { + xfree(channel); + return ERR_PTR(ENOMEM); + } + + printk(XENLOG_DEBUG "scmi: Got shmem after vmap %p\n", channel->shmem); + channel->paddr = addr; + channel->shmem->channel_status = SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE; + spin_lock_init(&channel->lock); + spin_lock(&scmi_data.channel_list_lock); + list_add(&channel->list, &scmi_data.channel_list); + spin_unlock(&scmi_data.channel_list_lock); + return channel; +} + +static int map_memory_to_domain(struct domain *d, uint64_t addr, uint64_t len) +{ + return iomem_permit_access(d, paddr_to_pfn(addr), + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); +} + +static int unmap_memory_from_domain(struct domain *d, uint64_t addr, + uint64_t len) +{ + return iomem_deny_access(d, paddr_to_pfn(addr), + paddr_to_pfn(PAGE_ALIGN(addr + len -1))); +} + +static int dt_update_domain_range(struct domain *d, uint64_t addr, + uint64_t size) +{ + struct dt_device_node *shmem_node; + __be32 *hw_reg; + const struct dt_property *pp; + uint32_t len; + + shmem_node = dt_find_compatible_node(NULL, NULL, "arm,scmi-shmem"); + + if ( !shmem_node ) + { + printk(XENLOG_ERR "scmi: Unable to find %s node in DT\n", SCMI_SHMEM); + return -EINVAL; + } + + pp = dt_find_property(shmem_node, "reg", &len); + if ( !pp ) + { + printk(XENLOG_ERR "scmi: Unable to find regs entry in shmem node\n"); + return -ENOENT; + } + + hw_reg = pp->value; + dt_set_range(&hw_reg, shmem_node, addr, size); + + return 0; +} + +static void free_channel_list(void) +{ + struct scmi_channel *curr, *_curr; + + spin_lock(&scmi_data.channel_list_lock); + list_for_each_entry_safe (curr, _curr, &scmi_data.channel_list, list) + { + vunmap(curr->shmem); + list_del(&curr->list); + xfree(curr); + } + + spin_unlock(&scmi_data.channel_list_lock); +} + +static __init bool scmi_probe(struct dt_device_node *scmi_node) +{ + struct dt_device_node *shmem_node; + int ret, i; + struct scmi_channel *channel, *agent_channel; + int n_agents; + scmi_msg_header_t hdr; + struct rx_t { + int32_t status; + uint32_t attributes; + } rx; + + uint32_t func_id; + + ASSERT(scmi_node != NULL); + + INIT_LIST_HEAD(&scmi_data.channel_list); + spin_lock_init(&scmi_data.channel_list_lock); + + if ( !dt_property_read_u32(scmi_node, SCMI_SMC_ID, &func_id) ) + { + printk(XENLOG_ERR "scmi: Unable to read smc-id from DT\n"); + return false; + } + + shmem_node = dt_find_node_by_name(NULL, SCMI_SHARED_MEMORY); + if ( IS_ERR_OR_NULL(shmem_node) ) + { + printk(XENLOG_ERR + "scmi: Device tree error, can't parse shmem phandle %ld\n", + PTR_ERR(shmem_node)); + return false; + } + + ret = dt_device_get_address(shmem_node, 0, &scmi_data.shmem_addr, + &scmi_data.shmem_size); + if ( IS_ERR_VALUE(ret) ) + return false; + + channel = smc_create_channel(HYP_CHANNEL, func_id, scmi_data.shmem_addr); + if ( IS_ERR(channel) ) + return false; + + hdr.id = SCMI_BASE_PROTOCOL_ATTIBUTES; + hdr.type = 0; + hdr.protocol = SCMI_BASE_PROTOCOL; + + ret = do_smc_xfer(channel, &hdr, NULL, 0, &rx, sizeof(rx)); + if ( ret ) + goto clean; + + ret = check_scmi_status(rx.status); + if ( ret ) + goto clean; + + n_agents = FIELD_GET(MSG_N_AGENTS_MASK, rx.attributes); + printk(XENLOG_DEBUG "scmi: Got agent count %d\n", n_agents); + + n_agents = (n_agents > scmi_data.shmem_size / PAGE_SIZE) ? + scmi_data.shmem_size / PAGE_SIZE : n_agents; + + for ( i = 1; i < n_agents; i++ ) + { + uint32_t tx_agent_id = 0xFFFFFFFF; + struct { + int32_t status; + uint32_t agent_id; + char name[16]; + } da_rx; + + agent_channel = smc_create_channel(i, func_id, scmi_data.shmem_addr + + i * PAGE_SIZE); + if ( IS_ERR(agent_channel) ) + { + ret = PTR_ERR(agent_channel); + goto clean; + } + + hdr.id = SCMI_BASE_DISCOVER_AGENT; + hdr.type = 0; + hdr.protocol = SCMI_BASE_PROTOCOL; + + ret = do_smc_xfer(agent_channel, &hdr, &tx_agent_id, + sizeof(tx_agent_id), &da_rx, sizeof(da_rx)); + if ( ret ) + goto clean; + + ret = check_scmi_status(da_rx.status); + if ( ret ) + goto clean; + + printk(XENLOG_DEBUG "scmi: status=0x%x id=0x%x name=%s\n", + da_rx.status, da_rx.agent_id, da_rx.name); + + agent_channel->agent_id = da_rx.agent_id; + } + + scmi_data.initialized = true; + return true; + +clean: + free_channel_list(); + return ret == 0; +} + +static int scmi_domain_init(struct domain *d) +{ + struct scmi_channel *channel; + int ret; + + if ( !scmi_data.initialized ) + return 0; + + channel = aquire_scmi_channel(d->domain_id); + if ( IS_ERR_OR_NULL(channel) ) + return -ENOENT; + + printk(XENLOG_INFO "scmi: Aquire SCMI channel id = 0x%x , domain_id = %d" + "paddr = 0x%lx\n", channel->chan_id, channel->domain_id, + channel->paddr); + + if ( is_hardware_domain(d) ) + { + ret = map_memory_to_domain(d, scmi_data.shmem_addr, + scmi_data.shmem_size); + if ( IS_ERR_VALUE(ret) ) + goto error; + + ret = dt_update_domain_range(d, channel->paddr, PAGE_SIZE); + if ( IS_ERR_VALUE(ret) ) + { + int rc = unmap_memory_from_domain(d, scmi_data.shmem_addr, + scmi_data.shmem_size); + if ( rc ) + printk(XENLOG_ERR "Unable to unmap_memory_from_domain\n"); + + goto error; + } + } + + d->arch.sci = channel; + + return 0; +error: + relinquish_scmi_channel(channel); + + return ret; +} + +static int scmi_add_device_by_devid(struct domain *d, uint32_t scmi_devid) +{ + struct scmi_channel *channel, *agent_channel; + scmi_msg_header_t hdr; + scmi_perms_tx_t tx; + struct rx_t { + int32_t status; + uint32_t attributes; + } rx; + int ret; + + if ( !scmi_data.initialized ) + return 0; + + printk(XENLOG_DEBUG "scmi: scmi_devid = %d\n", scmi_devid); + + agent_channel = get_channel_by_domain(d->domain_id); + if ( IS_ERR_OR_NULL(agent_channel) ) + return PTR_ERR(agent_channel); + + channel = get_channel_by_id(HYP_CHANNEL); + if ( IS_ERR_OR_NULL(channel) ) + return PTR_ERR(channel); + + hdr.id = SCMI_BASE_SET_DEVICE_PERMISSIONS; + hdr.type = 0; + hdr.protocol = SCMI_BASE_PROTOCOL; + + tx.agent_id = agent_channel->agent_id; + tx.device_id = scmi_devid; + tx.flags = SCMI_ALLOW_ACCESS; + + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(&rx)); + if ( IS_ERR_VALUE(ret) ) + return ret; + + ret = check_scmi_status(rx.status); + if ( IS_ERR_VALUE(ret) ) + return ret; + + return 0; +} + +static int scmi_add_dt_device(struct domain *d, struct dt_device_node *dev) +{ + uint32_t scmi_devid; + + if ( (!scmi_data.initialized) || (!d->arch.sci) ) + return 0; + + if ( !dt_property_read_u32(dev, "scmi_devid", &scmi_devid) ) + return 0; + + printk(XENLOG_INFO "scmi: dt_node = %s\n", dt_node_full_name(dev)); + + return scmi_add_device_by_devid(d, scmi_devid); +} + +static int scmi_relinquish_resources(struct domain *d) +{ + int ret; + struct scmi_channel *channel, *agent_channel; + scmi_msg_header_t hdr; + struct reset_agent_tx { + uint32_t agent_id; + uint32_t flags; + } tx; + uint32_t rx; + + if ( !d->arch.sci ) + return 0; + + agent_channel = d->arch.sci; + + spin_lock(&agent_channel->lock); + tx.agent_id = agent_channel->agent_id; + spin_unlock(&agent_channel->lock); + + channel = get_channel_by_id(HYP_CHANNEL); + if ( !channel ) + { + printk(XENLOG_ERR + "scmi: Unable to get Hypervisor scmi channel for domain %d\n", + d->domain_id); + return -EINVAL; + } + + hdr.id = SCMI_BASE_RESET_AGENT_CONFIGURATION; + hdr.type = 0; + hdr.protocol = SCMI_BASE_PROTOCOL; + + tx.flags = 0; + + ret = do_smc_xfer(channel, &hdr, &tx, sizeof(tx), &rx, sizeof(rx)); + if ( ret ) + return ret; + + ret = check_scmi_status(rx); + + return ret; +} + +static void scmi_domain_destroy(struct domain *d) +{ + struct scmi_channel *channel; + + if ( !d->arch.sci ) + return; + + channel = d->arch.sci; + spin_lock(&channel->lock); + + relinquish_scmi_channel(channel); + printk(XENLOG_DEBUG "scmi: Free domain %d\n", d->domain_id); + + d->arch.sci = NULL; + + unmap_memory_from_domain(d, channel->paddr, PAGE_SIZE); + spin_unlock(&channel->lock); + return; +} + +static bool scmi_handle_call(struct domain *d, void *args) +{ + bool res = false; + struct scmi_channel *agent_channel; + struct arm_smccc_res resp; + struct cpu_user_regs *regs = args; + + if ( !d->arch.sci ) + return false; + + agent_channel = d->arch.sci; + spin_lock(&agent_channel->lock); + + if ( agent_channel->func_id != regs->x0 ) + { + printk(XENLOG_ERR "scmi: func_id mismatch, exiting\n"); + goto unlock; + } + + arm_smccc_smc(agent_channel->func_id, 0, 0, 0, 0, 0, 0, + agent_channel->chan_id, &resp); + + set_user_reg(regs, 0, resp.a0); + set_user_reg(regs, 1, resp.a1); + set_user_reg(regs, 2, resp.a2); + set_user_reg(regs, 3, resp.a3); + res = true; +unlock: + spin_unlock(&agent_channel->lock); + + return res; +} + +static int scmi_get_channel_paddr(void *scmi_ops, + struct xen_arch_domainconfig *config) +{ + struct scmi_channel *agent_channel = scmi_ops; + + if ( !agent_channel ) + return -EINVAL; + + config->sci_agent_paddr = agent_channel->paddr; + return 0; +} + +static const struct dt_device_match scmi_smc_match[] __initconst = +{ + DT_MATCH_SCMI_SMC, + { /* sentinel */ }, +}; + +static const struct sci_mediator_ops scmi_ops = +{ + .probe = scmi_probe, + .domain_init = scmi_domain_init, + .domain_destroy = scmi_domain_destroy, + .add_dt_device = scmi_add_dt_device, + .relinquish_resources = scmi_relinquish_resources, + .handle_call = scmi_handle_call, + .get_channel_info = scmi_get_channel_paddr +}; + +REGISTER_SCI_MEDIATOR(scmi_smc, "SCMI-SMC", XEN_DOMCTL_CONFIG_SCI_SCMI_SMC, + scmi_smc_match, &scmi_ops); + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h index 9180be5e86..a67237942d 100644 --- a/xen/include/public/arch-arm.h +++ b/xen/include/public/arch-arm.h @@ -315,6 +315,7 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t); #define XEN_DOMCTL_CONFIG_TEE_OPTEE 1 #define XEN_DOMCTL_CONFIG_SCI_NONE 0 +#define XEN_DOMCTL_CONFIG_SCI_SCMI_SMC 1 struct xen_arch_domainconfig { /* IN/OUT */
This is the implementation of SCI interface, called SCMI-SMC driver, which works as the mediator between XEN Domains and Firmware (SCP, ATF etc). This allows devices from the Domains to work with clocks, resets and power-domains without access to CPG. The following features are implemented: - request SCMI channels from ATF and pass channels to Domains; - set device permissions for Domains based on the Domain partial device-tree. Devices with permissions are able to work with clocks, resets and power-domains via SCMI; - redirect scmi messages from Domains to ATF. Signed-off-by: Oleksii Moisieiev <oleksii_moisieiev@epam.com> --- xen/arch/arm/Kconfig | 2 + xen/arch/arm/sci/Kconfig | 10 + xen/arch/arm/sci/Makefile | 1 + xen/arch/arm/sci/scmi_smc.c | 795 ++++++++++++++++++++++++++++++++++ xen/include/public/arch-arm.h | 1 + 5 files changed, 809 insertions(+) create mode 100644 xen/arch/arm/sci/Kconfig create mode 100644 xen/arch/arm/sci/scmi_smc.c