[RFC,03/37] s390/protvirt: add ultravisor initialization
diff mbox series

Message ID 20191024114059.102802-4-frankja@linux.ibm.com
State New
Headers show
Series
  • KVM: s390: Add support for protected VMs
Related show

Commit Message

Janosch Frank Oct. 24, 2019, 11:40 a.m. UTC
From: Vasily Gorbik <gor@linux.ibm.com>

Before being able to host protected virtual machines, donate some of
the memory to the ultravisor. Besides that the ultravisor might impose
addressing limitations for memory used to back protected VM storage. Treat
that limit as protected virtualization host's virtual memory limit.

Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
---
 arch/s390/include/asm/uv.h | 16 ++++++++++++
 arch/s390/kernel/setup.c   |  3 +++
 arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+)

Comments

David Hildenbrand Oct. 25, 2019, 9:21 a.m. UTC | #1
On 24.10.19 13:40, Janosch Frank wrote:
> From: Vasily Gorbik <gor@linux.ibm.com>
> 
> Before being able to host protected virtual machines, donate some of
> the memory to the ultravisor. Besides that the ultravisor might impose
> addressing limitations for memory used to back protected VM storage. Treat
> that limit as protected virtualization host's virtual memory limit.
> 
> Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
> ---
>   arch/s390/include/asm/uv.h | 16 ++++++++++++
>   arch/s390/kernel/setup.c   |  3 +++
>   arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
>   3 files changed, 72 insertions(+)
> 
> diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h
> index 6db1bc495e67..82a46fb913e7 100644
> --- a/arch/s390/include/asm/uv.h
> +++ b/arch/s390/include/asm/uv.h
> @@ -23,12 +23,14 @@
>   #define UVC_RC_NO_RESUME	0x0007
>   
>   #define UVC_CMD_QUI			0x0001
> +#define UVC_CMD_INIT_UV			0x000f
>   #define UVC_CMD_SET_SHARED_ACCESS	0x1000
>   #define UVC_CMD_REMOVE_SHARED_ACCESS	0x1001
>   
>   /* Bits in installed uv calls */
>   enum uv_cmds_inst {
>   	BIT_UVC_CMD_QUI = 0,
> +	BIT_UVC_CMD_INIT_UV = 1,
>   	BIT_UVC_CMD_SET_SHARED_ACCESS = 8,
>   	BIT_UVC_CMD_REMOVE_SHARED_ACCESS = 9,
>   };
> @@ -59,6 +61,15 @@ struct uv_cb_qui {
>   	u64 reserved98;
>   } __packed __aligned(8);
>   
> +struct uv_cb_init {
> +	struct uv_cb_header header;
> +	u64 reserved08[2];
> +	u64 stor_origin;
> +	u64 stor_len;
> +	u64 reserved28[4];
> +
> +} __packed __aligned(8);
> +
>   struct uv_cb_share {
>   	struct uv_cb_header header;
>   	u64 reserved08[3];
> @@ -158,8 +169,13 @@ static inline int is_prot_virt_host(void)
>   {
>   	return prot_virt_host;
>   }
> +
> +void setup_uv(void);
> +void adjust_to_uv_max(unsigned long *vmax);
>   #else
>   #define is_prot_virt_host() 0
> +static inline void setup_uv(void) {}
> +static inline void adjust_to_uv_max(unsigned long *vmax) {}
>   #endif
>   
>   #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) ||                          \
> diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
> index f36370f8af38..d29d83c0b8df 100644
> --- a/arch/s390/kernel/setup.c
> +++ b/arch/s390/kernel/setup.c
> @@ -567,6 +567,8 @@ static void __init setup_memory_end(void)
>   			vmax = _REGION1_SIZE; /* 4-level kernel page table */
>   	}
>   
> +	adjust_to_uv_max(&vmax);

I do wonder what would happen if vmax < max_physmem_end. Not sure if 
that is relevant at all.

> +
>   	/* module area is at the end of the kernel address space. */
>   	MODULES_END = vmax;
>   	MODULES_VADDR = MODULES_END - MODULES_LEN;
> @@ -1147,6 +1149,7 @@ void __init setup_arch(char **cmdline_p)
>   	 */
>   	memblock_trim_memory(1UL << (MAX_ORDER - 1 + PAGE_SHIFT));
>   
> +	setup_uv();
>   	setup_memory_end();
>   	setup_memory();
>   	dma_contiguous_reserve(memory_end);
> diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
> index 35ce89695509..f7778493e829 100644
> --- a/arch/s390/kernel/uv.c
> +++ b/arch/s390/kernel/uv.c
> @@ -45,4 +45,57 @@ static int __init prot_virt_setup(char *val)
>   	return rc;
>   }
>   early_param("prot_virt", prot_virt_setup);
> +
> +static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
> +{
> +	struct uv_cb_init uvcb = {
> +		.header.cmd = UVC_CMD_INIT_UV,
> +		.header.len = sizeof(uvcb),
> +		.stor_origin = stor_base,
> +		.stor_len = stor_len,
> +	};
> +	int cc;
> +
> +	cc = uv_call(0, (uint64_t)&uvcb);
> +	if (cc || uvcb.header.rc != UVC_RC_EXECUTED) {
> +		pr_err("Ultravisor init failed with cc: %d rc: 0x%hx\n", cc,
> +		       uvcb.header.rc);
> +		return -1;
> +	}
> +	return 0;
> +}
> +
> +void __init setup_uv(void)
> +{
> +	unsigned long uv_stor_base;
> +
> +	if (!prot_virt_host)
> +		return;
> +
> +	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
> +		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
> +		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
> +	if (!uv_stor_base) {
> +		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
> +			uv_info.uv_base_stor_len);
> +		goto fail;
> +	}

If I'm not wrong, we could setup/reserve a CMA area here and defer the 
actual allocation. Then, any MOVABLE data can end up on this CMA area 
until needed.

But I am neither an expert on CMA nor on UV, so most probably what I say 
is wrong ;)

> +
> +	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
> +		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
> +		goto fail;
> +	}
> +
> +	pr_info("Reserving %luMB as ultravisor base storage\n",
> +		uv_info.uv_base_stor_len >> 20);
> +	return;
> +fail:
> +	prot_virt_host = 0;
> +}
> +
> +void adjust_to_uv_max(unsigned long *vmax)
> +{
> +	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
> +		*vmax = uv_info.max_sec_stor_addr;
> +}
>   #endif
> 

Looks good to me from what I can tell.
Vasily Gorbik Oct. 28, 2019, 3:48 p.m. UTC | #2
On Fri, Oct 25, 2019 at 11:21:05AM +0200, David Hildenbrand wrote:
> On 24.10.19 13:40, Janosch Frank wrote:
> > From: Vasily Gorbik <gor@linux.ibm.com>
> > 
> > Before being able to host protected virtual machines, donate some of
> > the memory to the ultravisor. Besides that the ultravisor might impose
> > addressing limitations for memory used to back protected VM storage. Treat
> > that limit as protected virtualization host's virtual memory limit.
> > 
> > Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
> > ---
> >   arch/s390/include/asm/uv.h | 16 ++++++++++++
> >   arch/s390/kernel/setup.c   |  3 +++
> >   arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
> >   3 files changed, 72 insertions(+)
> > 
> > --- a/arch/s390/kernel/setup.c
> > +++ b/arch/s390/kernel/setup.c
> > @@ -567,6 +567,8 @@ static void __init setup_memory_end(void)
> >   			vmax = _REGION1_SIZE; /* 4-level kernel page table */
> >   	}
> > +	adjust_to_uv_max(&vmax);
> 
> I do wonder what would happen if vmax < max_physmem_end. Not sure if that is
> relevant at all.

Then identity mapping would be shorter then actual physical memory available
and everything above would be lost. But in reality "max_sec_stor_addr"
is big enough to not worry about it in the foreseeable future at all.

> > +void __init setup_uv(void)
> > +{
> > +	unsigned long uv_stor_base;
> > +
> > +	if (!prot_virt_host)
> > +		return;
> > +
> > +	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
> > +		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
> > +		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
> > +	if (!uv_stor_base) {
> > +		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
> > +			uv_info.uv_base_stor_len);
> > +		goto fail;
> > +	}
> 
> If I'm not wrong, we could setup/reserve a CMA area here and defer the
> actual allocation. Then, any MOVABLE data can end up on this CMA area until
> needed.
> 
> But I am neither an expert on CMA nor on UV, so most probably what I say is
> wrong ;)

From pure memory management this sounds like a good idea. And I tried
it and cma_declare_contiguous fulfills our needs, just had to export
cma_alloc/cma_release symbols. Nevertheless, delaying ultravisor init means we
would be potentially left with vmax == max_sec_stor_addr even if we wouldn't
be able to run protected VMs after all (currently setup_uv() is called
before kernel address space layout setup). Another much more fundamental
reason is that ultravisor init has to be called with a single cpu running,
which means it's easy to do before bringing other cpus up and we currently
don't have api to stop cpus at a later point (stop_machine won't cut it).

> > +
> > +	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
> > +		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
> > +		goto fail;
> > +	}
> > +
> > +	pr_info("Reserving %luMB as ultravisor base storage\n",
> > +		uv_info.uv_base_stor_len >> 20);
> > +	return;
> > +fail:
> > +	prot_virt_host = 0;
> > +}
> > +
> > +void adjust_to_uv_max(unsigned long *vmax)
> > +{
> > +	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
> > +		*vmax = uv_info.max_sec_stor_addr;
> > +}
> >   #endif
> > 
> 
> Looks good to me from what I can tell.
> 
> -- 
> 
> Thanks,
> 
> David / dhildenb
>
David Hildenbrand Oct. 28, 2019, 3:54 p.m. UTC | #3
On 28.10.19 16:48, Vasily Gorbik wrote:
> On Fri, Oct 25, 2019 at 11:21:05AM +0200, David Hildenbrand wrote:
>> On 24.10.19 13:40, Janosch Frank wrote:
>>> From: Vasily Gorbik <gor@linux.ibm.com>
>>>
>>> Before being able to host protected virtual machines, donate some of
>>> the memory to the ultravisor. Besides that the ultravisor might impose
>>> addressing limitations for memory used to back protected VM storage. Treat
>>> that limit as protected virtualization host's virtual memory limit.
>>>
>>> Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
>>> ---
>>>    arch/s390/include/asm/uv.h | 16 ++++++++++++
>>>    arch/s390/kernel/setup.c   |  3 +++
>>>    arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
>>>    3 files changed, 72 insertions(+)
>>>
>>> --- a/arch/s390/kernel/setup.c
>>> +++ b/arch/s390/kernel/setup.c
>>> @@ -567,6 +567,8 @@ static void __init setup_memory_end(void)
>>>    			vmax = _REGION1_SIZE; /* 4-level kernel page table */
>>>    	}
>>> +	adjust_to_uv_max(&vmax);
>>
>> I do wonder what would happen if vmax < max_physmem_end. Not sure if that is
>> relevant at all.
> 
> Then identity mapping would be shorter then actual physical memory available
> and everything above would be lost. But in reality "max_sec_stor_addr"
> is big enough to not worry about it in the foreseeable future at all.
> 
>>> +void __init setup_uv(void)
>>> +{
>>> +	unsigned long uv_stor_base;
>>> +
>>> +	if (!prot_virt_host)
>>> +		return;
>>> +
>>> +	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
>>> +		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
>>> +		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
>>> +	if (!uv_stor_base) {
>>> +		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
>>> +			uv_info.uv_base_stor_len);
>>> +		goto fail;
>>> +	}
>>
>> If I'm not wrong, we could setup/reserve a CMA area here and defer the
>> actual allocation. Then, any MOVABLE data can end up on this CMA area until
>> needed.
>>
>> But I am neither an expert on CMA nor on UV, so most probably what I say is
>> wrong ;)
> 
>  From pure memory management this sounds like a good idea. And I tried
> it and cma_declare_contiguous fulfills our needs, just had to export
> cma_alloc/cma_release symbols. Nevertheless, delaying ultravisor init means we
> would be potentially left with vmax == max_sec_stor_addr even if we wouldn't
> be able to run protected VMs after all (currently setup_uv() is called
> before kernel address space layout setup). Another much more fundamental
> reason is that ultravisor init has to be called with a single cpu running,
> which means it's easy to do before bringing other cpus up and we currently
> don't have api to stop cpus at a later point (stop_machine won't cut it).

Interesting point, I guess. One could hack around that. Emphasis on 
*hack* :) In stop_machine() you caught all CPUs. You could just 
temporarily SIGP STOP all running ones, issue the UV init call, and SIGP 
START them again. Not sure how that works with SMP, though ...

But yeah, this is stuff for the future, just an idea from my side :)
Christian Borntraeger Nov. 1, 2019, 10:07 a.m. UTC | #4
On 24.10.19 13:40, Janosch Frank wrote:
> From: Vasily Gorbik <gor@linux.ibm.com>
> 
> Before being able to host protected virtual machines, donate some of
> the memory to the ultravisor. Besides that the ultravisor might impose
> addressing limitations for memory used to back protected VM storage. Treat
> that limit as protected virtualization host's virtual memory limit.
> 
> Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>


> ---
>  arch/s390/include/asm/uv.h | 16 ++++++++++++
>  arch/s390/kernel/setup.c   |  3 +++
>  arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
>  3 files changed, 72 insertions(+)
> 
> diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h
> index 6db1bc495e67..82a46fb913e7 100644
> --- a/arch/s390/include/asm/uv.h
> +++ b/arch/s390/include/asm/uv.h
> @@ -23,12 +23,14 @@
>  #define UVC_RC_NO_RESUME	0x0007
>  
>  #define UVC_CMD_QUI			0x0001
> +#define UVC_CMD_INIT_UV			0x000f
>  #define UVC_CMD_SET_SHARED_ACCESS	0x1000
>  #define UVC_CMD_REMOVE_SHARED_ACCESS	0x1001
>  
>  /* Bits in installed uv calls */
>  enum uv_cmds_inst {
>  	BIT_UVC_CMD_QUI = 0,
> +	BIT_UVC_CMD_INIT_UV = 1,
>  	BIT_UVC_CMD_SET_SHARED_ACCESS = 8,
>  	BIT_UVC_CMD_REMOVE_SHARED_ACCESS = 9,
>  };
> @@ -59,6 +61,15 @@ struct uv_cb_qui {
>  	u64 reserved98;
>  } __packed __aligned(8);
>  
> +struct uv_cb_init {
> +	struct uv_cb_header header;
> +	u64 reserved08[2];
> +	u64 stor_origin;
> +	u64 stor_len;
> +	u64 reserved28[4];
> +
> +} __packed __aligned(8);
> +
>  struct uv_cb_share {
>  	struct uv_cb_header header;
>  	u64 reserved08[3];
> @@ -158,8 +169,13 @@ static inline int is_prot_virt_host(void)
>  {
>  	return prot_virt_host;
>  }
> +
> +void setup_uv(void);
> +void adjust_to_uv_max(unsigned long *vmax);
>  #else
>  #define is_prot_virt_host() 0
> +static inline void setup_uv(void) {}
> +static inline void adjust_to_uv_max(unsigned long *vmax) {}
>  #endif
>  
>  #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) ||                          \
> diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
> index f36370f8af38..d29d83c0b8df 100644
> --- a/arch/s390/kernel/setup.c
> +++ b/arch/s390/kernel/setup.c
> @@ -567,6 +567,8 @@ static void __init setup_memory_end(void)
>  			vmax = _REGION1_SIZE; /* 4-level kernel page table */
>  	}
>  
> +	adjust_to_uv_max(&vmax);
> +
>  	/* module area is at the end of the kernel address space. */
>  	MODULES_END = vmax;
>  	MODULES_VADDR = MODULES_END - MODULES_LEN;
> @@ -1147,6 +1149,7 @@ void __init setup_arch(char **cmdline_p)
>  	 */
>  	memblock_trim_memory(1UL << (MAX_ORDER - 1 + PAGE_SHIFT));
>  
> +	setup_uv();
>  	setup_memory_end();
>  	setup_memory();
>  	dma_contiguous_reserve(memory_end);
> diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
> index 35ce89695509..f7778493e829 100644
> --- a/arch/s390/kernel/uv.c
> +++ b/arch/s390/kernel/uv.c
> @@ -45,4 +45,57 @@ static int __init prot_virt_setup(char *val)
>  	return rc;
>  }
>  early_param("prot_virt", prot_virt_setup);
> +
> +static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
> +{
> +	struct uv_cb_init uvcb = {
> +		.header.cmd = UVC_CMD_INIT_UV,
> +		.header.len = sizeof(uvcb),
> +		.stor_origin = stor_base,
> +		.stor_len = stor_len,
> +	};
> +	int cc;
> +
> +	cc = uv_call(0, (uint64_t)&uvcb);
> +	if (cc || uvcb.header.rc != UVC_RC_EXECUTED) {
> +		pr_err("Ultravisor init failed with cc: %d rc: 0x%hx\n", cc,
> +		       uvcb.header.rc);
> +		return -1;
> +	}
> +	return 0;
> +}
> +
> +void __init setup_uv(void)
> +{
> +	unsigned long uv_stor_base;
> +
> +	if (!prot_virt_host)
> +		return;
> +
> +	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
> +		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
> +		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
> +	if (!uv_stor_base) {
> +		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
> +			uv_info.uv_base_stor_len);
> +		goto fail;
> +	}
> +
> +	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
> +		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
> +		goto fail;
> +	}
> +
> +	pr_info("Reserving %luMB as ultravisor base storage\n",
> +		uv_info.uv_base_stor_len >> 20);
> +	return;
> +fail:
> +	prot_virt_host = 0;
> +}
> +
> +void adjust_to_uv_max(unsigned long *vmax)
> +{
> +	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
> +		*vmax = uv_info.max_sec_stor_addr;
> +}
>  #endif
>
Cornelia Huck Nov. 7, 2019, 3:28 p.m. UTC | #5
On Thu, 24 Oct 2019 07:40:25 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> From: Vasily Gorbik <gor@linux.ibm.com>
> 
> Before being able to host protected virtual machines, donate some of
> the memory to the ultravisor. Besides that the ultravisor might impose
> addressing limitations for memory used to back protected VM storage. Treat
> that limit as protected virtualization host's virtual memory limit.
> 
> Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
> ---
>  arch/s390/include/asm/uv.h | 16 ++++++++++++
>  arch/s390/kernel/setup.c   |  3 +++
>  arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
>  3 files changed, 72 insertions(+)

(...)

> diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
> index 35ce89695509..f7778493e829 100644
> --- a/arch/s390/kernel/uv.c
> +++ b/arch/s390/kernel/uv.c
> @@ -45,4 +45,57 @@ static int __init prot_virt_setup(char *val)
>  	return rc;
>  }
>  early_param("prot_virt", prot_virt_setup);
> +
> +static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
> +{
> +	struct uv_cb_init uvcb = {
> +		.header.cmd = UVC_CMD_INIT_UV,
> +		.header.len = sizeof(uvcb),
> +		.stor_origin = stor_base,
> +		.stor_len = stor_len,
> +	};
> +	int cc;
> +
> +	cc = uv_call(0, (uint64_t)&uvcb);
> +	if (cc || uvcb.header.rc != UVC_RC_EXECUTED) {
> +		pr_err("Ultravisor init failed with cc: %d rc: 0x%hx\n", cc,
> +		       uvcb.header.rc);
> +		return -1;

Is there any reasonable case where that call might fail if we have the
facility installed? Bad stor_base, maybe?

> +	}
> +	return 0;
> +}
> +
> +void __init setup_uv(void)
> +{
> +	unsigned long uv_stor_base;
> +
> +	if (!prot_virt_host)
> +		return;
> +
> +	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
> +		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
> +		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
> +	if (!uv_stor_base) {
> +		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
> +			uv_info.uv_base_stor_len);
> +		goto fail;
> +	}
> +
> +	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
> +		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
> +		goto fail;
> +	}
> +
> +	pr_info("Reserving %luMB as ultravisor base storage\n",
> +		uv_info.uv_base_stor_len >> 20);
> +	return;
> +fail:
> +	prot_virt_host = 0;

So, what happens if the user requested protected virtualization and any
of the above failed? We turn off host support, so any attempt to start
a protected virtualization guest on that host will fail (hopefully with
a meaningful error), I guess.

Is there any use case where we'd want to make failure to set this up
fatal?

> +}
> +
> +void adjust_to_uv_max(unsigned long *vmax)
> +{
> +	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
> +		*vmax = uv_info.max_sec_stor_addr;
> +}
>  #endif
Janosch Frank Nov. 7, 2019, 3:32 p.m. UTC | #6
On 11/7/19 4:28 PM, Cornelia Huck wrote:
> On Thu, 24 Oct 2019 07:40:25 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> From: Vasily Gorbik <gor@linux.ibm.com>
>>
>> Before being able to host protected virtual machines, donate some of
>> the memory to the ultravisor. Besides that the ultravisor might impose
>> addressing limitations for memory used to back protected VM storage. Treat
>> that limit as protected virtualization host's virtual memory limit.
>>
>> Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
>> ---
>>  arch/s390/include/asm/uv.h | 16 ++++++++++++
>>  arch/s390/kernel/setup.c   |  3 +++
>>  arch/s390/kernel/uv.c      | 53 ++++++++++++++++++++++++++++++++++++++
>>  3 files changed, 72 insertions(+)
> 
> (...)
> 
>> diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
>> index 35ce89695509..f7778493e829 100644
>> --- a/arch/s390/kernel/uv.c
>> +++ b/arch/s390/kernel/uv.c
>> @@ -45,4 +45,57 @@ static int __init prot_virt_setup(char *val)
>>  	return rc;
>>  }
>>  early_param("prot_virt", prot_virt_setup);
>> +
>> +static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
>> +{
>> +	struct uv_cb_init uvcb = {
>> +		.header.cmd = UVC_CMD_INIT_UV,
>> +		.header.len = sizeof(uvcb),
>> +		.stor_origin = stor_base,
>> +		.stor_len = stor_len,
>> +	};
>> +	int cc;
>> +
>> +	cc = uv_call(0, (uint64_t)&uvcb);
>> +	if (cc || uvcb.header.rc != UVC_RC_EXECUTED) {
>> +		pr_err("Ultravisor init failed with cc: %d rc: 0x%hx\n", cc,
>> +		       uvcb.header.rc);
>> +		return -1;
> 
> Is there any reasonable case where that call might fail if we have the
> facility installed? Bad stor_base, maybe?

Yes, wrong storage locations, length, etc...
Also if we are running with more than one CPU or the Ultravisor
encountered some internal error.

> 
>> +	}
>> +	return 0;
>> +}
>> +
>> +void __init setup_uv(void)
>> +{
>> +	unsigned long uv_stor_base;
>> +
>> +	if (!prot_virt_host)
>> +		return;
>> +
>> +	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
>> +		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
>> +		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
>> +	if (!uv_stor_base) {
>> +		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
>> +			uv_info.uv_base_stor_len);
>> +		goto fail;
>> +	}
>> +
>> +	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
>> +		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
>> +		goto fail;
>> +	}
>> +
>> +	pr_info("Reserving %luMB as ultravisor base storage\n",
>> +		uv_info.uv_base_stor_len >> 20);
>> +	return;
>> +fail:
>> +	prot_virt_host = 0;
> 
> So, what happens if the user requested protected virtualization and any
> of the above failed? We turn off host support, so any attempt to start
> a protected virtualization guest on that host will fail (hopefully with
> a meaningful error), I guess.
> 

STFLE 161, and the associated diag308 subcodes 8-10 will not be
available to any VM. So yes, the stuv that starts a protected guest will
print a message.

> Is there any use case where we'd want to make failure to set this up
> fatal?

Not really.

> 
>> +}
>> +
>> +void adjust_to_uv_max(unsigned long *vmax)
>> +{
>> +	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
>> +		*vmax = uv_info.max_sec_stor_addr;
>> +}
>>  #endif
>

Patch
diff mbox series

diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h
index 6db1bc495e67..82a46fb913e7 100644
--- a/arch/s390/include/asm/uv.h
+++ b/arch/s390/include/asm/uv.h
@@ -23,12 +23,14 @@ 
 #define UVC_RC_NO_RESUME	0x0007
 
 #define UVC_CMD_QUI			0x0001
+#define UVC_CMD_INIT_UV			0x000f
 #define UVC_CMD_SET_SHARED_ACCESS	0x1000
 #define UVC_CMD_REMOVE_SHARED_ACCESS	0x1001
 
 /* Bits in installed uv calls */
 enum uv_cmds_inst {
 	BIT_UVC_CMD_QUI = 0,
+	BIT_UVC_CMD_INIT_UV = 1,
 	BIT_UVC_CMD_SET_SHARED_ACCESS = 8,
 	BIT_UVC_CMD_REMOVE_SHARED_ACCESS = 9,
 };
@@ -59,6 +61,15 @@  struct uv_cb_qui {
 	u64 reserved98;
 } __packed __aligned(8);
 
+struct uv_cb_init {
+	struct uv_cb_header header;
+	u64 reserved08[2];
+	u64 stor_origin;
+	u64 stor_len;
+	u64 reserved28[4];
+
+} __packed __aligned(8);
+
 struct uv_cb_share {
 	struct uv_cb_header header;
 	u64 reserved08[3];
@@ -158,8 +169,13 @@  static inline int is_prot_virt_host(void)
 {
 	return prot_virt_host;
 }
+
+void setup_uv(void);
+void adjust_to_uv_max(unsigned long *vmax);
 #else
 #define is_prot_virt_host() 0
+static inline void setup_uv(void) {}
+static inline void adjust_to_uv_max(unsigned long *vmax) {}
 #endif
 
 #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) ||                          \
diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index f36370f8af38..d29d83c0b8df 100644
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -567,6 +567,8 @@  static void __init setup_memory_end(void)
 			vmax = _REGION1_SIZE; /* 4-level kernel page table */
 	}
 
+	adjust_to_uv_max(&vmax);
+
 	/* module area is at the end of the kernel address space. */
 	MODULES_END = vmax;
 	MODULES_VADDR = MODULES_END - MODULES_LEN;
@@ -1147,6 +1149,7 @@  void __init setup_arch(char **cmdline_p)
 	 */
 	memblock_trim_memory(1UL << (MAX_ORDER - 1 + PAGE_SHIFT));
 
+	setup_uv();
 	setup_memory_end();
 	setup_memory();
 	dma_contiguous_reserve(memory_end);
diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
index 35ce89695509..f7778493e829 100644
--- a/arch/s390/kernel/uv.c
+++ b/arch/s390/kernel/uv.c
@@ -45,4 +45,57 @@  static int __init prot_virt_setup(char *val)
 	return rc;
 }
 early_param("prot_virt", prot_virt_setup);
+
+static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
+{
+	struct uv_cb_init uvcb = {
+		.header.cmd = UVC_CMD_INIT_UV,
+		.header.len = sizeof(uvcb),
+		.stor_origin = stor_base,
+		.stor_len = stor_len,
+	};
+	int cc;
+
+	cc = uv_call(0, (uint64_t)&uvcb);
+	if (cc || uvcb.header.rc != UVC_RC_EXECUTED) {
+		pr_err("Ultravisor init failed with cc: %d rc: 0x%hx\n", cc,
+		       uvcb.header.rc);
+		return -1;
+	}
+	return 0;
+}
+
+void __init setup_uv(void)
+{
+	unsigned long uv_stor_base;
+
+	if (!prot_virt_host)
+		return;
+
+	uv_stor_base = (unsigned long)memblock_alloc_try_nid(
+		uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
+		MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
+	if (!uv_stor_base) {
+		pr_info("Failed to reserve %lu bytes for ultravisor base storage\n",
+			uv_info.uv_base_stor_len);
+		goto fail;
+	}
+
+	if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
+		memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
+		goto fail;
+	}
+
+	pr_info("Reserving %luMB as ultravisor base storage\n",
+		uv_info.uv_base_stor_len >> 20);
+	return;
+fail:
+	prot_virt_host = 0;
+}
+
+void adjust_to_uv_max(unsigned long *vmax)
+{
+	if (prot_virt_host && *vmax > uv_info.max_sec_stor_addr)
+		*vmax = uv_info.max_sec_stor_addr;
+}
 #endif