[PULL,1/3] qemu-coroutine-sleep: introduce qemu_co_sleep_wake

Commit Message

Eric Blake Oct. 23, 2019, 2:01 a.m. UTC

From: Vladimir Sementsov-Ogievskiy <[e-mail]>

Introduce a function to gracefully wake a coroutine sleeping in
qemu_co_sleep_ns().

Signed-off-by: Vladimir Sementsov-Ogievskiy <[e-mail]>
Reviewed-by: Kevin Wolf <[e-mail]>
Reviewed-by: Eric Blake <[e-mail]>
Message-Id: <[e-mail]>
Signed-off-by: Eric Blake <[e-mail]>
---
 include/qemu/coroutine.h    | 23 +++++++++++++++--
 util/qemu-coroutine-sleep.c | 51 +++++++++++++++++++++++++++----------
 2 files changed, 59 insertions(+), 15 deletions(-)

Comments

Peter Maydell Nov. 8, 2019, 6:42 p.m. UTC | #1

On Wed, 23 Oct 2019 at 03:04, Eric Blake <[e-mail]> wrote:
>
> From: Vladimir Sementsov-Ogievskiy <[e-mail]>
>
> Introduce a function to gracefully wake a coroutine sleeping in
> qemu_co_sleep_ns().
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <[e-mail]>
> Reviewed-by: Kevin Wolf <[e-mail]>
> Reviewed-by: Eric Blake <[e-mail]>
> Message-Id: <[e-mail]>
> Signed-off-by: Eric Blake <[e-mail]>

Hi; Coverity reports an issue in this patch (CID 1406474):


> ---
>  include/qemu/coroutine.h    | 23 +++++++++++++++--
>  util/qemu-coroutine-sleep.c | 51 +++++++++++++++++++++++++++----------
>  2 files changed, 59 insertions(+), 15 deletions(-)
>
> diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h
> index 9801e7f5a497..8d55663062ad 100644
> --- a/include/qemu/coroutine.h
> +++ b/include/qemu/coroutine.h
> @@ -273,10 +273,29 @@ void qemu_co_rwlock_wrlock(CoRwlock *lock);
>   */
>  void qemu_co_rwlock_unlock(CoRwlock *lock);
>
> +typedef struct QemuCoSleepState QemuCoSleepState;
> +
>  /**
> - * Yield the coroutine for a given duration
> + * Yield the coroutine for a given duration. During this yield, @sleep_state
> + * (if not NULL) is set to an opaque pointer, which may be used for
> + * qemu_co_sleep_wake(). Be careful, the pointer is set back to zero when the
> + * timer fires. Don't save the obtained value to other variables and don't call
> + * qemu_co_sleep_wake from another aio context.
>   */
> -void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns);
> +void coroutine_fn qemu_co_sleep_ns_wakeable(QEMUClockType type, int64_t ns,
> +                                            QemuCoSleepState **sleep_state);
> +static inline void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns)
> +{
> +    qemu_co_sleep_ns_wakeable(type, ns, NULL);
> +}
> +
> +/**
> + * Wake a coroutine if it is sleeping in qemu_co_sleep_ns. The timer will be
> + * deleted. @sleep_state must be the variable whose address was given to
> + * qemu_co_sleep_ns() and should be checked to be non-NULL before calling
> + * qemu_co_sleep_wake().
> + */
> +void qemu_co_sleep_wake(QemuCoSleepState *sleep_state);
>
>  /**
>   * Yield until a file descriptor becomes readable
> diff --git a/util/qemu-coroutine-sleep.c b/util/qemu-coroutine-sleep.c
> index 4bfdd30cbf13..ae91b92b6e78 100644
> --- a/util/qemu-coroutine-sleep.c
> +++ b/util/qemu-coroutine-sleep.c
> @@ -17,31 +17,56 @@
>  #include "qemu/timer.h"
>  #include "block/aio.h"
>
> -static void co_sleep_cb(void *opaque)
> +static const char *qemu_co_sleep_ns__scheduled = "qemu_co_sleep_ns";
> +
> +struct QemuCoSleepState {
> +    Coroutine *co;
> +    QEMUTimer *ts;
> +    QemuCoSleepState **user_state_pointer;
> +};
> +
> +void qemu_co_sleep_wake(QemuCoSleepState *sleep_state)
>  {
> -    Coroutine *co = opaque;
> -
>      /* Write of schedule protected by barrier write in aio_co_schedule */
> -    atomic_set(&co->scheduled, NULL);
> -    aio_co_wake(co);
> +    const char *scheduled = atomic_cmpxchg(&sleep_state->co->scheduled,
> +                                           qemu_co_sleep_ns__scheduled, NULL);
> +
> +    assert(scheduled == qemu_co_sleep_ns__scheduled);
> +    if (sleep_state->user_state_pointer) {
> +        *sleep_state->user_state_pointer = NULL;
> +    }
> +    timer_del(sleep_state->ts);
> +    aio_co_wake(sleep_state->co);
> +}
> +
> +static void co_sleep_cb(void *opaque)
> +{
> +    qemu_co_sleep_wake(opaque);
>  }
>
> -void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns)
> +void coroutine_fn qemu_co_sleep_ns_wakeable(QEMUClockType type, int64_t ns,
> +                                            QemuCoSleepState **sleep_state)
>  {
>      AioContext *ctx = qemu_get_current_aio_context();
> -    QEMUTimer *ts;
> -    Coroutine *co = qemu_coroutine_self();
> +    QemuCoSleepState state = {
> +        .co = qemu_coroutine_self(),
> +        .ts = aio_timer_new(ctx, type, SCALE_NS, co_sleep_cb, &state),
> +        .user_state_pointer = sleep_state,
> +    };

Here 'state' is a variable on the stack...

> -    const char *scheduled = atomic_cmpxchg(&co->scheduled, NULL, __func__);
> +    const char *scheduled = atomic_cmpxchg(&state.co->scheduled, NULL,
> +                                           qemu_co_sleep_ns__scheduled);
>      if (scheduled) {
>          fprintf(stderr,
>                  "%s: Co-routine was already scheduled in '%s'\n",
>                  __func__, scheduled);
>          abort();
>      }
> -    ts = aio_timer_new(ctx, type, SCALE_NS, co_sleep_cb, co);
> -    timer_mod(ts, qemu_clock_get_ns(type) + ns);
> +
> +    if (sleep_state) {
> +        *sleep_state = &state;

...here we save a pointer to it into *sleep_state which was
passed to us by the caller...

> +    }
> +    timer_mod(state.ts, qemu_clock_get_ns(type) + ns);
>      qemu_coroutine_yield();
> -    timer_del(ts);
> -    timer_free(ts);
> +    timer_free(state.ts);

...and here we return from this function, which means 'state'
is no longer in valid memory, but the caller has still been
given a pointer to it.

>  }

Is this just Coverity getting confused by our coroutine code?
(I certainly find it confusing...)

thanks
-- PMM

Eric Blake Nov. 8, 2019, 10:35 p.m. UTC | #2

On 11/8/19 12:42 PM, Peter Maydell wrote:
> On Wed, 23 Oct 2019 at 03:04, Eric Blake <[e-mail]> wrote:
>>
>> From: Vladimir Sementsov-Ogievskiy <[e-mail]>
>>
>> Introduce a function to gracefully wake a coroutine sleeping in
>> qemu_co_sleep_ns().
>>
>> Signed-off-by: Vladimir Sementsov-Ogievskiy <[e-mail]>
>> Reviewed-by: Kevin Wolf <[e-mail]>
>> Reviewed-by: Eric Blake <[e-mail]>
>> Message-Id: <[e-mail]>
>> Signed-off-by: Eric Blake <[e-mail]>
> 
> Hi; Coverity reports an issue in this patch (CID 1406474):
> 

>> 
>> - * Yield the coroutine for a given duration
>> + * Yield the coroutine for a given duration. During this yield, @sleep_state
>> + * (if not NULL) is set to an opaque pointer, which may be used for
>> + * qemu_co_sleep_wake(). Be careful, the pointer is set back to zero when the
>> + * timer fires. Don't save the obtained value to other variables and don't call
>> + * qemu_co_sleep_wake from another aio context.
>>   */
>> -void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns);

Here, we document the rules on what will happen to *sleep_state (in 
particular, since we store a stack local variable into it, the caller 
must not leak it further, and it will be reclaimed back to zero before 
this function finally finishes).

>> -void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns)
>> +void coroutine_fn qemu_co_sleep_ns_wakeable(QEMUClockType type, int64_t ns,
>> +                                            QemuCoSleepState **sleep_state)
>>   {
>>       AioContext *ctx = qemu_get_current_aio_context();
>> -    QEMUTimer *ts;
>> -    Coroutine *co = qemu_coroutine_self();
>> +    QemuCoSleepState state = {
>> +        .co = qemu_coroutine_self(),
>> +        .ts = aio_timer_new(ctx, type, SCALE_NS, co_sleep_cb, &state),
>> +        .user_state_pointer = sleep_state,
>> +    };
> 
> Here 'state' is a variable on the stack...
> 
>> -    const char *scheduled = atomic_cmpxchg(&co->scheduled, NULL, __func__);
>> +    const char *scheduled = atomic_cmpxchg(&state.co->scheduled, NULL,
>> +                                           qemu_co_sleep_ns__scheduled);
>>       if (scheduled) {
>>           fprintf(stderr,
>>                   "%s: Co-routine was already scheduled in '%s'\n",
>>                   __func__, scheduled);
>>           abort();
>>       }
>> -    ts = aio_timer_new(ctx, type, SCALE_NS, co_sleep_cb, co);
>> -    timer_mod(ts, qemu_clock_get_ns(type) + ns);
>> +
>> +    if (sleep_state) {
>> +        *sleep_state = &state;
> 
> ...here we save a pointer to it into *sleep_state which was
> passed to us by the caller...
> 
>> +    }
>> +    timer_mod(state.ts, qemu_clock_get_ns(type) + ns);
>>       qemu_coroutine_yield();

And here is where we yield, which is the only point at which the caller 
will see a non-zero value in *sleep_state in the first place, at which 
point the caller must follow the rules we document.

>> -    timer_del(ts);
>> -    timer_free(ts);
>> +    timer_free(state.ts);
> 
> ...and here we return from this function, which means 'state'
> is no longer in valid memory, but the caller has still been
> given a pointer to it.
> 
>>   }
> 
> Is this just Coverity getting confused by our coroutine code?
> (I certainly find it confusing...)

Yes, Coverity is unable to see that we require that the caller MUST obey 
rules with the use of it's access to sleep_state. However, it might be 
possible after the yield to assert(!sleep_state || *sleep_state == NULL) 
to prove that the caller's use of our temporary leak of a stack variable 
was solely to get this coroutine to resume from yield, and that the 
resumption cleared it, so that the end of the function is not leaking 
anything.

I guess it's worth experimenting to see if such a patch would silence 
Coverity without breaking the code...

Patch

diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h
index 9801e7f5a497..8d55663062ad 100644
--- a/include/qemu/coroutine.h
+++ b/include/qemu/coroutine.h
@@ -273,10 +273,29 @@  void qemu_co_rwlock_wrlock(CoRwlock *lock);
  */
 void qemu_co_rwlock_unlock(CoRwlock *lock);

+typedef struct QemuCoSleepState QemuCoSleepState;
+
 /**
- * Yield the coroutine for a given duration
+ * Yield the coroutine for a given duration. During this yield, @sleep_state
+ * (if not NULL) is set to an opaque pointer, which may be used for
+ * qemu_co_sleep_wake(). Be careful, the pointer is set back to zero when the
+ * timer fires. Don't save the obtained value to other variables and don't call
+ * qemu_co_sleep_wake from another aio context.
  */
-void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns);
+void coroutine_fn qemu_co_sleep_ns_wakeable(QEMUClockType type, int64_t ns,
+                                            QemuCoSleepState **sleep_state);
+static inline void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns)
+{
+    qemu_co_sleep_ns_wakeable(type, ns, NULL);
+}
+
+/**
+ * Wake a coroutine if it is sleeping in qemu_co_sleep_ns. The timer will be
+ * deleted. @sleep_state must be the variable whose address was given to
+ * qemu_co_sleep_ns() and should be checked to be non-NULL before calling
+ * qemu_co_sleep_wake().
+ */
+void qemu_co_sleep_wake(QemuCoSleepState *sleep_state);

 /**
  * Yield until a file descriptor becomes readable
diff --git a/util/qemu-coroutine-sleep.c b/util/qemu-coroutine-sleep.c
index 4bfdd30cbf13..ae91b92b6e78 100644
--- a/util/qemu-coroutine-sleep.c
+++ b/util/qemu-coroutine-sleep.c
@@ -17,31 +17,56 @@ 
 #include "qemu/timer.h"
 #include "block/aio.h"

-static void co_sleep_cb(void *opaque)
+static const char *qemu_co_sleep_ns__scheduled = "qemu_co_sleep_ns";
+
+struct QemuCoSleepState {
+    Coroutine *co;
+    QEMUTimer *ts;
+    QemuCoSleepState **user_state_pointer;
+};
+
+void qemu_co_sleep_wake(QemuCoSleepState *sleep_state)
 {
-    Coroutine *co = opaque;
-
     /* Write of schedule protected by barrier write in aio_co_schedule */
-    atomic_set(&co->scheduled, NULL);
-    aio_co_wake(co);
+    const char *scheduled = atomic_cmpxchg(&sleep_state->co->scheduled,
+                                           qemu_co_sleep_ns__scheduled, NULL);
+
+    assert(scheduled == qemu_co_sleep_ns__scheduled);
+    if (sleep_state->user_state_pointer) {
+        *sleep_state->user_state_pointer = NULL;
+    }
+    timer_del(sleep_state->ts);
+    aio_co_wake(sleep_state->co);
+}
+
+static void co_sleep_cb(void *opaque)
+{
+    qemu_co_sleep_wake(opaque);
 }

-void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns)
+void coroutine_fn qemu_co_sleep_ns_wakeable(QEMUClockType type, int64_t ns,
+                                            QemuCoSleepState **sleep_state)
 {
     AioContext *ctx = qemu_get_current_aio_context();
-    QEMUTimer *ts;
-    Coroutine *co = qemu_coroutine_self();
+    QemuCoSleepState state = {
+        .co = qemu_coroutine_self(),
+        .ts = aio_timer_new(ctx, type, SCALE_NS, co_sleep_cb, &state),
+        .user_state_pointer = sleep_state,
+    };

-    const char *scheduled = atomic_cmpxchg(&co->scheduled, NULL, __func__);
+    const char *scheduled = atomic_cmpxchg(&state.co->scheduled, NULL,
+                                           qemu_co_sleep_ns__scheduled);
     if (scheduled) {
         fprintf(stderr,
                 "%s: Co-routine was already scheduled in '%s'\n",
                 __func__, scheduled);
         abort();
     }
-    ts = aio_timer_new(ctx, type, SCALE_NS, co_sleep_cb, co);
-    timer_mod(ts, qemu_clock_get_ns(type) + ns);
+
+    if (sleep_state) {
+        *sleep_state = &state;
+    }
+    timer_mod(state.ts, qemu_clock_get_ns(type) + ns);
     qemu_coroutine_yield();
-    timer_del(ts);
-    timer_free(ts);
+    timer_free(state.ts);
 }