Message ID | 20190712081744.87097-4-brendanhiggins@google.com (mailing list archive) |
---|---|
State | Superseded |
Headers | show |
Series | kunit: introduce KUnit, the Linux kernel unit testing framework | expand |
Quoting Brendan Higgins (2019-07-12 01:17:29) > diff --git a/include/kunit/string-stream.h b/include/kunit/string-stream.h > new file mode 100644 > index 0000000000000..0552a05781afe > --- /dev/null > +++ b/include/kunit/string-stream.h > @@ -0,0 +1,49 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +/* > + * C++ stream style string builder used in KUnit for building messages. > + * > + * Copyright (C) 2019, Google LLC. > + * Author: Brendan Higgins <brendanhiggins@google.com> > + */ > + > +#ifndef _KUNIT_STRING_STREAM_H > +#define _KUNIT_STRING_STREAM_H > + > +#include <linux/types.h> > +#include <linux/spinlock.h> > +#include <linux/kref.h> What is this include for? I'd expect to see linux/list.h instead. > +#include <stdarg.h> > + > +struct string_stream_fragment { > + struct list_head node; > + char *fragment; > +}; > + > +struct string_stream { > + size_t length; > + struct list_head fragments; > + /* length and fragments are protected by this lock */ > + spinlock_t lock; > +}; > + > diff --git a/kunit/string-stream.c b/kunit/string-stream.c > new file mode 100644 > index 0000000000000..0463a92dad74b > --- /dev/null > +++ b/kunit/string-stream.c > @@ -0,0 +1,147 @@ > +// SPDX-License-Identifier: GPL-2.0 > +/* > + * C++ stream style string builder used in KUnit for building messages. > + * > + * Copyright (C) 2019, Google LLC. > + * Author: Brendan Higgins <brendanhiggins@google.com> > + */ > + > +#include <linux/list.h> > +#include <linux/slab.h> > +#include <kunit/string-stream.h> > +#include <kunit/test.h> > + > +int string_stream_vadd(struct string_stream *stream, > + const char *fmt, > + va_list args) > +{ > + struct string_stream_fragment *frag_container; > + int len; > + va_list args_for_counting; > + unsigned long flags; > + > + /* Make a copy because `vsnprintf` could change it */ > + va_copy(args_for_counting, args); > + > + /* Need space for null byte. */ > + len = vsnprintf(NULL, 0, fmt, args_for_counting) + 1; > + > + va_end(args_for_counting); > + > + frag_container = kmalloc(sizeof(*frag_container), GFP_KERNEL); This is confusing in that it allocates with GFP_KERNEL but then grabs a spinlock to add and remove from the fragment list. Is it ever going to be called from a place where it can't sleep? If so, the GFP_KERNEL needs to be changed. Otherwise, maybe a mutex would work better to protect access to the fragment list. I also wonder if it would be better to just have a big slop buffer of a 4K page or something so that we almost never have to allocate anything with a string_stream and we can just rely on a reader consuming data while writers are writing. That might work out better, but I don't quite understand the use case for the string stream. > + if (!frag_container) > + return -ENOMEM; > + > + frag_container->fragment = kmalloc(len, GFP_KERNEL); > + if (!frag_container->fragment) { > + kfree(frag_container); > + return -ENOMEM; > + } > + > + len = vsnprintf(frag_container->fragment, len, fmt, args); > + spin_lock_irqsave(&stream->lock, flags); > + stream->length += len; > + list_add_tail(&frag_container->node, &stream->fragments); > + spin_unlock_irqrestore(&stream->lock, flags); > + > + return 0; > +} > + [...] > + > +bool string_stream_is_empty(struct string_stream *stream) > +{ > + bool is_empty; > + unsigned long flags; > + > + spin_lock_irqsave(&stream->lock, flags); I'm not sure what benefit grabbing the lock is having here. If the list isn't empty after this is called then the race isn't resolved by grabbing and releasing the lock. The function is returning stale data in that case. > + is_empty = list_empty(&stream->fragments); > + spin_unlock_irqrestore(&stream->lock, flags); > + > + return is_empty; > +} > + > +static int string_stream_init(struct kunit_resource *res, void *context) > +{ > + struct string_stream *stream; > + > + stream = kzalloc(sizeof(*stream), GFP_KERNEL); > + if (!stream) > + return -ENOMEM; > + > + res->allocation = stream; > + INIT_LIST_HEAD(&stream->fragments); > + spin_lock_init(&stream->lock); > + > + return 0; > +} > + > +static void string_stream_free(struct kunit_resource *res) > +{ > + struct string_stream *stream = res->allocation; > + > + string_stream_clear(stream); > + kfree(stream); > +} > + > +struct string_stream *alloc_string_stream(struct kunit *test) > +{ > + struct kunit_resource *res; > + > + res = kunit_alloc_resource(test, > + string_stream_init, > + string_stream_free, > + NULL); > + > + if (!res) > + return NULL; > + > + return res->allocation; Maybe kunit_alloc_resource() should just return res->allocation, or NULL, so that these functions can be simplified to 'return kunit_alloc_resource()'? Does the caller ever care to do anything with struct kunit_resource anyway?
On Mon, Jul 15, 2019 at 1:43 PM Stephen Boyd <sboyd@kernel.org> wrote: > > Quoting Brendan Higgins (2019-07-12 01:17:29) > > diff --git a/include/kunit/string-stream.h b/include/kunit/string-stream.h > > new file mode 100644 > > index 0000000000000..0552a05781afe > > --- /dev/null > > +++ b/include/kunit/string-stream.h > > @@ -0,0 +1,49 @@ > > +/* SPDX-License-Identifier: GPL-2.0 */ > > +/* > > + * C++ stream style string builder used in KUnit for building messages. > > + * > > + * Copyright (C) 2019, Google LLC. > > + * Author: Brendan Higgins <brendanhiggins@google.com> > > + */ > > + > > +#ifndef _KUNIT_STRING_STREAM_H > > +#define _KUNIT_STRING_STREAM_H > > + > > +#include <linux/types.h> > > +#include <linux/spinlock.h> > > +#include <linux/kref.h> > > What is this include for? I'd expect to see linux/list.h instead. Sorry about that. I used to reference count this before I made it a kunit managed resource. > > +#include <stdarg.h> > > + > > +struct string_stream_fragment { > > + struct list_head node; > > + char *fragment; > > +}; > > + > > +struct string_stream { > > + size_t length; > > + struct list_head fragments; > > + /* length and fragments are protected by this lock */ > > + spinlock_t lock; > > +}; > > + > > diff --git a/kunit/string-stream.c b/kunit/string-stream.c > > new file mode 100644 > > index 0000000000000..0463a92dad74b > > --- /dev/null > > +++ b/kunit/string-stream.c > > @@ -0,0 +1,147 @@ > > +// SPDX-License-Identifier: GPL-2.0 > > +/* > > + * C++ stream style string builder used in KUnit for building messages. > > + * > > + * Copyright (C) 2019, Google LLC. > > + * Author: Brendan Higgins <brendanhiggins@google.com> > > + */ > > + > > +#include <linux/list.h> > > +#include <linux/slab.h> > > +#include <kunit/string-stream.h> > > +#include <kunit/test.h> > > + > > +int string_stream_vadd(struct string_stream *stream, > > + const char *fmt, > > + va_list args) > > +{ > > + struct string_stream_fragment *frag_container; > > + int len; > > + va_list args_for_counting; > > + unsigned long flags; > > + > > + /* Make a copy because `vsnprintf` could change it */ > > + va_copy(args_for_counting, args); > > + > > + /* Need space for null byte. */ > > + len = vsnprintf(NULL, 0, fmt, args_for_counting) + 1; > > + > > + va_end(args_for_counting); > > + > > + frag_container = kmalloc(sizeof(*frag_container), GFP_KERNEL); > > This is confusing in that it allocates with GFP_KERNEL but then grabs a > spinlock to add and remove from the fragment list. Is it ever going to > be called from a place where it can't sleep? If so, the GFP_KERNEL needs > to be changed. Otherwise, maybe a mutex would work better to protect > access to the fragment list. Right, using a mutex here would be fine. Sorry, I meant to filter for my usage of them after you asked me to remove them in 01, but evidently I forgot to do so. Sorry, will fix. > I also wonder if it would be better to just have a big slop buffer of a > 4K page or something so that we almost never have to allocate anything > with a string_stream and we can just rely on a reader consuming data > while writers are writing. That might work out better, but I don't quite > understand the use case for the string stream. That makes sense, but might that also waste memory since we will almost never need that much memory? > > + if (!frag_container) > > + return -ENOMEM; > > + > > + frag_container->fragment = kmalloc(len, GFP_KERNEL); > > + if (!frag_container->fragment) { > > + kfree(frag_container); > > + return -ENOMEM; > > + } > > + > > + len = vsnprintf(frag_container->fragment, len, fmt, args); > > + spin_lock_irqsave(&stream->lock, flags); > > + stream->length += len; > > + list_add_tail(&frag_container->node, &stream->fragments); > > + spin_unlock_irqrestore(&stream->lock, flags); > > + > > + return 0; > > +} > > + > [...] > > + > > +bool string_stream_is_empty(struct string_stream *stream) > > +{ > > + bool is_empty; > > + unsigned long flags; > > + > > + spin_lock_irqsave(&stream->lock, flags); > > I'm not sure what benefit grabbing the lock is having here. If the list > isn't empty after this is called then the race isn't resolved by > grabbing and releasing the lock. The function is returning stale data in > that case. Good point, I didn't realize list_empty was protected by READ_ONCE. Will fix. > > + is_empty = list_empty(&stream->fragments); > > + spin_unlock_irqrestore(&stream->lock, flags); > > + > > + return is_empty; > > +} > > + > > +static int string_stream_init(struct kunit_resource *res, void *context) > > +{ > > + struct string_stream *stream; > > + > > + stream = kzalloc(sizeof(*stream), GFP_KERNEL); > > + if (!stream) > > + return -ENOMEM; > > + > > + res->allocation = stream; > > + INIT_LIST_HEAD(&stream->fragments); > > + spin_lock_init(&stream->lock); > > + > > + return 0; > > +} > > + > > +static void string_stream_free(struct kunit_resource *res) > > +{ > > + struct string_stream *stream = res->allocation; > > + > > + string_stream_clear(stream); > > + kfree(stream); > > +} > > + > > +struct string_stream *alloc_string_stream(struct kunit *test) > > +{ > > + struct kunit_resource *res; > > + > > + res = kunit_alloc_resource(test, > > + string_stream_init, > > + string_stream_free, > > + NULL); > > + > > + if (!res) > > + return NULL; > > + > > + return res->allocation; > > Maybe kunit_alloc_resource() should just return res->allocation, or > NULL, so that these functions can be simplified to 'return > kunit_alloc_resource()'? Does the caller ever care to do anything with > struct kunit_resource anyway? Another good point. I think originally I thought it might, but now with the mandatory init function, the user has to provide a function where they can do the init work. They might as well do it there. Will fix.
Quoting Brendan Higgins (2019-07-15 14:11:50) > On Mon, Jul 15, 2019 at 1:43 PM Stephen Boyd <sboyd@kernel.org> wrote: > > > > I also wonder if it would be better to just have a big slop buffer of a > > 4K page or something so that we almost never have to allocate anything > > with a string_stream and we can just rely on a reader consuming data > > while writers are writing. That might work out better, but I don't quite > > understand the use case for the string stream. > > That makes sense, but might that also waste memory since we will > almost never need that much memory? Why do we care? These are unit tests. Having allocations in here makes things more complicated, whereas it would be simpler to have a pointer and a spinlock operating on a chunk of memory that gets flushed out periodically.
On Mon, Jul 15, 2019 at 3:04 PM Stephen Boyd <sboyd@kernel.org> wrote: > > Quoting Brendan Higgins (2019-07-15 14:11:50) > > On Mon, Jul 15, 2019 at 1:43 PM Stephen Boyd <sboyd@kernel.org> wrote: > > > > > > I also wonder if it would be better to just have a big slop buffer of a > > > 4K page or something so that we almost never have to allocate anything > > > with a string_stream and we can just rely on a reader consuming data > > > while writers are writing. That might work out better, but I don't quite > > > understand the use case for the string stream. > > > > That makes sense, but might that also waste memory since we will > > almost never need that much memory? > > Why do we care? These are unit tests. Agreed. > Having allocations in here makes > things more complicated, whereas it would be simpler to have a pointer > and a spinlock operating on a chunk of memory that gets flushed out > periodically. I am not so sure. I have to have the logic to allocate memory in some case no matter what (what if I need more memory that my preallocated chuck?). I think it is simpler to always request an allocation than to only sometimes request an allocation.
On Mon, Jul 15, 2019 at 3:11 PM Brendan Higgins <brendanhiggins@google.com> wrote: > > On Mon, Jul 15, 2019 at 3:04 PM Stephen Boyd <sboyd@kernel.org> wrote: > > > > Quoting Brendan Higgins (2019-07-15 14:11:50) > > > On Mon, Jul 15, 2019 at 1:43 PM Stephen Boyd <sboyd@kernel.org> wrote: > > > > > > > > I also wonder if it would be better to just have a big slop buffer of a > > > > 4K page or something so that we almost never have to allocate anything > > > > with a string_stream and we can just rely on a reader consuming data > > > > while writers are writing. That might work out better, but I don't quite > > > > understand the use case for the string stream. > > > > > > That makes sense, but might that also waste memory since we will > > > almost never need that much memory? > > > > Why do we care? These are unit tests. > > Agreed. > > > Having allocations in here makes > > things more complicated, whereas it would be simpler to have a pointer > > and a spinlock operating on a chunk of memory that gets flushed out > > periodically. > > I am not so sure. I have to have the logic to allocate memory in some > case no matter what (what if I need more memory that my preallocated > chuck?). I think it is simpler to always request an allocation than to > only sometimes request an allocation. Another even simpler alternative might be to just allocate memory using kunit_kmalloc as we need it and just let the kunit_resource code handle cleaning it all up when the test case finishes. What do you think?
Quoting Brendan Higgins (2019-07-15 15:43:20) > On Mon, Jul 15, 2019 at 3:11 PM Brendan Higgins > <brendanhiggins@google.com> wrote: > > > > On Mon, Jul 15, 2019 at 3:04 PM Stephen Boyd <sboyd@kernel.org> wrote: > > > > > > Quoting Brendan Higgins (2019-07-15 14:11:50) > > > > On Mon, Jul 15, 2019 at 1:43 PM Stephen Boyd <sboyd@kernel.org> wrote: > > > > > > > > > > I also wonder if it would be better to just have a big slop buffer of a > > > > > 4K page or something so that we almost never have to allocate anything > > > > > with a string_stream and we can just rely on a reader consuming data > > > > > while writers are writing. That might work out better, but I don't quite > > > > > understand the use case for the string stream. > > > > > > > > That makes sense, but might that also waste memory since we will > > > > almost never need that much memory? > > > > > > Why do we care? These are unit tests. > > > > Agreed. > > > > > Having allocations in here makes > > > things more complicated, whereas it would be simpler to have a pointer > > > and a spinlock operating on a chunk of memory that gets flushed out > > > periodically. > > > > I am not so sure. I have to have the logic to allocate memory in some > > case no matter what (what if I need more memory that my preallocated > > chuck?). I think it is simpler to always request an allocation than to > > only sometimes request an allocation. > > Another even simpler alternative might be to just allocate memory > using kunit_kmalloc as we need it and just let the kunit_resource code > handle cleaning it all up when the test case finishes. Sure, sounds like a nice way to avoid duplicating similar logic to maintain a list of things to free later. > > What do you think? If you go the allocation route then you'll need to have the flags to know what context you're in to allocate appropriately. Does that mean all the string operations will now take GFP flags?
On Tue, Jul 16, 2019 at 8:34 AM Stephen Boyd <sboyd@kernel.org> wrote: > > Quoting Brendan Higgins (2019-07-15 15:43:20) > > On Mon, Jul 15, 2019 at 3:11 PM Brendan Higgins > > <brendanhiggins@google.com> wrote: > > > > > > On Mon, Jul 15, 2019 at 3:04 PM Stephen Boyd <sboyd@kernel.org> wrote: > > > > > > > > Quoting Brendan Higgins (2019-07-15 14:11:50) > > > > > On Mon, Jul 15, 2019 at 1:43 PM Stephen Boyd <sboyd@kernel.org> wrote: > > > > > > > > > > > > I also wonder if it would be better to just have a big slop buffer of a > > > > > > 4K page or something so that we almost never have to allocate anything > > > > > > with a string_stream and we can just rely on a reader consuming data > > > > > > while writers are writing. That might work out better, but I don't quite > > > > > > understand the use case for the string stream. > > > > > > > > > > That makes sense, but might that also waste memory since we will > > > > > almost never need that much memory? > > > > > > > > Why do we care? These are unit tests. > > > > > > Agreed. > > > > > > > Having allocations in here makes > > > > things more complicated, whereas it would be simpler to have a pointer > > > > and a spinlock operating on a chunk of memory that gets flushed out > > > > periodically. > > > > > > I am not so sure. I have to have the logic to allocate memory in some > > > case no matter what (what if I need more memory that my preallocated > > > chuck?). I think it is simpler to always request an allocation than to > > > only sometimes request an allocation. > > > > Another even simpler alternative might be to just allocate memory > > using kunit_kmalloc as we need it and just let the kunit_resource code > > handle cleaning it all up when the test case finishes. > > Sure, sounds like a nice way to avoid duplicating similar logic to > maintain a list of things to free later. I think I will go that route for now. > > > > What do you think? > > If you go the allocation route then you'll need to have the flags to > know what context you're in to allocate appropriately. Does that mean > all the string operations will now take GFP flags? We could set the GFP flags in the constructor, store them in a field, and then just reuse them. Thanks!
diff --git a/include/kunit/string-stream.h b/include/kunit/string-stream.h new file mode 100644 index 0000000000000..0552a05781afe --- /dev/null +++ b/include/kunit/string-stream.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * C++ stream style string builder used in KUnit for building messages. + * + * Copyright (C) 2019, Google LLC. + * Author: Brendan Higgins <brendanhiggins@google.com> + */ + +#ifndef _KUNIT_STRING_STREAM_H +#define _KUNIT_STRING_STREAM_H + +#include <linux/types.h> +#include <linux/spinlock.h> +#include <linux/kref.h> +#include <stdarg.h> + +struct string_stream_fragment { + struct list_head node; + char *fragment; +}; + +struct string_stream { + size_t length; + struct list_head fragments; + /* length and fragments are protected by this lock */ + spinlock_t lock; +}; + +struct kunit; + +struct string_stream *alloc_string_stream(struct kunit *test); + +void string_stream_get(struct string_stream *stream); + +int string_stream_put(struct string_stream *stream); + +int string_stream_add(struct string_stream *stream, const char *fmt, ...); + +int string_stream_vadd(struct string_stream *stream, + const char *fmt, + va_list args); + +char *string_stream_get_string(struct string_stream *stream); + +void string_stream_clear(struct string_stream *stream); + +bool string_stream_is_empty(struct string_stream *stream); + +#endif /* _KUNIT_STRING_STREAM_H */ diff --git a/kunit/Makefile b/kunit/Makefile index 5efdc4dea2c08..275b565a0e81f 100644 --- a/kunit/Makefile +++ b/kunit/Makefile @@ -1 +1,2 @@ -obj-$(CONFIG_KUNIT) += test.o +obj-$(CONFIG_KUNIT) += test.o \ + string-stream.o diff --git a/kunit/string-stream.c b/kunit/string-stream.c new file mode 100644 index 0000000000000..0463a92dad74b --- /dev/null +++ b/kunit/string-stream.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * C++ stream style string builder used in KUnit for building messages. + * + * Copyright (C) 2019, Google LLC. + * Author: Brendan Higgins <brendanhiggins@google.com> + */ + +#include <linux/list.h> +#include <linux/slab.h> +#include <kunit/string-stream.h> +#include <kunit/test.h> + +int string_stream_vadd(struct string_stream *stream, + const char *fmt, + va_list args) +{ + struct string_stream_fragment *frag_container; + int len; + va_list args_for_counting; + unsigned long flags; + + /* Make a copy because `vsnprintf` could change it */ + va_copy(args_for_counting, args); + + /* Need space for null byte. */ + len = vsnprintf(NULL, 0, fmt, args_for_counting) + 1; + + va_end(args_for_counting); + + frag_container = kmalloc(sizeof(*frag_container), GFP_KERNEL); + if (!frag_container) + return -ENOMEM; + + frag_container->fragment = kmalloc(len, GFP_KERNEL); + if (!frag_container->fragment) { + kfree(frag_container); + return -ENOMEM; + } + + len = vsnprintf(frag_container->fragment, len, fmt, args); + spin_lock_irqsave(&stream->lock, flags); + stream->length += len; + list_add_tail(&frag_container->node, &stream->fragments); + spin_unlock_irqrestore(&stream->lock, flags); + + return 0; +} + +int string_stream_add(struct string_stream *stream, const char *fmt, ...) +{ + va_list args; + int result; + + va_start(args, fmt); + result = string_stream_vadd(stream, fmt, args); + va_end(args); + + return result; +} + +void string_stream_clear(struct string_stream *stream) +{ + struct string_stream_fragment *frag_container, *frag_container_safe; + unsigned long flags; + + spin_lock_irqsave(&stream->lock, flags); + list_for_each_entry_safe(frag_container, + frag_container_safe, + &stream->fragments, + node) { + list_del(&frag_container->node); + kfree(frag_container->fragment); + kfree(frag_container); + } + stream->length = 0; + spin_unlock_irqrestore(&stream->lock, flags); +} + +char *string_stream_get_string(struct string_stream *stream) +{ + struct string_stream_fragment *frag_container; + size_t buf_len = stream->length + 1; /* +1 for null byte. */ + char *buf; + unsigned long flags; + + buf = kzalloc(buf_len, GFP_KERNEL); + if (!buf) + return NULL; + + spin_lock_irqsave(&stream->lock, flags); + list_for_each_entry(frag_container, &stream->fragments, node) + strlcat(buf, frag_container->fragment, buf_len); + spin_unlock_irqrestore(&stream->lock, flags); + + return buf; +} + +bool string_stream_is_empty(struct string_stream *stream) +{ + bool is_empty; + unsigned long flags; + + spin_lock_irqsave(&stream->lock, flags); + is_empty = list_empty(&stream->fragments); + spin_unlock_irqrestore(&stream->lock, flags); + + return is_empty; +} + +static int string_stream_init(struct kunit_resource *res, void *context) +{ + struct string_stream *stream; + + stream = kzalloc(sizeof(*stream), GFP_KERNEL); + if (!stream) + return -ENOMEM; + + res->allocation = stream; + INIT_LIST_HEAD(&stream->fragments); + spin_lock_init(&stream->lock); + + return 0; +} + +static void string_stream_free(struct kunit_resource *res) +{ + struct string_stream *stream = res->allocation; + + string_stream_clear(stream); + kfree(stream); +} + +struct string_stream *alloc_string_stream(struct kunit *test) +{ + struct kunit_resource *res; + + res = kunit_alloc_resource(test, + string_stream_init, + string_stream_free, + NULL); + + if (!res) + return NULL; + + return res->allocation; +}