===================================================================
@@ -1747,37 +1747,69 @@ EXPORT_SYMBOL(bio_flush_dcache_pages);
* bio unless they own it and thus know that it has an end_io
* function.
**/
+
+static DEFINE_PER_CPU(struct bio **, bio_end_queue) = { NULL };
+
void bio_endio(struct bio *bio, int error)
{
- while (bio) {
- BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
+ struct bio ***bio_end_queue_ptr;
+
+ unsigned long flags;
+
+ BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
+
+ if (error)
+ clear_bit(BIO_UPTODATE, &bio->bi_flags);
+ else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
+ error = -EIO;
+
+ if (!atomic_dec_and_test(&bio->bi_remaining))
+ return;
+
+ /* save the error to the bio */
+ bio->bi_error = error;
- if (error)
- clear_bit(BIO_UPTODATE, &bio->bi_flags);
- else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
- error = -EIO;
+ preempt_disable();
+ local_irq_save(flags);
+ bio_end_queue_ptr = &__get_cpu_var(bio_end_queue);
+
+ if (*bio_end_queue_ptr) {
+ **bio_end_queue_ptr = bio;
+ *bio_end_queue_ptr = &bio->bi_next;
+ bio->bi_next = NULL;
+ } else {
+ struct bio *bio_queue = NULL;
+ *bio_end_queue_ptr = &bio_queue;
- if (!atomic_dec_and_test(&bio->bi_remaining))
- return;
+next_bio:
+ local_irq_restore(flags);
+ /* restore the saved error */
+ error = bio->bi_error;
/*
- * Need to have a real endio function for chained bios,
- * otherwise various corner cases will break (like stacking
- * block devices that save/restore bi_end_io) - however, we want
- * to avoid unbounded recursion and blowing the stack. Tail call
- * optimization would handle this, but compiling with frame
- * pointers also disables gcc's sibling call optimization.
+ * Restore bi_remaining. Don't use atomic_set - there is no
+ * concurrent access and we want to avoid taking spinlock on
+ * architectures where atomic_set takes it.
*/
- if (bio->bi_end_io == bio_chain_endio) {
- struct bio *parent = bio->bi_private;
- bio_put(bio);
- bio = parent;
- } else {
- if (bio->bi_end_io)
- bio->bi_end_io(bio, error);
- bio = NULL;
+ bio->bi_remaining = (atomic_t)ATOMIC_INIT(0);
+
+ if (bio->bi_end_io)
+ bio->bi_end_io(bio, error);
+
+ local_irq_disable();
+
+ if (bio_queue) {
+ bio = bio_queue;
+ bio_queue = bio->bi_next;
+ if (likely(!bio_queue))
+ *bio_end_queue_ptr = &bio_queue;
+ goto next_bio;
}
+ *bio_end_queue_ptr = NULL;
}
+
+ local_irq_restore(flags);
+ preempt_enable();
}
EXPORT_SYMBOL(bio_endio);
===================================================================
@@ -65,7 +65,10 @@ struct bio {
unsigned int bi_seg_front_size;
unsigned int bi_seg_back_size;
- atomic_t bi_remaining;
+ union {
+ atomic_t bi_remaining;
+ int bi_error;
+ };
bio_end_io_t *bi_end_io;