@@ -129,6 +129,7 @@ config MD_RAID456
select ASYNC_PQ
select ASYNC_RAID6_RECOV
select BLK_BBU if MD_RAID456_BBU
+ select BTRQ if MULTICORE_RAID456
---help---
A RAID-5 set of N drives with a capacity of C MB per drive provides
the capacity of C * (N - 1) MB, and protects against a failure
@@ -1084,7 +1084,7 @@ static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu
&sh->ops.zero_sum_result, percpu->spare_page, &submit);
}
-static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
+static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
{
int overlap_clear = 0, i, disks = sh->disks;
struct dma_async_tx_descriptor *tx = NULL;
@@ -1149,34 +1149,9 @@ static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
put_cpu();
}
-#ifdef CONFIG_MULTICORE_RAID456
-static void async_run_ops(void *param, async_cookie_t cookie)
-{
- struct stripe_head *sh = param;
- unsigned long ops_request = sh->ops.request;
-
- clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state);
- wake_up(&sh->ops.wait_for_ops);
-
- __raid_run_ops(sh, ops_request);
- release_stripe(sh);
-}
-
-static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
-{
- /* since handle_stripe can be called outside of raid5d context
- * we need to ensure sh->ops.request is de-staged before another
- * request arrives
- */
- wait_event(sh->ops.wait_for_ops,
- !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state));
- sh->ops.request = ops_request;
- atomic_inc(&sh->count);
- async_schedule(async_run_ops, sh);
-}
-#else
-#define raid_run_ops __raid_run_ops
+#ifdef CONFIG_MULTICORE_RAID456
+static void stripe_work(struct btrq_work *work);
#endif
static int grow_one_stripe(raid5_conf_t *conf)
@@ -1189,9 +1164,10 @@ static int grow_one_stripe(raid5_conf_t *conf)
memset(sh, 0, sizeof(*sh) + (disks-1)*sizeof(struct r5dev));
sh->raid_conf = conf;
spin_lock_init(&sh->lock);
- #ifdef CONFIG_MULTICORE_RAID456
- init_waitqueue_head(&sh->ops.wait_for_ops);
- #endif
+#ifdef CONFIG_MULTICORE_RAID456
+ sh->work.func = stripe_work;
+ sh->work.flags = 0;
+#endif
if (grow_buffers(sh, disks)) {
shrink_buffers(sh, disks);
@@ -1307,9 +1283,10 @@ static int resize_stripes(raid5_conf_t *conf, int newsize)
nsh->raid_conf = conf;
spin_lock_init(&nsh->lock);
- #ifdef CONFIG_MULTICORE_RAID456
- init_waitqueue_head(&nsh->ops.wait_for_ops);
- #endif
+#ifdef CONFIG_MULTICORE_RAID456
+ nsh->work.func = stripe_work;
+ nsh->work.flags = 0;
+#endif
list_add(&nsh->lru, &newstripes);
}
@@ -3441,6 +3418,26 @@ static void handle_stripe6(struct stripe_head *sh)
return_io(return_bi);
}
+#ifdef CONFIG_MULTICORE_RAID456
+static void stripe_work(struct btrq_work *work)
+{
+ struct stripe_head *sh = container_of(work, struct stripe_head, work);
+
+ if (sh->raid_conf->level == 6)
+ handle_stripe6(sh);
+ else
+ handle_stripe5(sh);
+ release_stripe(sh);
+}
+
+static void handle_stripe(struct stripe_head *sh)
+{
+ raid5_conf_t *conf = sh->raid_conf;
+
+ atomic_inc(&sh->count);
+ btrq_queue_worker(&conf->workqueue, &sh->work);
+}
+#else
static void handle_stripe(struct stripe_head *sh)
{
if (sh->raid_conf->level == 6)
@@ -3448,6 +3445,7 @@ static void handle_stripe(struct stripe_head *sh)
else
handle_stripe5(sh);
}
+#endif
static void raid5_activate_delayed(raid5_conf_t *conf)
{
@@ -4546,6 +4544,9 @@ static void free_conf(raid5_conf_t *conf)
raid5_free_percpu(conf);
kfree(conf->disks);
kfree(conf->stripe_hashtbl);
+#ifdef CONFIG_MULTICORE_RAID456
+ btrq_stop_workers(&conf->workqueue);
+#endif
kfree(conf);
}
@@ -4683,6 +4684,16 @@ static raid5_conf_t *setup_conf(mddev_t *mddev)
atomic_set(&conf->preread_active_stripes, 0);
atomic_set(&conf->active_aligned_reads, 0);
conf->bypass_threshold = BYPASS_THRESHOLD;
+#ifdef CONFIG_MULTICORE_RAID456
+ sprintf(conf->queue_name, "%s_%s", mdname(mddev), mddev->pers->name);
+ btrq_init_workers(&conf->workqueue, conf->queue_name,
+ min_t(unsigned long, num_online_cpus() + 2, 8), NULL);
+ if (btrq_start_workers(&conf->workqueue, 1)) {
+ printk(KERN_ERR "raid5: failed to start thread pool for %s\n",
+ mdname(mddev));
+ goto abort;
+ }
+#endif
conf->raid_disks = mddev->raid_disks;
if (mddev->reshape_position == MaxSector)
@@ -3,6 +3,7 @@
#include <linux/raid/xor.h>
#include <linux/dmaengine.h>
+#include <linux/btrqueue.h>
/*
*
@@ -214,6 +215,9 @@ struct stripe_head {
int disks; /* disks in stripe */
enum check_states check_state;
enum reconstruct_states reconstruct_state;
+#ifdef CONFIG_MULTICORE_RAID456
+ struct btrq_work work;
+#endif
/**
* struct stripe_operations
* @target - STRIPE_OP_COMPUTE_BLK target
@@ -224,10 +228,6 @@ struct stripe_head {
struct stripe_operations {
int target, target2;
enum sum_check_flags zero_sum_result;
- #ifdef CONFIG_MULTICORE_RAID456
- unsigned long request;
- wait_queue_head_t wait_for_ops;
- #endif
} ops;
struct r5dev {
struct bio req;
@@ -302,7 +302,6 @@ struct r6_state {
#define STRIPE_FULL_WRITE 13 /* all blocks are set to be overwritten */
#define STRIPE_BIOFILL_RUN 14
#define STRIPE_COMPUTE_RUN 15
-#define STRIPE_OPS_REQ_PENDING 16
/*
* Operation request flags
@@ -382,6 +381,10 @@ struct raid5_private_data {
int bypass_count; /* bypassed prereads */
int bypass_threshold; /* preread nice */
struct list_head *last_hold; /* detect hold_list promotions */
+#ifdef CONFIG_MULTICORE_RAID456
+ struct btrq_workers workqueue;
+ char queue_name[20];
+#endif
atomic_t reshape_stripes; /* stripes with pending writes for reshape */
/* unfortunately we need two cache names as we temporarily have