@@ -1526,6 +1526,11 @@ static void pp_init(struct parallel_processes *pp,
if (!opts->get_next_task)
BUG("you need to specify a get_next_task function");
+ if (opts->ungroup) {
+ if (opts->on_stderr_output)
+ BUG("on_stderr_output and ungroup are incompatible with each other");
+ }
+
CALLOC_ARRAY(pp->children, n);
if (!opts->ungroup)
CALLOC_ARRAY(pp->pfd, n);
@@ -1645,14 +1650,19 @@ static void pp_buffer_stderr(struct parallel_processes *pp,
for (size_t i = 0; i < opts->processes; i++) {
if (pp->children[i].state == GIT_CP_WORKING &&
pp->pfd[i].revents & (POLLIN | POLLHUP)) {
- int n = strbuf_read_once(&pp->children[i].err,
- pp->children[i].process.err, 0);
+ ssize_t n = strbuf_read_once(&pp->children[i].err,
+ pp->children[i].process.err, 0);
if (n == 0) {
close(pp->children[i].process.err);
pp->children[i].state = GIT_CP_WAIT_CLEANUP;
- } else if (n < 0)
+ } else if (n < 0) {
if (errno != EAGAIN)
die_errno("read");
+ } else if (opts->on_stderr_output) {
+ opts->on_stderr_output(&pp->children[i].err,
+ pp->children[i].err.len - n,
+ opts->data, pp->children[i].data);
+ }
}
}
}
@@ -408,6 +408,25 @@ typedef int (*start_failure_fn)(struct strbuf *out,
void *pp_cb,
void *pp_task_cb);
+/**
+ * This callback is called whenever output from a child process is buffered
+ *
+ * See run_processes_parallel() below for a discussion of the "struct
+ * strbuf *out" parameter.
+ *
+ * The offset refers to the number of bytes originally in "out" before
+ * the output from the child process was buffered. Therefore, the buffer
+ * range, "out + buf" to the end of "out", would contain the buffer of
+ * the child process output.
+ *
+ * pp_cb is the callback cookie as passed into run_processes_parallel,
+ * pp_task_cb is the callback cookie as passed into get_next_task_fn.
+ *
+ * This function is incompatible with "ungroup"
+ */
+typedef void (*on_stderr_output_fn)(struct strbuf *out, size_t offset,
+ void *pp_cb, void *pp_task_cb);
+
/**
* This callback is called on every child process that finished processing.
*
@@ -461,6 +480,12 @@ struct run_process_parallel_opts
*/
start_failure_fn start_failure;
+ /**
+ * on_stderr_output: See on_stderr_output_fn() above. Unless you need
+ * to capture output from child processes, leave this as NULL.
+ */
+ on_stderr_output_fn on_stderr_output;
+
/**
* task_finished: See task_finished_fn() above. This can be
* NULL to omit any special handling.
@@ -52,6 +52,20 @@ static int no_job(struct child_process *cp,
return 0;
}
+static void on_stderr_output(struct strbuf *out,
+ size_t offset,
+ void *pp_cb UNUSED,
+ void *pp_task_cb UNUSED)
+{
+ struct string_list list = STRING_LIST_INIT_DUP;
+ struct string_list_item *item;
+
+ string_list_split(&list, out->buf + offset, '\n', -1);
+ for_each_string_list_item(item, &list)
+ fprintf(stderr, "on_stderr_output: %s\n", item->string);
+ string_list_clear(&list, 0);
+}
+
static int task_finished(int result,
struct strbuf *err,
void *pp_cb,
@@ -439,6 +453,12 @@ int cmd__run_command(int argc, const char **argv)
opts.ungroup = 1;
}
+ if (!strcmp(argv[1], "--on-stderr-output")) {
+ argv += 1;
+ argc -= 1;
+ opts.on_stderr_output = on_stderr_output;
+ }
+
jobs = atoi(argv[2]);
strvec_clear(&proc.args);
strvec_pushv(&proc.args, (const char **)argv + 3);
@@ -135,6 +135,15 @@ test_expect_success 'run_command runs in parallel with more jobs available than
test_cmp expect actual
'
+test_expect_success 'run_command runs in parallel with more jobs available than tasks --on-stderr-output' '
+ test-tool run-command --on-stderr-output run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+ test_must_be_empty out &&
+ test 4 = $(grep -c "on_stderr_output: Hello" err) &&
+ test 4 = $(grep -c "on_stderr_output: World" err) &&
+ sed "/on_stderr_output/d" err >err1 &&
+ test_cmp expect err1
+'
+
test_expect_success 'run_command runs ungrouped in parallel with more jobs available than tasks' '
test-tool run-command --ungroup run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
test_line_count = 8 out &&
@@ -147,6 +156,15 @@ test_expect_success 'run_command runs in parallel with as many jobs as tasks' '
test_cmp expect actual
'
+test_expect_success 'run_command runs in parallel with as many jobs as tasks --on-stderr-output' '
+ test-tool run-command --on-stderr-output run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+ test_must_be_empty out &&
+ test 4 = $(grep -c "on_stderr_output: Hello" err) &&
+ test 4 = $(grep -c "on_stderr_output: World" err) &&
+ sed "/on_stderr_output/d" err >err1 &&
+ test_cmp expect err1
+'
+
test_expect_success 'run_command runs ungrouped in parallel with as many jobs as tasks' '
test-tool run-command --ungroup run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
test_line_count = 8 out &&
@@ -159,6 +177,15 @@ test_expect_success 'run_command runs in parallel with more tasks than jobs avai
test_cmp expect actual
'
+test_expect_success 'run_command runs in parallel with more tasks than jobs available --on-stderr-output' '
+ test-tool run-command --on-stderr-output run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+ test_must_be_empty out &&
+ test 4 = $(grep -c "on_stderr_output: Hello" err) &&
+ test 4 = $(grep -c "on_stderr_output: World" err) &&
+ sed "/on_stderr_output/d" err >err1 &&
+ test_cmp expect err1
+'
+
test_expect_success 'run_command runs ungrouped in parallel with more tasks than jobs available' '
test-tool run-command --ungroup run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
test_line_count = 8 out &&
@@ -180,6 +207,12 @@ test_expect_success 'run_command is asked to abort gracefully' '
test_cmp expect actual
'
+test_expect_success 'run_command is asked to abort gracefully --on-stderr-output' '
+ test-tool run-command --on-stderr-output run-command-abort 3 false >out 2>err &&
+ test_must_be_empty out &&
+ test_cmp expect err
+'
+
test_expect_success 'run_command is asked to abort gracefully (ungroup)' '
test-tool run-command --ungroup run-command-abort 3 false >out 2>err &&
test_must_be_empty out &&
@@ -196,6 +229,12 @@ test_expect_success 'run_command outputs ' '
test_cmp expect actual
'
+test_expect_success 'run_command outputs --on-stderr-output' '
+ test-tool run-command --on-stderr-output run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+ test_must_be_empty out &&
+ test_cmp expect err
+'
+
test_expect_success 'run_command outputs (ungroup) ' '
test-tool run-command --ungroup run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
test_must_be_empty out &&
Add duplicate_output_fn as an optionally set function in run_process_parallel_opts. If set, output from each child process is copied and passed to the callback function whenever output from the child process is buffered to allow for separate parsing. Fix two items in pp_buffer_stderr: * strbuf_read_once returns a ssize_t but the variable it is set to is an int so fix that. * Add missing brackets to "else if" statement The ungroup/duplicate_output incompatibility check is nested to prepare for future imcompatibles modes with ungroup. Signed-off-by: Calvin Wan <calvinwan@google.com> --- run-command.c | 16 ++++++++++++--- run-command.h | 25 ++++++++++++++++++++++++ t/helper/test-run-command.c | 20 +++++++++++++++++++ t/t0061-run-command.sh | 39 +++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-)