[v2,2/2] mingw: fix mingw_open_append to work with named pipes

Commit Message

Linus Arver via GitGitGadget Sept. 10, 2018, 5:05 p.m. UTC

From: Jeff Hostetler <[e-mail]>

Signed-off-by: Jeff Hostetler <[e-mail]>
---
 compat/mingw.c                | 38 ++++++++++++++++++++++++++++++++---
 t/t0051-windows-named-pipe.sh |  2 +-
 2 files changed, 36 insertions(+), 4 deletions(-)

Comments

Johannes Sixt Sept. 10, 2018, 7:45 p.m. UTC | #1

Am 10.09.18 um 19:05 schrieb Jeff Hostetler via GitGitGadget:
> diff --git a/compat/mingw.c b/compat/mingw.c
> index 858ca14a57..f87376b26a 100644
> --- a/compat/mingw.c
> +++ b/compat/mingw.c
> @@ -341,6 +341,19 @@ int mingw_mkdir(const char *path, int mode)
>   	return ret;
>   }
>   
> +/*
> + * Calling CreateFile() using FILE_APPEND_DATA and without FILE_WRITE_DATA
> + * is documented in [1] as opening a writable file handle in append mode.
> + * (It is believed that) this is atomic since it is maintained by the
> + * kernel unlike the O_APPEND flag which is racily maintained by the CRT.
> + *
> + * [1] https://docs.microsoft.com/en-us/windows/desktop/fileio/file-access-rights-constants
> + *
> + * This trick does not appear to work for named pipes.  Instead it creates
> + * a named pipe client handle that cannot be written to.  Callers should
> + * just use the regular _wopen() for them.  (And since client handle gets
> + * bound to a unique server handle, it isn't really an issue.)
> + */
>   static int mingw_open_append(wchar_t const *wfilename, int oflags, ...)
>   {
>   	HANDLE handle;
> @@ -360,10 +373,12 @@ static int mingw_open_append(wchar_t const *wfilename, int oflags, ...)
>   			NULL, create, FILE_ATTRIBUTE_NORMAL, NULL);
>   	if (handle == INVALID_HANDLE_VALUE)
>   		return errno = err_win_to_posix(GetLastError()), -1;
> +
>   	/*
>   	 * No O_APPEND here, because the CRT uses it only to reset the
> -	 * file pointer to EOF on write(); but that is not necessary
> -	 * for a file created with FILE_APPEND_DATA.
> +	 * file pointer to EOF before each write(); but that is not
> +	 * necessary (and may lead to races) for a file created with
> +	 * FILE_APPEND_DATA.
>   	 */
>   	fd = _open_osfhandle((intptr_t)handle, O_BINARY);
>   	if (fd < 0)
> @@ -371,6 +386,23 @@ static int mingw_open_append(wchar_t const *wfilename, int oflags, ...)
>   	return fd;
>   }
>   
> +#define IS_SBS(ch) (((ch) == '/') || ((ch) == '\\'))
> +/*
> + * Does the pathname map to the local named pipe filesystem?
> + * That is, does it have a "//./pipe/" prefix?
> + */
> +static int mingw_is_local_named_pipe_path(const char *filename)
> +{
> +	return (IS_SBS(filename[0]) &&
> +		IS_SBS(filename[1]) &&
> +		filename[2] == '.'  &&
> +		IS_SBS(filename[3]) &&
> +		!strncasecmp(filename+4, "pipe", 4) &&
> +		IS_SBS(filename[8]) &&
> +		filename[9]);
> +}
> +#undef IS_SBS
> +
>   int mingw_open (const char *filename, int oflags, ...)
>   {
>   	typedef int (*open_fn_t)(wchar_t const *wfilename, int oflags, ...);
> @@ -387,7 +419,7 @@ int mingw_open (const char *filename, int oflags, ...)
>   	if (filename && !strcmp(filename, "/dev/null"))
>   		filename = "nul";
>   
> -	if (oflags & O_APPEND)
> +	if ((oflags & O_APPEND) && !mingw_is_local_named_pipe_path(filename))
>   		open_fn = mingw_open_append;
>   	else
>   		open_fn = _wopen;

This looks reasonable.

I wonder which part of the code uses local named pipes. Is it downstream 
in Git for Windows or one of the topics in flight?

-- Hannes

Jeff Hostetler Sept. 10, 2018, 8:07 p.m. UTC | #2

On 9/10/2018 3:45 PM, Johannes Sixt wrote:
> Am 10.09.18 um 19:05 schrieb Jeff Hostetler via GitGitGadget:
>> diff --git a/compat/mingw.c b/compat/mingw.c
>> index 858ca14a57..f87376b26a 100644
>> --- a/compat/mingw.c
>> +++ b/compat/mingw.c
>> @@ -341,6 +341,19 @@ int mingw_mkdir(const char *path, int mode)
>>       return ret;
>>   }
>> +/*
>> + * Calling CreateFile() using FILE_APPEND_DATA and without 
>> FILE_WRITE_DATA
>> + * is documented in [1] as opening a writable file handle in append 
>> mode.
>> + * (It is believed that) this is atomic since it is maintained by the
>> + * kernel unlike the O_APPEND flag which is racily maintained by the 
>> CRT.
>> + *
>> + * [1] 
>> https://docs.microsoft.com/en-us/windows/desktop/fileio/file-access-rights-constants 
>>
>> + *
>> + * This trick does not appear to work for named pipes.  Instead it 
>> creates
>> + * a named pipe client handle that cannot be written to.  Callers should
>> + * just use the regular _wopen() for them.  (And since client handle 
>> gets
>> + * bound to a unique server handle, it isn't really an issue.)
>> + */
>>   static int mingw_open_append(wchar_t const *wfilename, int oflags, ...)
>>   {
>>       HANDLE handle;
>> @@ -360,10 +373,12 @@ static int mingw_open_append(wchar_t const 
>> *wfilename, int oflags, ...)
>>               NULL, create, FILE_ATTRIBUTE_NORMAL, NULL);
>>       if (handle == INVALID_HANDLE_VALUE)
>>           return errno = err_win_to_posix(GetLastError()), -1;
>> +
>>       /*
>>        * No O_APPEND here, because the CRT uses it only to reset the
>> -     * file pointer to EOF on write(); but that is not necessary
>> -     * for a file created with FILE_APPEND_DATA.
>> +     * file pointer to EOF before each write(); but that is not
>> +     * necessary (and may lead to races) for a file created with
>> +     * FILE_APPEND_DATA.
>>        */
>>       fd = _open_osfhandle((intptr_t)handle, O_BINARY);
>>       if (fd < 0)
>> @@ -371,6 +386,23 @@ static int mingw_open_append(wchar_t const 
>> *wfilename, int oflags, ...)
>>       return fd;
>>   }
>> +#define IS_SBS(ch) (((ch) == '/') || ((ch) == '\\'))
>> +/*
>> + * Does the pathname map to the local named pipe filesystem?
>> + * That is, does it have a "//./pipe/" prefix?
>> + */
>> +static int mingw_is_local_named_pipe_path(const char *filename)
>> +{
>> +    return (IS_SBS(filename[0]) &&
>> +        IS_SBS(filename[1]) &&
>> +        filename[2] == '.'  &&
>> +        IS_SBS(filename[3]) &&
>> +        !strncasecmp(filename+4, "pipe", 4) &&
>> +        IS_SBS(filename[8]) &&
>> +        filename[9]);
>> +}
>> +#undef IS_SBS
>> +
>>   int mingw_open (const char *filename, int oflags, ...)
>>   {
>>       typedef int (*open_fn_t)(wchar_t const *wfilename, int oflags, 
>> ...);
>> @@ -387,7 +419,7 @@ int mingw_open (const char *filename, int oflags, 
>> ...)
>>       if (filename && !strcmp(filename, "/dev/null"))
>>           filename = "nul";
>> -    if (oflags & O_APPEND)
>> +    if ((oflags & O_APPEND) && 
>> !mingw_is_local_named_pipe_path(filename))
>>           open_fn = mingw_open_append;
>>       else
>>           open_fn = _wopen;
> 
> This looks reasonable.

Thanks for the review.

> 
> I wonder which part of the code uses local named pipes. Is it downstream 
> in Git for Windows or one of the topics in flight?
> 
> -- Hannes

I'm wanting to use them as a tracing target option in my trace2 series
currently in progress.

Jeff

Junio C Hamano Sept. 10, 2018, 10 p.m. UTC | #3

Johannes Sixt <[e-mail]> writes:

>>   +#define IS_SBS(ch) (((ch) == '/') || ((ch) == '\\'))

I think you already have mingw_is_dir_sep() and its shorter alias
is_dir_sep() available to you.

>> +/*
>> + * Does the pathname map to the local named pipe filesystem?
>> + * That is, does it have a "//./pipe/" prefix?
>> + */
>> +static int mingw_is_local_named_pipe_path(const char *filename)

There is no need to prefix mingw_ to this function that is file
local static.  Isn't is_local_named_pipe() descriptive and unique
enough?

>> +{
>> +	return (IS_SBS(filename[0]) &&
>> +		IS_SBS(filename[1]) &&
>> +		filename[2] == '.'  &&
>> +		IS_SBS(filename[3]) &&
>> +		!strncasecmp(filename+4, "pipe", 4) &&
>> +		IS_SBS(filename[8]) &&
>> +		filename[9]);
>> +}
>> +#undef IS_SBS

It is kind-of surprising that there hasn't been any existing need
for a helper function that would allow us to write this function
like so:

	static int is_local_named_pipe(const char *path)
	{
		return path_is_in_directory(path, "//./pipe/");
	}

Not a suggestion to add such a thing; as long as we know there is no
other codepath that would benefit from having one, a generalization
like that can and should wait.

>>   int mingw_open (const char *filename, int oflags, ...)
>>   {
>>   	typedef int (*open_fn_t)(wchar_t const *wfilename, int oflags, ...);
>> @@ -387,7 +419,7 @@ int mingw_open (const char *filename, int oflags, ...)
>>   	if (filename && !strcmp(filename, "/dev/null"))
>>   		filename = "nul";
>>   -	if (oflags & O_APPEND)
>> +	if ((oflags & O_APPEND) && !mingw_is_local_named_pipe_path(filename))
>>   		open_fn = mingw_open_append;
>>   	else
>>   		open_fn = _wopen;
>
> This looks reasonable.
>
> I wonder which part of the code uses local named pipes. Is it
> downstream in Git for Windows or one of the topics in flight?
>
> -- Hannes

Jeff Hostetler Sept. 11, 2018, 2:25 p.m. UTC | #4

On 9/10/2018 6:00 PM, Junio C Hamano wrote:
> Johannes Sixt <[e-mail]> writes:
> 
>>>    +#define IS_SBS(ch) (((ch) == '/') || ((ch) == '\\'))
> 
> I think you already have mingw_is_dir_sep() and its shorter alias
> is_dir_sep() available to you.

good catch. thanks.


>>> +/*
>>> + * Does the pathname map to the local named pipe filesystem?
>>> + * That is, does it have a "//./pipe/" prefix?
>>> + */
>>> +static int mingw_is_local_named_pipe_path(const char *filename)
> 
> There is no need to prefix mingw_ to this function that is file
> local static.  Isn't is_local_named_pipe() descriptive and unique
> enough?

right. will do.


>>> +{
>>> +	return (IS_SBS(filename[0]) &&
>>> +		IS_SBS(filename[1]) &&
>>> +		filename[2] == '.'  &&
>>> +		IS_SBS(filename[3]) &&
>>> +		!strncasecmp(filename+4, "pipe", 4) &&
>>> +		IS_SBS(filename[8]) &&
>>> +		filename[9]);
>>> +}
>>> +#undef IS_SBS
> 
> It is kind-of surprising that there hasn't been any existing need
> for a helper function that would allow us to write this function
> like so:
> 
> 	static int is_local_named_pipe(const char *path)
> 	{
> 		return path_is_in_directory(path, "//./pipe/");
> 	}
> 
> Not a suggestion to add such a thing; as long as we know there is no
> other codepath that would benefit from having one, a generalization
> like that can and should wait.

Yeah, I don't think we need something that general just yet.  Named
pipes exist in a special namespace using the UNC/network-share syntax
(rather than the DOS drive-letter syntax), and we don't do much with
UNC paths yet.

Perhaps, later we could have something to try splitting a UNC
path into <server>, <volume>, and <relative-path> and then have
is_local_named_pipe() verify the first 2 are as expected.
Or have a function like is_path_in_volume(path, server, volume)
that does the tests without cutting up strings and allocating.
We could do either, but I don't think we need to be that general
yet.

Thanks,
Jeff

Patch

diff --git a/compat/mingw.c b/compat/mingw.c
index 858ca14a57..f87376b26a 100644
--- a/compat/mingw.c
+++ b/compat/mingw.c
@@ -341,6 +341,19 @@  int mingw_mkdir(const char *path, int mode)
 	return ret;
 }
 
+/*
+ * Calling CreateFile() using FILE_APPEND_DATA and without FILE_WRITE_DATA
+ * is documented in [1] as opening a writable file handle in append mode.
+ * (It is believed that) this is atomic since it is maintained by the
+ * kernel unlike the O_APPEND flag which is racily maintained by the CRT.
+ *
+ * [1] https://docs.microsoft.com/en-us/windows/desktop/fileio/file-access-rights-constants
+ *
+ * This trick does not appear to work for named pipes.  Instead it creates
+ * a named pipe client handle that cannot be written to.  Callers should
+ * just use the regular _wopen() for them.  (And since client handle gets
+ * bound to a unique server handle, it isn't really an issue.)
+ */
 static int mingw_open_append(wchar_t const *wfilename, int oflags, ...)
 {
 	HANDLE handle;
@@ -360,10 +373,12 @@  static int mingw_open_append(wchar_t const *wfilename, int oflags, ...)
 			NULL, create, FILE_ATTRIBUTE_NORMAL, NULL);
 	if (handle == INVALID_HANDLE_VALUE)
 		return errno = err_win_to_posix(GetLastError()), -1;
+
 	/*
 	 * No O_APPEND here, because the CRT uses it only to reset the
-	 * file pointer to EOF on write(); but that is not necessary
-	 * for a file created with FILE_APPEND_DATA.
+	 * file pointer to EOF before each write(); but that is not
+	 * necessary (and may lead to races) for a file created with
+	 * FILE_APPEND_DATA.
 	 */
 	fd = _open_osfhandle((intptr_t)handle, O_BINARY);
 	if (fd < 0)
@@ -371,6 +386,23 @@  static int mingw_open_append(wchar_t const *wfilename, int oflags, ...)
 	return fd;
 }
 
+#define IS_SBS(ch) (((ch) == '/') || ((ch) == '\\'))
+/*
+ * Does the pathname map to the local named pipe filesystem?
+ * That is, does it have a "//./pipe/" prefix?
+ */
+static int mingw_is_local_named_pipe_path(const char *filename)
+{
+	return (IS_SBS(filename[0]) &&
+		IS_SBS(filename[1]) &&
+		filename[2] == '.'  &&
+		IS_SBS(filename[3]) &&
+		!strncasecmp(filename+4, "pipe", 4) &&
+		IS_SBS(filename[8]) &&
+		filename[9]);
+}
+#undef IS_SBS
+
 int mingw_open (const char *filename, int oflags, ...)
 {
 	typedef int (*open_fn_t)(wchar_t const *wfilename, int oflags, ...);
@@ -387,7 +419,7 @@  int mingw_open (const char *filename, int oflags, ...)
 	if (filename && !strcmp(filename, "/dev/null"))
 		filename = "nul";
 
-	if (oflags & O_APPEND)
+	if ((oflags & O_APPEND) && !mingw_is_local_named_pipe_path(filename))
 		open_fn = mingw_open_append;
 	else
 		open_fn = _wopen;
diff --git a/t/t0051-windows-named-pipe.sh b/t/t0051-windows-named-pipe.sh
index e3c36341a0..10ac92d225 100755
--- a/t/t0051-windows-named-pipe.sh
+++ b/t/t0051-windows-named-pipe.sh
@@ -4,7 +4,7 @@  test_description='Windows named pipes'
 
 . ./test-lib.sh
 
-test_expect_failure MINGW 'o_append write to named pipe' '
+test_expect_success MINGW 'o_append write to named pipe' '
 	GIT_TRACE="$(pwd)/expect" git status >/dev/null 2>&1 &&
 	{ test-tool windows-named-pipe t0051 >actual 2>&1 & } &&
 	pid=$! &&