msvcrt: Return error on invalid handle in _open_osfhandle.

dlls/msvcrt/file.c

@@ -2240,6 +2240,7 @@ int CDECL MSVCRT__wcreat(const MSVCRT_wchar_t *path, int flags)
  */
 int CDECL MSVCRT__open_osfhandle(MSVCRT_intptr_t handle, int oflags)
 {
+  DWORD flags;
   int fd;
 
   /* MSVCRT__O_RDONLY (0) always matches, so set the read flag
@@ -2251,8 +2252,23 @@ int CDECL MSVCRT__open_osfhandle(MSVCRT_intptr_t handle, int oflags)
   if (!(oflags & (MSVCRT__O_BINARY | MSVCRT__O_TEXT)))
       oflags |= MSVCRT__O_BINARY;
 
-  fd = msvcrt_alloc_fd((HANDLE)handle, split_oflags(oflags));
+  flags = GetFileType((HANDLE)handle);
-  TRACE(":handle (%ld) fd (%d) flags 0x%08x\n", handle, fd, oflags);
+  if (flags==FILE_TYPE_UNKNOWN && GetLastError()!=NO_ERROR)
+  {
+    msvcrt_set_errno(GetLastError());
+    return -1;
+  }
+
+  if (flags == FILE_TYPE_CHAR)
+    flags = WX_NOSEEK;
+  else if (flags == FILE_TYPE_PIPE)
+    flags = WX_PIPE;
+  else
+    flags = 0;
+  flags |= split_oflags(oflags);
+
+  fd = msvcrt_alloc_fd((HANDLE)handle, flags);
+  TRACE(":handle (%ld) fd (%d) flags 0x%08x\n", handle, fd, flags);
   return fd;
 }
 

dlls/msvcrt/tests/file.c

@@ -35,6 +35,16 @@
 #include <errno.h>
 #include <locale.h>
 
+#define MSVCRT_FD_BLOCK_SIZE 32
+typedef struct {
+    HANDLE              handle;
+    unsigned char       wxflag;
+    char                lookahead[3];
+    int                 exflag;
+    CRITICAL_SECTION    crit;
+} ioinfo;
+static ioinfo **__pioinfo;
+
 static HANDLE proc_handles[2];
 
 static int (__cdecl *p_fopen_s)(FILE**, const char*, const char*);
@@ -60,6 +70,7 @@ static void init(void)
 
     p_fopen_s = (void*)GetProcAddress(hmod, "fopen_s");
     p__wfopen_s = (void*)GetProcAddress(hmod, "_wfopen_s");
+    __pioinfo = (void*)GetProcAddress(hmod, "__pioinfo");
 }
 
 static void test_filbuf( void )
@@ -2171,6 +2182,43 @@ static void test_mktemp(void)
     ok(_mktemp(buf) != NULL, "_mktemp(\"**XXXXXX\") == NULL\n");
 }
 
+static void test__open_osfhandle(void)
+{
+    ioinfo *info;
+    HANDLE h, tmp;
+    int fd;
+
+    errno = 0xdeadbeef;
+    fd = _open_osfhandle((intptr_t)INVALID_HANDLE_VALUE, 0);
+    ok(fd == -1, "_open_osfhandle returned %d\n", fd);
+    ok(errno == EBADF, "errno = %d\n", errno);
+
+    h = CreateFileA("open_osfhandle.tst", GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, 0, NULL);
+    fd = _open_osfhandle((intptr_t)h, 0);
+    ok(fd > 0, "_open_osfhandle returned %d (%d)\n", fd, errno);
+    info = &__pioinfo[fd/MSVCRT_FD_BLOCK_SIZE][fd%MSVCRT_FD_BLOCK_SIZE];
+    ok(info->handle == h, "info->handle = %p, expected %p\n", info->handle, h);
+    ok(info->wxflag == 1, "info->wxflag = %x, expected 1\n", info->wxflag);
+    close(fd);
+    ok(info->handle == INVALID_HANDLE_VALUE, "info->handle = %p, expected INVALID_HANDLE_VALUE\n", info->handle);
+    ok(info->wxflag == 0, "info->wxflag = %x, expected 0\n", info->wxflag);
+    DeleteFileA("open_osfhandle.tst");
+
+    errno = 0xdeadbeef;
+    fd = _open_osfhandle((intptr_t)h, 0);
+    ok(fd == -1, "_open_osfhandle returned %d\n", fd);
+    ok(errno == EBADF, "errno = %d\n", errno);
+
+    ok(CreatePipe(&h, &tmp, NULL, 0), "CreatePipe failed\n");
+    fd = _open_osfhandle((intptr_t)h, 0);
+    ok(fd > 0, "_open_osfhandle returned %d (%d)\n", fd, errno);
+    info = &__pioinfo[fd/MSVCRT_FD_BLOCK_SIZE][fd%MSVCRT_FD_BLOCK_SIZE];
+    ok(info->handle == h, "info->handle = %p, expected %p\n", info->handle, h);
+    ok(info->wxflag == 9, "info->wxflag = %x, expected 9\n", info->wxflag);
+    close(fd);
+    CloseHandle(tmp);
+}
+
 START_TEST(file)
 {
     int arg_c;
@@ -2235,6 +2283,7 @@ START_TEST(file)
     test_pipes(arg_v[0]);
     test_stdin();
     test_mktemp();
+    test__open_osfhandle();
 
     /* Wait for the (_P_NOWAIT) spawned processes to finish to make sure the report
      * file contains lines in the correct order