krnl386: Properly handle failure to set a 16-bit LDT entry.

dlls/krnl386.exe16/kernel.c

@@ -71,7 +71,7 @@ BOOL WINAPI DllMain( HINSTANCE hinst, DWORD reason, LPVOID reserved )
     switch(reason)
     {
     case DLL_PROCESS_ATTACH:
-        LoadLibrary16( "krnl386.exe" );
+        if (LoadLibrary16( "krnl386.exe" ) < 32) return FALSE;
         /* fall through */
     case DLL_THREAD_ATTACH:
         thread_attach();

dlls/krnl386.exe16/ne_module.c

@@ -43,6 +43,7 @@
 WINE_DEFAULT_DEBUG_CHANNEL(module);
 WINE_DECLARE_DEBUG_CHANNEL(loaddll);
 WINE_DECLARE_DEBUG_CHANNEL(relay);
+WINE_DECLARE_DEBUG_CHANNEL(winediag);
 
 #include "pshpack1.h"
 typedef struct _GPHANDLERDEF
@@ -623,7 +624,12 @@ static HMODULE16 build_module( const void *mapping, SIZE_T mapping_size, LPCSTR
            sizeof(OFSTRUCT) - sizeof(ofs->szPathName) + strlen(path) + 1;
 
     hModule = GlobalAlloc16( GMEM_FIXED | GMEM_ZEROINIT, size );
-    if (!hModule) return ERROR_BAD_FORMAT;
+    if (!hModule)
+    {
+        ERR_(winediag)( "Failed to create module for %s, 16-bit LDT support may be missing.\n",
+                        debugstr_a(path) );
+        return ERROR_BAD_FORMAT;
+    }
 
     FarSetOwner16( hModule, hModule );
     pModule = GlobalLock16( hModule );
@@ -1035,6 +1041,7 @@ static HINSTANCE16 MODULE_LoadModule16( LPCSTR libname, BOOL implicit, BOOL lib_
         TRACE("Trying built-in '%s'\n", libname);
         hinst = NE_DoLoadBuiltinModule( descr, file_name, mod32 );
         if (hinst > 32) TRACE_(loaddll)("Loaded module %s : builtin\n", debugstr_a(file_name));
+        else FreeLibrary( mod32 );
     }
     else
     {

dlls/krnl386.exe16/selector.c

@@ -52,7 +52,14 @@ WORD WINAPI AllocSelectorArray16( WORD count )
         wine_ldt_set_base( &entry, 0 );
         wine_ldt_set_limit( &entry, 1 ); /* avoid 0 base and limit */
         wine_ldt_set_flags( &entry, WINE_LDT_FLAGS_DATA );
-        for (i = 0; i < count; i++) wine_ldt_set_entry( sel + (i << __AHSHIFT), &entry );
+        for (i = 0; i < count; i++)
+        {
+            if (wine_ldt_set_entry( sel + (i << __AHSHIFT), &entry ) < 0)
+            {
+                wine_ldt_free_entries( sel, count );
+                return 0;
+            }
+        }
     }
     return sel;
 }
@@ -102,7 +109,7 @@ WORD WINAPI FreeSelector16( WORD sel )
  *
  * Set the LDT entries for an array of selectors.
  */
-static void SELECTOR_SetEntries( WORD sel, const void *base, DWORD size, unsigned char flags )
+static BOOL SELECTOR_SetEntries( WORD sel, const void *base, DWORD size, unsigned char flags )
 {
     LDT_ENTRY entry;
     WORD i, count;
@@ -113,11 +120,12 @@ static void SELECTOR_SetEntries( WORD sel, const void *base, DWORD size, unsigne
     count = (size + 0xffff) / 0x10000;
     for (i = 0; i < count; i++)
     {
-        wine_ldt_set_entry( sel + (i << __AHSHIFT), &entry );
+        if (wine_ldt_set_entry( sel + (i << __AHSHIFT), &entry ) < 0) return FALSE;
         wine_ldt_set_base( &entry, (char*)wine_ldt_get_base(&entry) + 0x10000);
         /* yep, Windows sets limit like that, not 64K sel units */
         wine_ldt_set_limit( &entry, wine_ldt_get_limit(&entry) - 0x10000 );
     }
+    return TRUE;
 }
 
 
@@ -132,8 +140,12 @@ WORD SELECTOR_AllocBlock( const void *base, DWORD size, unsigned char flags )
 
     if (!size) return 0;
     count = (size + 0xffff) / 0x10000;
-    sel = wine_ldt_alloc_entries( count );
+    if ((sel = wine_ldt_alloc_entries( count )))
-    if (sel) SELECTOR_SetEntries( sel, base, size, flags );
+    {
+        if (SELECTOR_SetEntries( sel, base, size, flags )) return sel;
+        wine_ldt_free_entries( sel, count );
+        sel = 0;
+    }
     return sel;
 }
 
@@ -202,8 +214,9 @@ WORD WINAPI AllocCStoDSAlias16( WORD sel )
     if (!newsel) return 0;
     wine_ldt_get_entry( sel, &entry );
     entry.HighWord.Bits.Type = WINE_LDT_FLAGS_DATA;
-    wine_ldt_set_entry( newsel, &entry );
+    if (wine_ldt_set_entry( newsel, &entry ) >= 0) return newsel;
-    return newsel;
+    wine_ldt_free_entries( newsel, 1 );
+    return 0;
 }
 
 
@@ -221,8 +234,9 @@ WORD WINAPI AllocDStoCSAlias16( WORD sel )
     if (!newsel) return 0;
     wine_ldt_get_entry( sel, &entry );
     entry.HighWord.Bits.Type = WINE_LDT_FLAGS_CODE;
-    wine_ldt_set_entry( newsel, &entry );
+    if (wine_ldt_set_entry( newsel, &entry ) >= 0) return newsel;
-    return newsel;
+    wine_ldt_free_entries( newsel, 1 );
+    return 0;
 }
 
 
@@ -260,7 +274,7 @@ WORD WINAPI SetSelectorBase( WORD sel, DWORD base )
     LDT_ENTRY entry;
     wine_ldt_get_entry( sel, &entry );
     wine_ldt_set_base( &entry, DOSMEM_MapDosToLinear(base) );
-    wine_ldt_set_entry( sel, &entry );
+    if (wine_ldt_set_entry( sel, &entry ) < 0) sel = 0;
     return sel;
 }
 
@@ -282,7 +296,7 @@ WORD WINAPI SetSelectorLimit16( WORD sel, DWORD limit )
     LDT_ENTRY entry;
     wine_ldt_get_entry( sel, &entry );
     wine_ldt_set_limit( &entry, limit );
-    wine_ldt_set_entry( sel, &entry );
+    if (wine_ldt_set_entry( sel, &entry ) < 0) sel = 0;
     return sel;
 }