wined3d: Support clipplanes with GLSL.

This is the Nth attemt to make clipping work with GLSL shaders. The patch now
uses the GLSL quirk table to handle cards that need a custom varying for
gl_ClipPos, and the code is adapted to the changed state table and shader
backend system.

dlls/wined3d/arb_program_shader.c

@@ -2082,6 +2082,8 @@ static void shader_arb_get_caps(WINED3DDEVTYPE devtype, const WineD3D_GL_Info *g
         TRACE_(d3d_caps)("Hardware pixel shader version 1.4 enabled (ARB_PROGRAM)\n");
         pCaps->MaxPixelShaderConst = GL_LIMITS(pshader_constantsF) - ARB_SHADER_RESERVED_PS_CONSTS;
     }
+
+    pCaps->VSClipping = FALSE; /* TODO: GL_NV_vertex_program2_option provides this */
 }
 
 static BOOL shader_arb_color_fixup_supported(struct color_fixup_desc fixup)

dlls/wined3d/directx.c

@@ -3799,6 +3799,7 @@ static HRESULT WINAPI IWineD3DImpl_CreateDevice(IWineD3D *iface, UINT Adapter,
     object->shader_backend->shader_get_caps(DeviceType, &adapter->gl_info, &shader_caps);
     object->d3d_vshader_constantF = shader_caps.MaxVertexShaderConst;
     object->d3d_pshader_constantF = shader_caps.MaxPixelShaderConst;
+    object->vs_clipping = shader_caps.VSClipping;
 
     memset(&ffp_caps, 0, sizeof(ffp_caps));
     frag_pipeline = select_fragment_implementation(adapter, DeviceType);
@@ -4058,6 +4059,18 @@ static BOOL match_fglrx(const WineD3D_GL_Info *gl_info) {
     return TRUE;
 }
 
+static BOOL match_dx10_capable(const WineD3D_GL_Info *gl_info) {
+    /* DX9 cards support 40 single float varyings in hardware, most drivers report 32. ATI misreports
+     * 44 varyings. So assume that if we have more than 44 varyings we have a dx10 card.
+     * This detection is for the gl_ClipPos varying quirk. If a d3d9 card really supports more than 44
+     * varyings and we subtract one in dx9 shaders its not going to hurt us because the dx9 limit is
+     * hardcoded
+     *
+     * dx10 cards usually have 64 varyings
+     */
+    return gl_info->max_glsl_varyings > 44;
+}
+
 static void quirk_arb_constants(WineD3D_GL_Info *gl_info) {
     TRACE_(d3d_caps)("Using ARB vs constant limit(=%u) for GLSL\n", gl_info->vs_arb_constantsF);
     gl_info->vs_glsl_constantsF = gl_info->vs_arb_constantsF;
@@ -4161,6 +4174,10 @@ static void quirk_texcoord_w(WineD3D_GL_Info *gl_info) {
     gl_info->set_texcoord_w = TRUE;
 }
 
+static void quirk_clip_varying(WineD3D_GL_Info *gl_info) {
+    gl_info->glsl_clip_varying = TRUE;
+}
+
 struct driver_quirk quirk_table[] = {
     {
         match_ati_r300_to_500,
@@ -4198,6 +4215,11 @@ struct driver_quirk quirk_table[] = {
         match_fglrx,
         quirk_one_point_sprite,
         "Fglrx point sprite crash workaround"
+    },
+    {
+        match_dx10_capable,
+        quirk_clip_varying,
+        "Reserved varying for gl_ClipPos"
     }
 };
 

dlls/wined3d/glsl_shader.c

@@ -704,6 +704,17 @@ static void shader_glsl_update_float_pixel_constants(IWineD3DDevice *iface, UINT
     }
 }
 
+static int vec4_varyings(DWORD shader_major, const WineD3D_GL_Info *gl_info)
+{
+    int ret = GL_LIMITS(glsl_varyings) / 4;
+    /* 4.0 shaders do not write clip coords because d3d10 does not support user clipplanes */
+    if(shader_major > 3) return ret;
+
+    /* 3.0 shaders may need an extra varying for the clip coord on some cards(mostly dx10 ones) */
+    if(gl_info->glsl_clip_varying) ret -= 1;
+    return ret;
+}
+
 /** Generate the variable & register declarations for the GLSL output target */
 static void shader_generate_glsl_declarations(IWineD3DBaseShader *iface, const shader_reg_maps *reg_maps,
         SHADER_BUFFER *buffer, const WineD3D_GL_Info *gl_info,
@@ -748,9 +759,11 @@ static void shader_generate_glsl_declarations(IWineD3DBaseShader *iface, const s
                 /* Subtract the other potential uniforms from the max available (bools, ints, and 1 row of projection matrix).
                  * Subtract another uniform for immediate values, which have to be loaded via uniform by the driver as well.
                  * The shader code only uses 0.5, 2.0, 1.0, 128 and -128 in vertex shader code, so one vec4 should be enough
-                 * (Unfortunately the Nvidia driver doesn't store 128 and -128 in one float
+                 * (Unfortunately the Nvidia driver doesn't store 128 and -128 in one float).
+                 *
+                 * Writing gl_ClipPos requires one uniform for each clipplane as well.
                  */
-                max_constantsF = GL_LIMITS(vshader_constantsF) - 3;
+                max_constantsF = GL_LIMITS(vshader_constantsF) - 3 - GL_LIMITS(clipplanes);
                 max_constantsF -= count_bits(This->baseShader.reg_maps.integer_constants);
                 /* Strictly speaking a bool only uses one scalar, but the nvidia(Linux) compiler doesn't pack them properly,
                  * so each scalar requires a full vec4. We could work around this by packing the booleans ourselves, but
@@ -902,13 +915,13 @@ static void shader_generate_glsl_declarations(IWineD3DBaseShader *iface, const s
     {
         if (use_vs(device->stateBlock))
         {
-            shader_addline(buffer, "varying vec4 IN[%u];\n", GL_LIMITS(glsl_varyings) / 4);
+            shader_addline(buffer, "varying vec4 IN[%u];\n", vec4_varyings(reg_maps->shader_version.major, gl_info));
         } else {
             /* TODO: Write a replacement shader for the fixed function vertex pipeline, so this isn't needed.
              * For fixed function vertex processing + 3.0 pixel shader we need a separate function in the
              * pixel shader that reads the fixed function color into the packed input registers.
              */
-            shader_addline(buffer, "vec4 IN[%u];\n", GL_LIMITS(glsl_varyings) / 4);
+            shader_addline(buffer, "vec4 IN[%u];\n", vec4_varyings(reg_maps->shader_version.major, gl_info));
         }
     }
 
@@ -1085,7 +1098,7 @@ static void shader_glsl_get_register_name(const struct wined3d_shader_register *
             if (This->baseShader.reg_maps.shader_version.major >= 3)
             {
                 DWORD idx = ((IWineD3DPixelShaderImpl *)This)->input_reg_map[reg->idx];
-                DWORD in_count = GL_LIMITS(glsl_varyings) / 4;
+                DWORD in_count = vec4_varyings(This->baseShader.reg_maps.shader_version.major, gl_info);
 
                 if (reg->rel_addr)
                 {
@@ -3237,7 +3250,7 @@ static void handle_ps3_input(SHADER_BUFFER *buffer, const WineD3D_GL_Info *gl_in
     DWORD usage, usage_idx, usage_out, usage_idx_out;
     DWORD *set;
     DWORD in_idx;
-    DWORD in_count = GL_LIMITS(glsl_varyings) / 4;
+    DWORD in_count = vec4_varyings(3, gl_info);
     char reg_mask[6], reg_mask_out[6];
     char destination[50];
 
@@ -3467,7 +3480,7 @@ static GLhandleARB generate_param_reorder_function(IWineD3DVertexShader *vertexs
         semantics_out = vs->semantics_out;
 
         /* This one is tricky: a 3.0 pixel shader reads from a 3.0 vertex shader */
-        shader_addline(&buffer, "varying vec4 IN[%u];\n", GL_LIMITS(glsl_varyings) / 4);
+        shader_addline(&buffer, "varying vec4 IN[%u];\n", vec4_varyings(3, gl_info));
         shader_addline(&buffer, "void order_ps_input(in vec4 OUT[%u]) {\n", MAX_REG_OUTPUT);
 
         /* First, sort out position and point size. Those are not passed to the pixel shader */
@@ -3498,7 +3511,7 @@ static GLhandleARB generate_param_reorder_function(IWineD3DVertexShader *vertexs
 
         shader_addline(&buffer, "}\n");
     } else if(ps_major >= 3 && vs_major < 3) {
-        shader_addline(&buffer, "varying vec4 IN[%u];\n", GL_LIMITS(glsl_varyings) / 4);
+        shader_addline(&buffer, "varying vec4 IN[%u];\n", vec4_varyings(3, gl_info));
         shader_addline(&buffer, "void order_ps_input() {\n");
         /* The vertex shader wrote to the builtin varyings. There is no need to figure out position and
          * point size, but we depend on the optimizers kindness to find out that the pixel shader doesn't
@@ -3713,7 +3726,7 @@ static void set_glsl_shader_program(IWineD3DDevice *iface, BOOL use_ps, BOOL use
 
     if (pshader
             && ((IWineD3DPixelShaderImpl *)pshader)->baseShader.reg_maps.shader_version.major >= 3
-            && ((IWineD3DPixelShaderImpl *)pshader)->declared_in_count > GL_LIMITS(glsl_varyings) / 4)
+            && ((IWineD3DPixelShaderImpl *)pshader)->declared_in_count > vec4_varyings(3, gl_info))
     {
         TRACE("Shader %d needs vertex color clamping disabled\n", programId);
         entry->vertex_color_clamp = GL_FALSE;
@@ -4242,6 +4255,7 @@ static GLuint shader_glsl_generate_vshader(IWineD3DVertexShader *iface,
      */
     shader_addline(buffer, "gl_Position.y = gl_Position.y * posFixup.y;\n");
     shader_addline(buffer, "gl_Position.xy += posFixup.zw * gl_Position.ww;\n");
+    shader_addline(buffer, "gl_ClipVertex = gl_Position;\n");
 
     /* Z coord [0;1]->[-1;1] mapping, see comment in transform_projection in state.c
      *
@@ -4310,6 +4324,8 @@ static void shader_glsl_get_caps(WINED3DDEVTYPE devtype, const WineD3D_GL_Info *
      */
     pCaps->PixelShader1xMaxValue = 8.0;
     TRACE_(d3d_caps)("Hardware pixel shader version %d.%d enabled (GLSL)\n", (pCaps->PixelShaderVersion >> 8) & 0xff, pCaps->PixelShaderVersion & 0xff);
+
+    pCaps->VSClipping = TRUE;
 }
 
 static BOOL shader_glsl_color_fixup_supported(struct color_fixup_desc fixup)

dlls/wined3d/state.c

@@ -525,7 +525,7 @@ static void state_clipping(DWORD state, IWineD3DStateBlockImpl *stateblock, Wine
     DWORD enable  = 0xFFFFFFFF;
     DWORD disable = 0x00000000;
 
-    if (use_vs(stateblock))
+    if (!stateblock->wineD3DDevice->vs_clipping && use_vs(stateblock))
     {
         /* The spec says that opengl clipping planes are disabled when using shaders. Direct3D planes aren't,
          * so that is an issue. The MacOS ATI driver keeps clipping planes activated with shaders in some
@@ -3496,9 +3496,22 @@ static void clipplane(DWORD state, IWineD3DStateBlockImpl *stateblock, WineD3DCo
     }
 
     /* Clip Plane settings are affected by the model view in OpenGL, the View transform in direct3d */
-    glMatrixMode(GL_MODELVIEW);
+    if(!use_vs(stateblock)) {
-    glPushMatrix();
+        glMatrixMode(GL_MODELVIEW);
-    glLoadMatrixf(&stateblock->transforms[WINED3DTS_VIEW].u.m[0][0]);
+        glPushMatrix();
+        glLoadMatrixf(&stateblock->transforms[WINED3DTS_VIEW].u.m[0][0]);
+    } else {
+        /* with vertex shaders, clip planes are not transformed in direct3d,
+         * in OpenGL they are still transformed by the model view.
+         * Use this to swap the y coordinate if necessary
+         */
+        glMatrixMode(GL_MODELVIEW);
+        glPushMatrix();
+        glLoadIdentity();
+        if(stateblock->wineD3DDevice->render_offscreen) {
+            glScalef(1.0, -1.0, 1.0);
+        }
+    }
 
     TRACE("Clipplane [%f,%f,%f,%f]\n",
           stateblock->clipplane[index][0],
@@ -4403,7 +4416,7 @@ static void vertexdeclaration(DWORD state, IWineD3DStateBlockImpl *stateblock, W
 
         if(context->last_was_vshader) {
             updateFog = TRUE;
-            if(!isStateDirty(context, STATE_RENDER(WINED3DRS_CLIPPLANEENABLE))) {
+            if(!device->vs_clipping && !isStateDirty(context, STATE_RENDER(WINED3DRS_CLIPPLANEENABLE))) {
                 state_clipping(STATE_RENDER(WINED3DRS_CLIPPLANEENABLE), stateblock, context);
             }
         }
@@ -4414,17 +4427,19 @@ static void vertexdeclaration(DWORD state, IWineD3DStateBlockImpl *stateblock, W
         if(!context->last_was_vshader) {
             unsigned int i;
             static BOOL warned = FALSE;
-            /* Disable all clip planes to get defined results on all drivers. See comment in the
+            if(!device->vs_clipping) {
-             * state_clipping state handler
+                /* Disable all clip planes to get defined results on all drivers. See comment in the
-             */
+                 * state_clipping state handler
-            for(i = 0; i < GL_LIMITS(clipplanes); i++) {
+                 */
-                glDisable(GL_CLIP_PLANE0 + i);
+                for(i = 0; i < GL_LIMITS(clipplanes); i++) {
-                checkGLcall("glDisable(GL_CLIP_PLANE0 + i)");
+                    glDisable(GL_CLIP_PLANE0 + i);
-            }
+                    checkGLcall("glDisable(GL_CLIP_PLANE0 + i)");
+                }
 
-            if(!warned && stateblock->renderState[WINED3DRS_CLIPPLANEENABLE]) {
+                if(!warned && stateblock->renderState[WINED3DRS_CLIPPLANEENABLE]) {
-                FIXME("Clipping not supported with vertex shaders\n");
+                    FIXME("Clipping not supported with vertex shaders\n");
-                warned = TRUE;
+                    warned = TRUE;
+                }
             }
             if(wasrhw) {
                 /* Apply the transform matrices when switching from rhw drawing to vertex shaders. Vertex

dlls/wined3d/wined3d_gl.h

@@ -3962,6 +3962,7 @@ typedef struct _WineD3D_GL_Info {
   BOOL arb_vs_offset_limit;
   BOOL set_texcoord_w;
   DWORD reserved_glsl_constants;
+  BOOL glsl_clip_varying;
 
   BOOL supported[OPENGL_SUPPORTED_EXT_END + 1];
 

dlls/wined3d/wined3d_private.h

@@ -730,6 +730,8 @@ struct shader_caps {
     DWORD               MaxPShaderInstructionsExecuted;
     DWORD               MaxVertexShader30InstructionSlots;
     DWORD               MaxPixelShader30InstructionSlots;
+
+    BOOL                VSClipping;
 };
 
 enum tex_types
@@ -1461,6 +1463,7 @@ struct IWineD3DDeviceImpl
 
     unsigned int max_ffp_textures, max_ffp_texture_stages;
     DWORD d3d_vshader_constantF, d3d_pshader_constantF; /* Advertised d3d caps, not GL ones */
+    DWORD vs_clipping;
 
     WORD view_ident : 1;                /* true iff view matrix is identity */
     WORD untransformed : 1;