Henri Verbeet : vkd3d-shader/tpf: Do not write RDEF constant buffer entries for HLSL_CLASS_OBJECT variables.

[Alexandre Julliard]

Module: vkd3d
Branch: master
Commit: aa5380f32a89e5dd849c4b06ec99b36ab9f87cd5
URL:    https://gitlab.winehq.org/wine/vkd3d/-/commit/aa5380f32a89e5dd849c4b06ec99b3...

Author: Henri Verbeet 
Date:   Mon Nov  6 20:03:21 2023 +0100

vkd3d-shader/tpf: Do not write RDEF constant buffer entries for HLSL_CLASS_OBJECT variables.

RWBuffer objects would trigger a vkd3d_unreachable() in sm4_base_type().
It would be easy enough to add the required case there, but (manual,
unfortunately) tests show that we aren't supposed to write constant
buffer entries for objects in the first place, as you'd expect.

This particular path ends up being exercised by vkd3d's internal UAV
clear shaders, but unfortunately it looks like our RDEF data may have
more issues; the ability to write tests for it would seem helpful.

---

libs/vkd3d-shader/tpf.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libs/vkd3d-shader/tpf.c b/libs/vkd3d-shader/tpf.c
index 8ae1657d..fbc04f61 100644
--- a/libs/vkd3d-shader/tpf.c
+++ b/libs/vkd3d-shader/tpf.c
@@ -3460,7 +3460,8 @@ static void write_sm4_rdef(struct hlsl_ctx *ctx, struct dxbc_writer *dxbc)

LIST_FOR_EACH_ENTRY(var, &ctx->extern_vars, struct hlsl_ir_var, extern_entry)
         {
-            if (var->is_uniform && var->buffer == cbuffer)
+            if (var->is_uniform && var->buffer == cbuffer
+                    && var->data_type->class != HLSL_CLASS_OBJECT)
                 ++var_count;
         }

@@ -3494,7 +3495,8 @@ static void write_sm4_rdef(struct hlsl_ctx *ctx, struct dxbc_writer *dxbc)

LIST_FOR_EACH_ENTRY(var, &ctx->extern_vars, struct hlsl_ir_var, extern_entry)
         {
-            if (var->is_uniform && var->buffer == cbuffer)
+            if (var->is_uniform && var->buffer == cbuffer
+                    && var->data_type->class != HLSL_CLASS_OBJECT)
             {
                 uint32_t flags = 0;

@@ -3521,7 +3523,8 @@ static void write_sm4_rdef(struct hlsl_ctx *ctx, struct dxbc_writer *dxbc)
         j = 0;
         LIST_FOR_EACH_ENTRY(var, &ctx->extern_vars, struct hlsl_ir_var, extern_entry)
         {
-            if (var->is_uniform && var->buffer == cbuffer)
+            if (var->is_uniform && var->buffer == cbuffer
+                    && var->data_type->class != HLSL_CLASS_OBJECT)
             {
                 const unsigned int var_size = (profile->major_version >= 5 ? 10 : 6);
                 size_t var_offset = vars_start + j * var_size * sizeof(uint32_t);