Get SubViewPort in to the compositor

jovlem

Hello,

I'm making a compositor effect.
I want / need to use a subviewport that renders everything on cull mask layer 6.
In the compositor effect that will be then used as a mask.

But I just can't get this part working.
I tried a few things, like sending it as a "uniform image2D" or as a "uniform sampler2D".
But no matter what, I always did get a black screen.

So maybe one of the options actually did work, but I get a black screen because the subviewport is not rendered yet when it reaches the compositor effect.
I'm not saying this is the case. Maybe ( probably ) it's just me doing something wrong.
But at this point, I tried so many things that I'm stuck.

Also I use GDScript. Can it work with GDScript as I don't know how to code in C ( and also don't want to switch )?

Below the script, and at the bottom a simple compute shader just for testing.
Also look at the things that are commented. These I tried.
EDIT: I can't get to insert the code correctly. So it's a bit messy, sorry. ( Maybe a mod can fix it? )

`@tool
extends CompositorEffect
class_name ScreenSpaceMeshBlend

var rd: RenderingDevice
var shader: RID
var pipeline: RID

var subViewport: Object = null

var samplerRID: RID

func _init():
effect_callback_type = EFFECT_CALLBACK_TYPE_POST_TRANSPARENT
enabled = true

func _render_callback(p_type, p_render_data):
rd = RenderingServer.get_rendering_device()
if !rd: return

if not shader.is_valid():
	var shaderFile: RDShaderFile = load("res://materials/shaders/screen_space_mesh_blend_A.glsl")
	var shaderSpirv: RDShaderSPIRV = shaderFile.get_spirv()
	shader = rd.shader_create_from_spirv(shaderSpirv)
	pipeline = rd.compute_pipeline_create(shader)

var render_scene_buffers = p_render_data.get_render_scene_buffers()
var render_size: Vector2i = render_scene_buffers.get_internal_size()

if !render_scene_buffers: return

var screenTextureRID: RID = render_scene_buffers.get_color_layer(0)
var uniformScreenTexture: RDUniform = RDUniform.new()
uniformScreenTexture.uniform_type = RenderingDevice.UNIFORM_TYPE_IMAGE
uniformScreenTexture.binding = 0
uniformScreenTexture.add_id(screenTextureRID)


if subViewport == null:
	subViewport = register._get_node_by_id("MaskViewport")

if !subViewport: return

var tex = subViewport.get_texture()
var texRID: RID = tex.get_rid()

#var subViewportRID: RID = subViewport.get_viewport_rid()
#var renderServerTextureRID: RID = RenderingServer.viewport_get_texture(subViewportRID)
var subViewportTextureRID: RID = RenderingServer.texture_get_rd_texture(texRID, 0)

if !samplerRID.is_valid():
	var sampler_state: RDSamplerState = RDSamplerState.new()
	sampler_state.mag_filter = RenderingDevice.SAMPLER_FILTER_LINEAR
	sampler_state.min_filter = RenderingDevice.SAMPLER_FILTER_LINEAR
	samplerRID = rd.sampler_create(sampler_state)


var uniformSubViewportTexture: RDUniform = RDUniform.new()
uniformSubViewportTexture.uniform_type = RenderingDevice.UNIFORM_TYPE_IMAGE
#uniformSubViewportTexture.uniform_type = RenderingDevice.UNIFORM_TYPE_SAMPLER_WITH_TEXTURE
uniformSubViewportTexture.binding = 1

#uniformSubViewportTexture.add_id(samplerRID)
uniformSubViewportTexture.add_id(subViewportTextureRID)

var uniform_set = rd.uniform_set_create([uniformScreenTexture, uniformSubViewportTexture], shader, 0)

var x_groups: int = ceil(render_size.x / 8.0)
var y_groups: int = ceil(render_size.y / 8.0)
var compute_list = rd.compute_list_begin()
rd.compute_list_bind_compute_pipeline(compute_list, pipeline)
rd.compute_list_bind_uniform_set(compute_list, uniform_set, 0)
rd.compute_list_dispatch(compute_list, x_groups, y_groups, 1)
rd.compute_list_end()

rd.free_rid(uniform_set)`

`#[compute]
#version 450

layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in;

layout(rgba8, set = 0, binding = 0) uniform image2D screen_image;

//layout(set = 0, binding = 1) uniform sampler2D subViewport_ID;
layout(rgba8, set = 0, binding = 1) uniform image2D subViewport_ID;

void main() {
ivec2 pixel_coords = ivec2(gl_GlobalInvocationID.xy);
ivec2 size = imageSize(screen_image);

if (pixel_coords.x >= size.x || pixel_coords.y >= size.y) {
	return;
}

//needed if I use uniform sampler2D
//vec2 uv = (vec2(pixel_coords) + 0.5) / vec2(size); 

vec4 mask_color = imageLoad(subViewport_ID, pixel_coords);

imageStore(screen_image, pixel_coords, vec4(mask_color.rgb, 1.0));