Transformation matrices in a shader

io · June 28, 2011, 2:05pm

Hi
something I don t completely understand about transformations in a shader, let s take for example the texture control fading shader where:

texture Src1 <string uiname="Source1";>;
sampler Src1Samp = sampler_state
{
    Texture   = (Src1);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
};

texture Src2 <string uiname="Source2";>;
sampler Src2Samp = sampler_state
{
    Texture   = (Src2);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
    AddressU = Border;
    AddressV = Border;
};

texture Mask<string uiname="Control";>;
sampler MaskSamp = sampler_state
{
    Texture   = (Mask);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
};

float4x4 tTex: TEXTUREMATRIX <string uiname="Source1 Transform";>;
float4x4 tTex2: TEXTUREMATRIX <string uiname="Source2 Transform";>;
float4x4 tMask: TEXTUREMATRIX <string uiname="Ctrl Transform";>;

source1: it is an input for a subpatch
source2: it is a texture from a 16:9 video
control: just a dynamic texture to control the fading

notice how I added the sampler state ¨border¨ to source 1 so that I can transform it and give it the right proportion (widescreen)

Now I can fix the proportions of source2 but source 1 is also cropped to the same transformation, that is: if I transform source2, source1 is cropped.

What I am trying to achieve is source2 to be transformed while source1 is not transformed.

tx

S.

tonfilm · June 28, 2011, 3:41pm

your transformation code is not listed here, but you need to sample each texture with its own texture coordinates… if you transform them in the vertex shader, which makes total sense, then you need to use as much texcoord channels as you have transformed coodinates. so add some channels to the vs2ps struct…

the border mode only tells the sampler how to sample the texture if its smaller than the view area.

io · June 28, 2011, 9:44pm

here is the code with the Vertex Shader…

float4x4 tW: WORLD;
float4x4 tV: VIEW;
float4x4 tP: PROJECTION;
float4x4 tWVP: WORLDVIEWPROJECTION;

texture Src1 <string uiname="Source1";>;
sampler Src1Samp = sampler_state
{
    Texture   = (Src1);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
};

texture Src2 <string uiname="Source2";>;
sampler Src2Samp = sampler_state
{
    Texture   = (Src2);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
    AddressU = Border;
    AddressV = Border;
};

texture Mask<string uiname="Control";>;
sampler MaskSamp = sampler_state
{
    Texture   = (Mask);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
};

float4x4 tTex: TEXTUREMATRIX <string uiname="Source1 Transform";>;
float4x4 tTex2: TEXTUREMATRIX <string uiname="Source2 Transform";>;
float4x4 tMask: TEXTUREMATRIX <string uiname="Ctrl Transform";>;

struct vs2ps
{
    float4 Pos  : POSITION;
    float2 TexCd : TEXCOORD0;
    float2 Tex2Cd : TEXCOORD1;
    float2 MaskCd : TEXCOORD2;
};

// --------------------------------------------------------------------------------------------------
// VERTEXSHADERS
// --------------------------------------------------------------------------------------------------
vs2ps VS(
    float4 PosO  : POSITION,
    float4 TexCd : TEXCOORD0,
    float2 Tex2Cd : TEXCOORD1,
    float2 MaskCd : TEXCOORD2)
{
    vs2ps Out = (vs2ps)0;
    Out.Pos = mul(PosO, tWVP);
    Out.TexCd = mul(TexCd, tTex);
    Out.Tex2Cd = mul(TexCd, tTex2);
    Out.MaskCd = mul(TexCd, tMask);
    return Out;
}

tonfilm · June 28, 2011, 11:29pm

ok, the VS inputs are float2, change them to float4 and it should work, thats a math thing, 2d vec * 4x4 matrix is no valid operation:

code(lang=hlsl):
vs2ps VS(
float4 PosO : POSITION,
float4 TexCd : TEXCOORD0,
float4 Tex2Cd : TEXCOORD1,
float4 MaskCd : TEXCOORD2)
{
vs2ps Out = (vs2ps)0;
Out.Pos = mul(PosO, tWVP);
Out.TexCd = mul(TexCd, tTex);
Out.Tex2Cd = mul(TexCd, tTex2);
Out.MaskCd = mul(TexCd, tMask);
return Out;
}

io · June 28, 2011, 11:57pm

Nop, I add the whole code…

float4x4 tW: WORLD;
float4x4 tV: VIEW;
float4x4 tP: PROJECTION;
float4x4 tWVP: WORLDVIEWPROJECTION;

texture Src1 <string uiname="Source1";>;
sampler Src1Samp = sampler_state
{
    Texture   = (Src1);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
};

texture Src2 <string uiname="Source2";>;
sampler Src2Samp = sampler_state
{
    Texture   = (Src2);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
    AddressU = Border;
    AddressV = Border;
};

texture Mask<string uiname="Control";>;
sampler MaskSamp = sampler_state
{
    Texture   = (Mask);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
};

float4x4 tTex: TEXTUREMATRIX <string uiname="Source1 Transform";>;
float4x4 tTex2: TEXTUREMATRIX <string uiname="Source2 Transform";>;
float4x4 tMask: TEXTUREMATRIX <string uiname="Ctrl Transform";>;

struct vs2ps
{
    float4 Pos  : POSITION;
    float4 TexCd : TEXCOORD0;
    float4 Tex2Cd : TEXCOORD1;
    float4 MaskCd : TEXCOORD2;
};

// --------------------------------------------------------------------------------------------------
// VERTEXSHADERS
// --------------------------------------------------------------------------------------------------
vs2ps VS(
    float4 PosO  : POSITION,
    float4 TexCd : TEXCOORD0,
    float4 Tex2Cd : TEXCOORD1,
    float4 MaskCd : TEXCOORD2)
{
    vs2ps Out = (vs2ps)0;
    Out.Pos = mul(PosO, tWVP);
    Out.TexCd = mul(TexCd, tTex);
    Out.Tex2Cd = mul(TexCd, tTex2);
    Out.MaskCd = mul(TexCd, tMask);
    return Out;
}

// --------------------------------------------------------------------------------------------------
// PIXELSHADERS:
// --------------------------------------------------------------------------------------------------

float4 psFade(vs2ps In): COLOR
{
    float4 sc1 = tex2D(Src1Samp, In.TexCd);
    float4 sc2 = tex2D(Src2Samp, In.Tex2Cd);
    float4 ctrl = tex2D(MaskSamp, In.MaskCd);
    float4 outColor = sc1 * (1-ctrl) + sc2 * (ctrl);
    return outColor;
}

float4 psinvFade(vs2ps In): COLOR
{
    float4 sc1 = tex2D(Src1Samp, In.TexCd);
    float4 sc2 = tex2D(Src2Samp, In.Tex2Cd);
    float4 ctrl = tex2D(MaskSamp, In.MaskCd);
    float4 outColor = sc1 * (ctrl) + sc2 * (1-ctrl);
    return outColor;
}

//show Source1 only
float4 psVS1(vs2ps In): COLOR
{
    float4 sc1 = tex2D(Src1Samp, In.TexCd);
    return sc1;
}

//show Source2 only
float4 psVS2(vs2ps In): COLOR
{
    float4 sc2 = tex2D(Src2Samp, In.Tex2Cd);
    return sc2;
}

//show control only
float4 psVSct(vs2ps In): COLOR
{
    float4 ctrl = tex2D(MaskSamp, In.MaskCd);
    return ctrl;
}

// --------------------------------------------------------------------------------------------------
// TECHNIQUES:
// --------------------------------------------------------------------------------------------------

technique Fade
{
    pass P0
    {
        VertexShader = compile vs_1_0 VS();
        PixelShader  = compile ps_1_0 psFade();
    }
}

technique invFade
{
    pass P0
    {
        VertexShader = compile vs_1_0 VS();
        PixelShader  = compile ps_1_0 psinvFade();
    }
}

technique ViewSource1
{
    pass P0
    {
        VertexShader = compile vs_1_0 VS();
        PixelShader  = compile ps_1_0 psVS1();
    }
}

technique ViewSource2
{
    pass P0
    {
        VertexShader = compile vs_1_0 VS();
        PixelShader  = compile ps_1_0 psVS2();
    }
}

technique ViewControl
{
    pass P0
    {
        VertexShader = compile vs_1_0 VS();
        PixelShader  = compile ps_1_0 psVSct();
    }
}

technique TFixedFunction
{
    pass P0
    {
        //transforms
        WorldTransform[0](0)   = (tW);
        ViewTransform       = (tV);
        ProjectionTransform = (tP);

        //texturing
        Sampler[0](0) = (Src1Samp);
        TextureTransform[0](0) = (tTex);
        TexCoordIndex[0](0) = 0;
        TextureTransformFlags[0](0) = COUNT2;
        //Wrap0 = U;  // useful when mesh is round like a sphere
        
        Lighting       = FALSE;

        //shaders
        VertexShader = NULL;
        PixelShader  = NULL;
    }
}

tonfilm · June 29, 2011, 12:21am

ok, looks good so far. as you don’t use the inputs anyways, you can remove them from the VS input. but switch to vs/ps 2_0 or vs 1_1 and ps 1_4 if you need an old shader model. 1_0 is really limited in value ranges, that might be the problem. and as i learned today newer DX shader compilers don’t support model 1 anymore…

io · June 29, 2011, 1:21am

What do you mean with the inputs of the vertex shader? The input for the transformation or the variable definitions?
tx
S.

tonfilm · June 29, 2011, 1:33am

its cosmetics, the input parameters of the VS, the code uses TexCd only:

code(lang=hlsl):
vs2ps VS(
float4 PosO : POSITION,
float4 TexCd : TEXCOORD0)
{
vs2ps Out = (vs2ps)0;
Out.Pos = mul(PosO, tWVP);
Out.TexCd = mul(TexCd, tTex);
Out.Tex2Cd = mul(TexCd, tTex2);
Out.MaskCd = mul(TexCd, tMask);
return Out;
}

io · June 29, 2011, 1:43am

Cleaned up a little bit but no joy

float4x4 tW: WORLD;
float4x4 tV: VIEW;
float4x4 tP: PROJECTION;
float4x4 tWVP: WORLDVIEWPROJECTION;
 
texture Src1 <string uiname="Source1";>;
sampler Src1Samp = sampler_state
{
    Texture   = (Src1);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
};
 
texture Src2 <string uiname="Source2";>;
sampler Src2Samp = sampler_state
{
    Texture   = (Src2);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
    AddressU = Border;
    AddressV = Border;
};
 
texture Mask<string uiname="Control";>;
sampler MaskSamp = sampler_state
{
    Texture   = (Mask);
    MipFilter = LINEAR;
    MinFilter = LINEAR;
    MagFilter = LINEAR;
};
 
float4x4 tTex: TEXTUREMATRIX <string uiname="Source1 Transform";>;
float4x4 tTex2: TEXTUREMATRIX <string uiname="Source2 Transform";>;
float4x4 tMask: TEXTUREMATRIX <string uiname="Ctrl Transform";>;
 
struct vs2ps
{
    float4 Pos  : POSITION;
    float4 TexCd : TEXCOORD0;
    float4 Tex2Cd : TEXCOORD1;
    float4 MaskCd : TEXCOORD2;
};
 
// --------------------------------------------------------------------------------------------------
// VERTEXSHADERS
// --------------------------------------------------------------------------------------------------
vs2ps VS(
    float4 PosO  : POSITION,
    float4 TexCd : TEXCOORD0)
{
    vs2ps Out = (vs2ps)0;
    Out.Pos = mul(PosO, tWVP);
    Out.TexCd = mul(TexCd, tTex);
    Out.Tex2Cd = mul(TexCd, tTex2);
    Out.MaskCd = mul(TexCd, tMask);
    return Out;
}
 
// --------------------------------------------------------------------------------------------------
// PIXELSHADERS:
// --------------------------------------------------------------------------------------------------
 
float4 psFade(vs2ps In): COLOR
{
    float4 sc1 = tex2D(Src1Samp, In.TexCd);
    float4 sc2 = tex2D(Src2Samp, In.Tex2Cd);
    float4 ctrl = tex2D(MaskSamp, In.MaskCd);
    float4 outColor = sc1 * (1-ctrl) + sc2 * (ctrl);
    return outColor;
}
 
 
// --------------------------------------------------------------------------------------------------
// TECHNIQUES:
// --------------------------------------------------------------------------------------------------
 
technique Fade
{
    pass P0
    {
        VertexShader = compile vs_2_0 VS();
        PixelShader  = compile ps_2_0 psFade();
    }
}

 
technique TFixedFunction
{
    pass P0
    {
        //transforms
        WorldTransform[0](0)   = (tW);
        ViewTransform       = (tV);
        ProjectionTransform = (tP);
 
        //texturing
        Sampler[0](0) = (Src1Samp);
        TextureTransform[0](0) = (tTex);
        TexCoordIndex[0](0) = 0;
        TextureTransformFlags[0](0) = COUNT2;
        //Wrap0 = U;  // useful when mesh is round like a sphere
 
        Lighting       = FALSE;
 
        //shaders
        VertexShader = NULL;
        PixelShader  = NULL;
    }
}

tonfilm · June 29, 2011, 11:57am

maybe its something in your patch… did a quick test and it works for me:

texcoords.zip (4.2 kB)

io · June 29, 2011, 1:15pm

Yes you are right, try to use the VLC plugin as source2 (that one with the transformation) and you ll see how it crops source1.
weird…
S.