[SOLVED!] Lightmap shader not working properly.

Mel · Post by **Mel** » Tue Dec 16, 2008 1:06 pm

Let's see if you guys see anything out of place. I guess there is no need for samples. If not, just ask.

this is the vertex shader:

Code: Select all

float3 fvLightPosition1;
float3 fvLightPosition2;
float3 fvEyePosition;
float4x4 matWorld;
float4x4 matWorldViewProjection;
float4x4 matWorldInverse;

struct VS_INPUT 
{
   float4 Position : POSITION0;
   float2 Texcoord1 : TEXCOORD0;
   float2 Texcoord2 : TEXCOORD1;
   float3 Normal :   NORMAL0;
   float3 Binormal : BINORMAL0;
   float3 Tangent :  TANGENT0;
};

struct VS_OUTPUT 
{
   float4 Position         :  POSITION0;
   float2 Texcoord1        :  TEXCOORD0;
   float2 Texcoord2        :  TEXCOORD1;
   float3 ViewDirection    :  TEXCOORD2;
   float3 LightDirection1  :  TEXCOORD3;
   float3 LightDirection2  :  TEXCOORD4;
   float3 basis1           :  TEXCOORD5;
   float3 basis2           :  TEXCOORD6;
   float3 basis3           :  TEXCOORD7;
   float  z                :  TEXCOORD8;
};

VS_OUTPUT vs_main( VS_INPUT IN )
{
   VS_OUTPUT OUT;
   OUT.Position = mul( IN.Position , matWorldViewProjection );
   OUT.Texcoord1 = IN.Texcoord1;
   OUT.Texcoord2 = IN.Texcoord2;
   OUT.z=OUT.Position.z;
   
   float3 RealPos = mul( IN.Position , matWorld);
   
   float3 viewVec = fvEyePosition - RealPos;
   float3 lightVec1 = fvLightPosition1 - RealPos;
   float3 lightVec2 = fvLightPosition2 - RealPos;

   float3 tViewVec;
   float3 tLightVec1;
   float3 tLightVec2;
   
   float3 OTangent =  normalize(IN.Tangent);
   float3 ONormal = normalize(IN.Normal);
   float3 OBinormal = cross( ONormal , OTangent );
   OTangent = cross( OBinormal,ONormal );
   
   float3 RTangent = mul( matWorldInverse , OTangent );
   float3 RBinormal = mul( matWorldInverse , OBinormal );
   float3 RNormal = mul( matWorldInverse , ONormal );

   tViewVec.x = dot(viewVec,RTangent);
   tViewVec.y = dot(viewVec,RBinormal);
   tViewVec.z = dot(viewVec,RNormal);
   
   tLightVec1.x = dot(lightVec1,RTangent);
   tLightVec1.y = dot(lightVec1,RBinormal);
   tLightVec1.z = dot(lightVec1,RNormal);

   tLightVec2.x = dot(lightVec2,RTangent);
   tLightVec2.y = dot(lightVec2,RBinormal);
   tLightVec2.z = dot(lightVec2,RNormal);
   
   OUT.ViewDirection = tViewVec;
   OUT.LightDirection1 = tLightVec1;
   OUT.LightDirection2 = tLightVec2;

   OUT.basis1.x = OTangent.x;
   OUT.basis1.y = OBinormal.x;
   OUT.basis1.z = ONormal.x;
   OUT.basis2.x = OTangent.y;
   OUT.basis2.y = OBinormal.y;
   OUT.basis2.z = ONormal.y;
   OUT.basis3.x = OTangent.z;
   OUT.basis3.y = OBinormal.z;
   OUT.basis3.z = ONormal.z;
      
   return( OUT );
   
}

And this is the pixel shader.

Code: Select all

float4 LightColor1;
float4 AmbientColor1;
float4 SpecularColor1;
float4 LightColor2;
float4 AmbientColor2;
float4 SpecularColor2;
float4x4 view_matrix;
float4 fogColor;
float near;
float far;
float density;

sampler2D baseMap;
sampler2D bumpMap;
sampler2D refMap;
sampler2D ShadowMap;

struct PS_INPUT 
{
   float2 Texcoord1      :   TEXCOORD0;
   float2 Texcoord2      :   TEXCOORD1;
   float3 ViewDirection  :   TEXCOORD2;
   float3 LightDirection1:   TEXCOORD3;
   float3 LightDirection2:   TEXCOORD4;
   float3 basis1         :   TEXCOORD4;
   float3 basis2         :   TEXCOORD5;
   float3 basis3         :   TEXCOORD6;
   float z               :   TEXCOORD7;
};

float4 ps_main( PS_INPUT IN ) : COLOR0
{ 
      float4 color=fogColor;
      if(IN.z < far){
      
      float ZDepth=min(max(near,IN.z),far);
      float fog=((far-near)-(ZDepth-near))/(far-near);
      fog=pow(fog,density);
      
      float4 base = tex2D(baseMap,IN.Texcoord1);
      float4 shadow = tex2D(ShadowMap,IN.Texcoord2);
      
      base*=shadow;
      
      float3 normal = tex2D(bumpMap,IN.Texcoord1);
      normal = normal*2-1;      

      float3 viewDir = normalize(IN.ViewDirection);
      float3 lightDir1 = normalize(IN.LightDirection1);
      float3 lightDir2 = normalize(IN.LightDirection2);
      
      normal = normalize(normal);
      float3 halfView1 = normalize(lightDir1+viewDir);
      float3 halfView2 = normalize(lightDir2+viewDir);
      
      float specPower = 128*base.w;      
      
      float diffuse1 = dot(normal,lightDir1);
      float specular1 = pow(saturate(dot(normal,halfView1)),specPower);

      float diffuse2 = dot(normal,lightDir2);
      float specular2 = pow(saturate(dot(normal,halfView2)),specPower);

      float3 reflection=normal+1;
      reflection.x= dot(normal,IN.basis1);
      reflection.y= dot(normal,IN.basis2);
      reflection.z= dot(normal,IN.basis3);
      float3 refVec = mul(view_matrix,reflection);
      refVec = normalize(refVec);
      refVec = refVec+1;
      refVec = normalize(refVec);
      
      float fresnel = 1-pow(dot(normal,viewDir),16);
      refVec.xy = refVec.xy*0.5 + 0.5;

      float4 Reflection = tex2D(refMap,refVec.xy)*fresnel*base.w;
      
      color = (saturate(AmbientColor1*base*0.25 + diffuse1*base*LightColor1 + specular1*base*SpecularColor1) +
               saturate(AmbientColor2*base*0.25 + diffuse2*base*LightColor2 + specular2*base*SpecularColor2) +
               Reflection)*fog + (1-fog)*color;
      }
      return ( color );
}

This shader should produce a phong and fresnel relfection shaded, fogged and shadow mapped surface. Instead of it, this is what it does:

Any idea of what is out of place?

hybrid · Post by **hybrid** » Tue Dec 16, 2008 2:06 pm

In Irrlicht you either have tangents and stuff, or a second texture coord. Maybe that's the problem?

BlindSide · Post by **BlindSide** » Tue Dec 16, 2008 2:21 pm

Seconded, and I think you mean lightmap.

Before you start proposing new vertex formats, have a go at generating tangents on the fly.

This part of TGM's shader should interest you the most:

Code: Select all

vec3 fvNormal = gl_NormalMatrix * gl_Normal;
vec3 fvTangent = -vec3(abs(gl_Normal.y) + abs(gl_Normal.z), abs(gl_Normal.x), 0);
vec3 fvBinormal = cross(fvTangent,gl_Normal);
fvTangent = gl_NormalMatrix * cross(fvBinormal, gl_Normal);
fvBinormal = gl_NormalMatrix * fvBinormal;

Where do I get gl_NormalMatrix you say? In HLSL you can substitute the World matrix without (too much) trouble. The rest is self-explanatory, rape and pillage, my friend.

sudi · Post by **sudi** » Tue Dec 16, 2008 2:28 pm

@BlindSide
But generating tangents on the fly takes a lot of gpu time. i tried that with a rather small mesh.

Mel · Post by **Mel** » Tue Dec 16, 2008 2:29 pm

@Hybrid. Hmmm.. most probably, i try to use all of them. I think that having vertices with the necesary data is the best way to have those data passed to the shader directly, at least, to get speed. But i also understand it requires a great efort to get. I'll try the normal generation, at least, for testing purposes.

BlindSide · Post by **BlindSide** » Tue Dec 16, 2008 2:32 pm

Well I tried it with hundreds of meshes, no problem so far (Mainly because once you add a complex pixel shader, the vertex just sits there idling anyway, better give it something to do, yeah yeah, unified shader architecture, if you have a card that new then performance isn't much of a problem.).

But hey, if you wanna bug the devs to add a new vertex format, by all means go ahead.

Mel · Post by **Mel** » Tue Dec 16, 2008 4:59 pm

Mixed feelings... Two news, a good and a bad one

First the good news.

I owe Blindside a Beer or something, because it worked! Now i have shader generated tangent space, at the price of a bit of rendering time, but, given the fact that things in graphics card go in paralell nowadays, it is almost unnoticeable (for a 30000 triangle mesh), and also, it happens that i almost had it done like that, i only needed a way to generate the whole space without the tangent, just the normal. So, i don't notice it at all.

Bad news.

Now the main character has lost her shaders. I don't know what is wrong now. When the shader wasn't working correctly, the character rendered well, but now, it seems as if she had turned to the good old fixed pipeline renderer. That on one side, the other thing is that, i had a fog done in the pixel shader which now won't show. I don't understand why, but it simply won't show.

This just keeps getting better, and better, but, given the reason of this thread, this is a "sloved" trouble, although it has made other appear.

Edit: Wooops! my bad. Right now i'm not at home, but i see there is a small trouble with the pixel shader semantics, maybe that is the source of all the troubles. Have to check it. seemingly, it will solve the fog trouble, don't know about the other.

dlangdev · Post by **dlangdev** » Tue Dec 16, 2008 7:30 pm

It would be nice to see a sample render later, though.

Mel · Post by **Mel** » Wed Dec 17, 2008 10:53 am

Well, i must say that i was testing the "bad news" with the 1.5 beta version. The release has the problems solved

so, the bad news are also gone.

As you asked, here you are. Thanks, thank you very much for your help. I might release the shaders later, right now, i don't have them available

Sorry if they're a bit big ^^U but i rather to show the actual screens.

What i don't like is the blurry look of everything takes when rendered with blindside's Xeffects, is there any way to avoid it?

Damm it, it has taken forever to upload this!

BlindSide · Post by **BlindSide** » Wed Dec 17, 2008 11:15 am

Hey the bumpmapping is looking sexy.

What i don't like is the blurry look of everything takes when rendered with blindside's Xeffects, is there any way to avoid it?

What resolution are you using for the screenRTT parameter for XEffects? Try playing around with that value, set it higher than the actual resolution if you have to (Might give you a bit of anti-aliasing-ness). It really looks like you have it set to 512x512 or something, which really isn't necessary anymore, Irrlicht 1.5 allows for large RTTs in DX.

Btw are you sticking with those stencil shadows? They look terrible

(Ok not so bad because the bad RTT resolution is blurring the edges.)

You know how you can pass a user texture to the XEffects post processing? Well try rendering JUST the stencil shadows (Black and white) to an RTT, pass that as the texture, blur the RTT (You can modify the provided blur shaders for this, but read from 4th component, more info in the effectWrapper header), and then multiply it with the original scene as a final pass.

Cheers

dlangdev · Post by **dlangdev** » Wed Dec 17, 2008 5:09 pm

Looks pretty good, mel.

Mel · Post by **Mel** » Wed Dec 17, 2008 5:40 pm

It is quite close to what i wanted to achieve, with regard to the MAX renders i did. About the resolution, i use the parameter 0,0 for the resolution of RTT's, same as screen.

fmx · Post by **fmx** » Wed Dec 17, 2008 8:20 pm

Great screens, everything looking really peachy!

Maybe the RTT size numbers get modified to NPOT values or something, because looks like serious aliasing suggesting definite mis-match between RTT and Screen-Resolution sizes.
I don't know about the inner-workings of XEffects though, so maybe its just something you're doing different / wrong?

BlindSide · Post by **BlindSide** » Thu Dec 18, 2008 4:31 am

I don't know why, but someone on the XEffects thread said:

EDIT: It seems that multiplying the resolution by two gives you the resolution you want... is this the correct way to go about this?

I might need to add a half-pixel offset for the screen quad texcoords or something.

Mel · Post by **Mel** » Thu Dec 18, 2008 11:48 am

Or perhaps, disabling the bilinear filter to the RTT. Given that both the screen and RTT would have the same resolution, it doesn't make sense filtering anything, i think. I might try and poke a bit to disable it, and see what results does it give as well.