hlsl shader help [SOLVED!]

[jingquan]

I tried to translate some normalmap + specular lighting shader from fx to pure hlsl, and now i'm kinda stuck as the normalmap (and specularmap) isn't shown or working. The blinn lighting works thought. What's wrong. I'm sure I passed the callbacks correctly.

here's the hlsl:

Code: Select all

float4x4 wvp; 
float4x4 worldIT; 
float4x4 viewInv; 
float4x4 world; 
float4 lightPosition; 

struct v2f {
      half4 position       : POSITION;
      half2 texCoord       : TEXCOORD0;
      half3 eyeVec      : TEXCOORD1;
      half3 lightVec      : TEXCOORD2;
      half3 worldNormal   : TEXCOORD3;
      half3 worldTangent   : TEXCOORD4;
      half3 worldBinormal   : TEXCOORD5;
}; 

   struct a2v 
{ 
   float4 position      : POSITION;
   float3 normal      : NORMAL;
   float2 texCoord      : TEXCOORD0;
   float3 tangent      : TEXCOORD0;
   float3 binormal      : TEXCOORD0;
}; 

v2f vsmain(in a2v In) 
{ 
v2f Out = (v2f)0; 
    Out.worldNormal = mul(In.normal, worldIT).xyz; 
    Out.worldTangent = mul(In.tangent, worldIT).xyz; 
    Out.worldBinormal = mul(In.binormal, worldIT).xyz; 
    //can use either positive or negative y format normal maps 
    //comment out this if statement to save 6 instructions! 
     //if (direction == true) Out.worldTangent = -Out.worldTangent; 
    float3 worldSpacePos = mul(In.position, world); 
    Out.lightVec = lightPosition - worldSpacePos; 
   Out.texCoord.xy = In.texCoord; 
    Out.eyeVec = viewInv[3].xyz - worldSpacePos; 
    Out.position = mul(In.position, wvp); 
    return Out; 
} 


    
float4 lightColor; 
float4 ambient; 
float4 specularColor; 
float4 surfColor; 
float shininess; 

sampler colorTextureSampler: register(s0); 
sampler normalMapSampler: register(s1); 
sampler specTextureSampler: register(s2); 

half4 blinn2(float3 N,
      float3 L,
      float3 V,
      float4 sColor,
      float shine)
   {
   float3 H = normalize(V+L);
   float4 lighting = lit(dot(L,N), dot(H,N), shine);
   return lighting.y + sColor*lighting.z;
   }

float4 psmain(in v2f In) : COLOR
{
  //fetch the diffuse and normal and spec maps
  float4 colorMap = tex2D(colorTextureSampler, In.texCoord.xy);
  float4 specMap = tex2D(specTextureSampler, In.texCoord.xy);
  float3 normal = tex2D(normalMapSampler, In.texCoord).xyz * 2.0 - 1.0;
  
  //create tangent space vectors
  float3 Nn = normalize(In.worldNormal);
  float3 Tn = normalize(In.worldTangent);
  float3 Bn = normalize(In.worldBinormal);
  
  //these vectors could be normalized, but it costs 4 more instructions
  //and makes almost no difference to image quality
  //half3 Nn = normalize(In.worldNormal);
  //half3 Tn = normalize(In.worldTangent);
  //half3 Bn = normalize(In.worldBinormal);
  
  //offset world space normal with normal map values
  float3 N = (Nn * normal.z) + (normal.x * Bn + normal.y * -Tn);
  //N = normalize(N);
  
  //create lighting vectors - view vector and light vector
  float3 V = normalize(In.eyeVec);
  float3 L = normalize(In.lightVec.xyz);
  
  //lighting
  
    //lighting
  
  //ambient light
  float4 C = ambient*colorMap * 0.33;
  
  //specular color
  float4 specCol = specularColor*specMap;
  
  //diffuse and specular
  C += lightColor * blinn2(N, L, V, specCol, shininess);
  C *= colorMap*surfColor;

  return C;
}

Many thanks.

[belfegor]

You cant get good results with current irrlicht vertex types.
I think that they store binormal/tangent info in texcoord2 or 3
so you can try and replace:

Code: Select all

struct a2v
{
   float4 position      : POSITION;
   float2 texCoord      : TEXCOORD0;
   float3 tangent      : TANGENT;// <--- to TEXCOORD2
   float3 binormal      : BINORMAL;// <--- to TEXCOORD3
   float3 normal      : NORMAL;
};

i might be wrong, better to see in irr source CDirect3D9Driver.cpp

[jingquan]

Thx, I've changed the code. but there's still no normalmapping. I've tried using TANGENT0 and BINORMAL0 and there seem to have no visible change. Is the 2 vertex passes causing the problem. Perhaps it's my callback, here take a look (sry, it's in C# but probably understandable):

Code: Select all

public class normal_callback
    {
        public static float shininess = 20f;
        public static Vector3D lightPosition = new Vector3D(0, 100, 0);
        public static Colorf lightColor = new Colorf(1f, 1f, 0.5f, 0.2f);
        public static Colorf ambient = new Colorf(1f, 0.4f, 0.4f, 0.4f);
        public static Colorf specularColor = new Colorf(1f, 1f, 1f, 1f);
        public static Colorf surfColor = new Colorf(1f, 1f, 1f, 1f);

        public static void OnShaderSet(MaterialRendererServices services, int userData)
        {
            Matrix4 wvp = services.VideoDriver.GetTransform(TransformationState.Projection);
            wvp *= services.VideoDriver.GetTransform(TransformationState.View);
            wvp *= services.VideoDriver.GetTransform(TransformationState.World);
            services.SetVertexShaderConstant("wvp", wvp.ToShader(), 16);
            Matrix4 viewInv = services.VideoDriver.GetTransform(TransformationState.View);
            viewInv.MakeInverse();
            services.SetVertexShaderConstant("viewInv", viewInv.ToShader(), 16);
            Matrix4 world = services.VideoDriver.GetTransform(TransformationState.World);
            services.SetVertexShaderConstant("world", world.ToShader(), 16);
            Matrix4 worldIT = services.VideoDriver.GetTransform(TransformationState.World);
            worldIT.GetTransposed();
            worldIT.MakeInverse();
            services.SetVertexShaderConstant("worldIT", worldIT.ToShader(), 16);
            services.SetVertexShaderConstant("lightPosition", lightPosition.ToShader(), 4);

            services.SetPixelShaderConstant("lightColor", lightColor.ToShader(), 4);
            services.SetPixelShaderConstant("ambient", ambient.ToShader(), 4);
            services.SetPixelShaderConstant("specularColor", specularColor.ToShader(), 4);
            services.SetPixelShaderConstant("surfColor", surfColor.ToShader(), 4);
            services.SetPixelShaderConstant("shininess", shininess);
        }
    }

I had made the matrices inverse or GetTransposed according to the shader and there should be no problem.

[belfegor]

Thx, I've changed the code. but there's still no normalmapping.

Is it possible to see some screenshot.

Do you have any warrnings in console.

I am not sure if :

Code: Select all

worldIT = worldIT.GetTransposed();

instead just:

Code: Select all

worldIT.GetTransposed();

because i dont use irrlicht shader system.

Also i noticed when you need to pass worldIT but you first transpose it and
then inverse, try other way around.

Good luck.

[jingquan]

Hello,

I tried changing the worldIT, it's still the same. I think that's not causing the problem as it works with world too.

There's no error in the console.

Here's a screenshot, the lighting works fine with the light position at vector3d(0, 100, 0). The specular blob reflects accurately into the camera. I tried changing the light color, position, ambient and everything works, except the bumpiness i should get.



Normal map (the one with shades of blue and pink, RGB) is set at the second texture layer and the specularmap (like a greyscale map) is set at the third.

Here's the link to the hlsl fx file: http://www.bencloward.com/shaders_Norma ... ghts.shtml

I converted exactly as it should be, just that i used one light instead of 3.

Could anyone point out where's the error?

[belfegor]

One more thing before i try this at home, try:

Code: Select all

float3 normal = tex2D(normalMapSampler, In.texCoord).xyz;
normal = 2 * (normal  - 0.5);

in pixel shader.

[jingquan]

Hello belfegor,

Many huge thanks for your consistent help, but it didn't work unfortunately. It looked exactly the same as before. BTW, I loaded it in rendermonkey and there it was, no sign of normalmapping as well.

I'm not quite sure if i had done the right way in the program. Here's the project file if you use rendermonkey: www.orandysoftware.com/user_files/normal.zip

I'm comparing my code with the hlsl samples to find out what's missing.....

[belfegor]

It looked exactly the same as before

Hmm, thats strange, it should be at least minimal difference.

Ok, ill test at home with RenderMonkey too.

[omaremad]

float3 tangent : TEXCOORD2;

float3 binormal : TEXCOORD3;

to

float3 tangent : TEXCOORD1;

float3 binormal : TEXCOORD2;

i think irrlicht has them under those tcoords for tangent vertex types.

[jingquan]

I tried that too, didn't work.

[jingquan]

Hey, some bump marks appeared when changed this line in the vertexshader:

Code: Select all

   float2 texCoord      : TEXCOORD0;
   float3 tangent      : TEXCOORD0;
   float3 binormal      : TEXCOORD0;

and removed this in pixelshader:

Code: Select all

N = normalize(N);

However, the specular reflections didn't exactly go according to the light vector.

[jingquan]

I updated the code above, fixing the specular map stuff. I don't really know what's causing the normalmapping problems....the shader or irrlicht?

[omaremad]

what vertex type are you using?

[jingquan]

I think is this:

Code: Select all

struct a2v 
{ 
   float4 position      : POSITION; 
   float2 texCoord      : TEXCOORD0; 
   float3 tangent      : TANGENT;
   float3 binormal      : BINORMAL;
   float3 normal      : NORMAL; 
};

Position, normal, tangent, binormal.

[belfegor]

Try this (it sort works):

main.cpp

Code: Select all

#include <irrlicht.h>
using namespace irr;
#pragma comment(lib, "Irrlicht.lib")

const s32 winWidth          = 800;
const s32 winHeight         = 600;

IrrlichtDevice* device           = 0;
scene::IBillboardSceneNode* Bill = 0;

class MyShaderCallBack : public video::IShaderConstantSetCallBack
{
	struct Vec4
	{
		Vec4(core::vector3df vec)
		{
			X = vec.X;
			Y = vec.Y;
			Z = vec.Z;
			W = 1.0f;
		}
		Vec4(core::vector3df vec, float w)
		{
			X = vec.X;
			Y = vec.Y;
			Z = vec.Z;
			W = w;
		}
		f32 X, Y, Z, W;
	};
public:
	virtual void OnSetConstants(video::IMaterialRendererServices* services, s32 userData)
	{
		video::IVideoDriver* Driver  = services->getVideoDriver();

		core::matrix4 World         = Driver->getTransform(video::ETS_WORLD);
		core::matrix4 View          = Driver->getTransform(video::ETS_VIEW);
		core::matrix4 Projection    = Driver->getTransform(video::ETS_PROJECTION);

		core::matrix4 WorldViewProjection   = Projection * View * World;
		core::matrix4 WorldInverseTranspose;
		World.getInverse(WorldInverseTranspose);
		WorldInverseTranspose.getTransposed(WorldInverseTranspose);

		core::matrix4 ViewInverse;
		View.getInverse(ViewInverse);

		Vec4 LightPosition(Bill->getAbsolutePosition());
		Vec4 LightColor(core::vector3df(0.6f, 0.6f, 0.6f));
		Vec4 SpecularColor(core::vector3df(0.8f, 0.8f, 0.8f));
		Vec4 SurfaceColor(core::vector3df(0.5f, 0.5f, 0.5f));
		Vec4 Ambient(core::vector3df(0.6f, 0.6f, 0.6f));

		f32 Shininess = 128.0f;

		services->setVertexShaderConstant("wvp", WorldViewProjection.pointer(), 16);
		services->setVertexShaderConstant("worldIT", WorldInverseTranspose.pointer(), 16);
		services->setVertexShaderConstant("viewInv", ViewInverse.pointer(), 16);
		services->setVertexShaderConstant("world", World.pointer(), 16);
		services->setVertexShaderConstant("lightPosition", &LightPosition.X, 4);

		services->setPixelShaderConstant("lightColor", &LightColor.X, 4);
		services->setPixelShaderConstant("specularColor", &SpecularColor.X, 4);
		services->setPixelShaderConstant("surfColor", &SurfaceColor.X, 4);
		services->setPixelShaderConstant("ambient", &Ambient.X, 4);
		services->setPixelShaderConstant("shininess", &Shininess, 1);
	}
};

class MyEventReceiver : public IEventReceiver
{
public:
	bool OnEvent(SEvent event)
	{
		if(event.EventType == EET_KEY_INPUT_EVENT &&
			event.KeyInput.Key == KEY_ESCAPE &&
			!event.KeyInput.PressedDown)
		{
			device->closeDevice();
			return true;
		}
		return false;
	}
};

int main()
{
	MyEventReceiver receiver;
	SIrrlichtCreationParameters param;
	param.AntiAlias                 = 0;
	param.Bits                      = 32;
	param.DriverType                = video::EDT_DIRECT3D9;
	param.EventReceiver             = &receiver;
	param.Fullscreen                = false;
	param.HighPrecisionFPU          = true;
	param.Stencilbuffer             = true;
	param.Vsync                     = false;
	param.RefreshRate               = 0;
	param.WindowSize.Width          = winWidth;
	param.WindowSize.Height         = winHeight;
	device = createDeviceEx(param);

	video::IVideoDriver* driver = device->getVideoDriver();
	scene::ISceneManager* smgr  = device->getSceneManager();

	scene::ICameraSceneNode* Cam = smgr->addCameraSceneNodeFPS();
	Cam->setPosition(core::vector3df(0.0f, 150.0f, 0.0f));

	c8* FileName = "normal.hlsl";
	s32 normal   = -1;
	video::IGPUProgrammingServices* gpu = driver->getGPUProgrammingServices();
	MyShaderCallBack* mc = new MyShaderCallBack();
	normal = gpu->addHighLevelShaderMaterialFromFiles(
		FileName,	"vsmain", video::EVST_VS_2_0,
		FileName, "psmain", video::EPST_PS_2_0,
		mc, video::EMT_SOLID);
	mc->drop();

	scene::IAnimatedMesh* roomMesh = smgr->getMesh("room.3ds");
	scene::ISceneNode* room = 0;
	if(roomMesh)
	{
		smgr->getMeshManipulator()->makePlanarTextureMapping(
			roomMesh->getMesh(0), 0.003f);
	}
	video::ITexture* colorMap  = driver->getTexture("rockwall.bmp");
	video::ITexture* heightMap = driver->getTexture("rockwall_height.bmp");
	video::ITexture* normalMap = driver->getTexture("rockwall_height.bmp");
	driver->makeNormalMapTexture(normalMap, 9.0f);
	scene::IMesh* tangentMesh = smgr->getMeshManipulator()->createMeshWithTangents(
		roomMesh->getMesh(0));
	room = smgr->addMeshSceneNode(tangentMesh);
	room->setMaterialTexture(0, colorMap);
	room->setMaterialTexture(1, normalMap);
	room->setMaterialTexture(2, heightMap);
	room->setMaterialType((video::E_MATERIAL_TYPE)normal);
	tangentMesh->drop();

	Bill = smgr->addBillboardSceneNode();
	Bill->setMaterialFlag(video::EMF_LIGHTING, false);
	Bill->setMaterialTexture(0, driver->getTexture("particle.bmp"));
	Bill->setMaterialType(video::EMT_TRANSPARENT_ADD_COLOR);
	Bill->setPosition(Cam->getPosition());

	scene::ISceneNodeAnimator* Anm = smgr->createFlyCircleAnimator(
		Bill->getPosition(), 180.0f, 0.002f);
	Bill->addAnimator(Anm);
	Anm->drop();

	while(device->run())
	{
		if(device->isWindowActive())
		{
			driver->beginScene(true,true,video::SColor(255, 128, 128, 128));
			smgr->drawAll();
			driver->endScene();
		}
	}
	device->drop();
	return 0;
}

normal.hlsl

Code: Select all

float4x4 wvp; 
float4x4 worldIT; 
float4x4 viewInv; 
float4x4 world; 
float4 lightPosition; 

struct v2f 
{ 
      float4 position       : POSITION; 
      float2 texCoord       : TEXCOORD0; 
      float3 eyeVec         : TEXCOORD4; 
      float3 lightVec       : TEXCOORD5; 
      float3 worldNormal    : TEXCOORD3; 
      float3 worldTangent   : TEXCOORD1; 
      float3 worldBinormal  : TEXCOORD2; 
}; 

struct a2v 
{ 
   float4 position   : POSITION; 
   float2 texCoord   : TEXCOORD0; 
   float3 tangent    : TEXCOORD1; 
   float3 binormal   : TEXCOORD2; 
   float3 normal     : NORMAL; 
}; 

v2f vsmain(a2v In) 
{ 
    v2f Out = (v2f)0; 
    Out.worldNormal = mul(In.normal, worldIT).xyz; 
    Out.worldTangent = mul(In.tangent, worldIT).xyz; 
    Out.worldBinormal = mul(In.binormal, worldIT).xyz; 
    //can use either positive or negative y format normal maps 
    //comment out this if statement to save 6 instructions! 
     //if (direction == true) Out.worldTangent = -Out.worldTangent; 
    float3 worldSpacePos = mul(In.position, world); 
    Out.lightVec = lightPosition - worldSpacePos; 
    Out.texCoord.xy = In.texCoord; 
    Out.eyeVec = viewInv[3].xyz - worldSpacePos; 
    Out.position = mul(In.position, wvp); 
    return Out; 
} 

float4 lightColor; 
float4 ambient; 
float4 specularColor; 
float4 surfColor; 
float  shininess; 

sampler colorTextureSampler : register(s0); 
sampler normalMapSampler    : register(s1); 
sampler specTextureSampler  : register(s2); 

float4 blinn2(float3 N, 
      float3 L, 
      float3 V) 
{ 
   float3 H = normalize(V+L); 
   float4 lighting = lit(dot(L,N), dot(H,N), shininess); 
   return lighting.y + specularColor*lighting.z; 
} 

float4 psmain(in v2f In) : COLOR 
{ 
  //fetch the diffuse and normal and spec maps 
  float4 colorMap = tex2D(colorTextureSampler, In.texCoord.xy); 
  float4 specMap = tex2D(specTextureSampler, In.texCoord.xy); 
  float3 normal = tex2D(normalMapSampler, In.texCoord).xyz; 
  normal = 2 * (normal  - 0.5);
  
  //create tangent space vectors 
  float3 Nn = In.worldNormal; 
  float3 Tn = In.worldTangent; 
  float3 Bn = In.worldBinormal; 
  
  //these vectors could be normalized, but it costs 4 more instructions 
  //and makes almost no difference to image quality 
  //half3 Nn = normalize(In.worldNormal); 
  //half3 Tn = normalize(In.worldTangent); 
  //half3 Bn = normalize(In.worldBinormal); 
  
  //offset world space normal with normal map values 
  float3 N = (Nn * normal.z) + (normal.x * Bn + normal.y * -Tn); 
  N = normalize(N); 
  
  //create lighting vectors - view vector and light vector 
  float3 V = normalize(In.eyeVec); 
  float3 L = normalize(In.lightVec.xyz); 
  
  //lighting 
  
    //lighting 
  
  //ambient light 
  float4 C = ambient*colorMap;// * 0.33; 
  
  //specular color 
  float4 specCol = specularColor*specMap; 
  
  //diffuse and specular 
  C += lightColor * blinn2(N, L, V); 
  C *= colorMap*surfColor; 

  return C; 
}