Because of the recent interest in this, I have decided to improve it:
Shader and Demo can be downloaded from: http://irrlichtirc.g0dsoft.com/BlindSide/celshading.zip
(Disregard all outdated instructions below, just follow what I do in the example provided.)
UPDATE:
To use this with the newest Irrlicht version do:
YOURSCENENODE->getMaterial(0).TextureLayer[2].BilinearFilter = false;
---------------
Hey guys I made a nice cel shader based on the OGRE and Omaremad's shader a while ago heres the code:
toonshade.hlsl:
Code: Select all
//-----------------------------------------------------------------------------
// Global variables
//-----------------------------------------------------------------------------
float4x4 mWorldViewProj; // World * View * Projection transformation
float4x4 mInvWorld; // Inverted world matrix
float3 mLightPos; // Light position
struct VS_OUTPUT
{
float4 Position: POSITION0;
float2 TexCoord: TEXCOORD0;
float2 diffuse: TEXCOORD1;
};
VS_OUTPUT vertexMain(
float4 Position: POSITION0,
float2 TexCoord: TEXCOORD0,
float2 diffuse: TEXCOORD1,
float3 Normal: NORMAL0)
{
VS_OUTPUT Output;
// calculate light vector
float3 N = normalize(Normal);
float3 L = normalize(mLightPos - Position.xyz);
// Calculate diffuse component
diffuse = max(dot(N, L),0);
// Shading offset
float offset = 0.1f;
//Subtract the offset and clamp the texture to 0
diffuse = clamp(diffuse+offset,0,1);
Output.Position = mul(Position,mWorldViewProj);
Output.diffuse = diffuse;
Output.TexCoord = TexCoord;
return( Output );
}
sampler2D tex0 : register(s0) ;
sampler1D diffuseRamp : register(s2) ;
float4 pixelMain(
float2 TexCoord: TEXCOORD0,
float2 diffuseIn: TEXCOORD1
) : COLOR0
{
float4 color = float4(1,1,1,1);
float4 diffuse = float4(0.8f, 0.8f, 0.8f, 1.0f);
// Step functions from textures
diffuseIn = tex1D(diffuseRamp, diffuseIn).x;
color = (diffuse * diffuseIn);
float4 col = tex2D( tex0, TexCoord);
col*=2;
return( color*col );
}
In the vertex shader callback make sure to send the light position as mLightPos and on your scene node make the 3rd (not 2nd not 4th) material similar to this:
But 10 times smaller, so you get a 1D texture 16 pixels wide determining the shades of the cel shading.
Preview:
Cheers.
) I hope that people will find more nice shaders and convert them to irrlicht (Unfortunately for me I only have PS 1.4 (And want to stick with it to ensure compatibility, and cuz im cheap 
) so I cannot see many nice shader effects, but alot is achievable with just 1.4, people like to use 2.0+ to make it easier)
The asm code was to show how to use the Effect Compiler FXC.exe to compile HLSL into asm.