Anti-Aliasing with RenderTarget

[Foaly]

Is there a way to enable anti-aliasing for rendering to a render target texture?

I've enabled anti aliasing both in the material and in the device creation parameters, but the anti aliasing does not work with the render target.

[The_Glitch]

I believe I read somewhere with dx9 you have to do it manually or something along those lines. Not sure what your using though.

[Mel]

The antialiasing doesn't work on rendertargets under DX9 or GL at all. There is a way though, to draw a solved multisampled backbuffer to a texture so you can have a rendertarget with MSAA, look for stretchRect on DX9 and DMA memory transfer on GL, (i think). But your best shot is to use an antialiasing post process. FXAA is capable of performing AA on a single rendertarget without any aditional information. It is not the best as MSAA would be, but works fairly well.

[devonsoft]

We actually hacked this in for DX9 in Octodad. Although some nvidia drivers seem to have a weird dithered screendoor MSAA which I couldn't figure out (still looks better than no AA tho). Here are the changes I made:

ITexture.h

Code: Select all

    
// Add ETCF_MSAA_RENDERTARGET to the enum E_TEXTURE_CREATION_FLAG so we dont have to modify every Texture constructor
 
//! Allow the Driver to use Non-Power-2-Textures
    /** BurningVideo can handle Non-Power-2 Textures in 2D (GUI), but not in 3D. */
    ETCF_ALLOW_NON_POWER_2 = 0x00000040,
 
   // If the driver supports, create MSAA render target
    ETCF_MSAA_RENDERTARGET = 0x00000080,
 
    /** This flag is never used, it only forces the compiler to compile
    these enumeration values to 32 bit. */
    ETCF_FORCE_32_BIT_DO_NOT_USE = 0x7fffffff
};

CD3D9Texture.h:

Code: Select all

//add a member variable;
D3DMULTISAMPLE_TYPE Samples; 

CD3D9Texture.cpp

Code: Select all

 
@@ -34,12 +34,16 @@ CD3D9Texture::CD3D9Texture(CD3D9Driver* driver, co
                           const io::path& name, const ECOLOR_FORMAT format)
 : ITexture(name), Texture(0), RTTSurface(0), Driver(driver), DepthSurface(0),
    TextureSize(size), ImageSize(size), Pitch(0), ColorFormat(ECF_UNKNOWN),
-   HasMipMaps(false), HardwareMipMaps(false), IsRenderTarget(true)
+  , Samples((D3DMULTISAMPLE_TYPE)0)
 {
    #ifdef _DEBUG
    setDebugName("CD3D9Texture");
    #endif
 
+   if (driver->getTextureCreationFlag(video::ETCF_MSAA_RENDERTARGET))
+      Samples = (D3DMULTISAMPLE_TYPE)driver->Params.AntiAlias;
+
    Device=driver->getExposedVideoData().D3D9.D3DDev9;
    if (Device)
        Device->AddRef();
 
@@ -138,15 +239,30 @@ void CD3D9Texture::createRenderTarget(const ECOLOR
    // create texture
    HRESULT hr;
 
-   hr = Device->CreateTexture(
-       TextureSize.Width,
-       TextureSize.Height,
-       1, // mip map level count, we don't want mipmaps here
-       D3DUSAGE_RENDERTARGET,
-       d3dformat,
-       D3DPOOL_DEFAULT,
-       &Texture,
-       NULL);
+   if (Samples > 0)
+   {
+      hr = Device->CreateRenderTarget(
+          TextureSize.Width,
+          TextureSize.Height,
+          d3dformat,
+         Samples,
+         0,
+          false,
+          &RTTSurface,
+          NULL);
+   }
+   else
+   {
+      hr = Device->CreateTexture(
+          TextureSize.Width,
+          TextureSize.Height,
+          1, // mip map level count, we don't want mipmaps here
+          D3DUSAGE_RENDERTARGET,
+          d3dformat,
+          D3DPOOL_DEFAULT,
+          &Texture,
+          NULL);
+   }
 
    if (FAILED(hr))
    {
@@ -657,14 +849,22 @@ IDirect3DSurface9* CD3D9Texture::getRenderTargetSu
    if (!IsRenderTarget)
        return 0;
 
-   IDirect3DSurface9 *pRTTSurface = 0;
-   if (Texture)
-       Texture->GetSurfaceLevel(0, &pRTTSurface);
+   if (Samples > 0)
+   {
+      return RTTSurface;
+   }
+   else
+   {
+      IDirect3DSurface9 *pRTTSurface = 0;
+      if (Texture)
+          Texture->GetSurfaceLevel(0, &pRTTSurface);
 
-   if (pRTTSurface)
-       pRTTSurface->Release();
+      if (pRTTSurface)
+          pRTTSurface->Release();
 
-   return pRTTSurface;
+
+      return pRTTSurface;
+   }
 }
 
 

CD3D9Driver.cpp

Code: Select all

 
@@ -1581,6 +1604,23 @@ void CD3D9Driver::draw2DImage(const video::ITextur
    if(!texture)
        return;
 
+   // if drawing MSAA render texture to screen use StretchRect instead of actually drawing
+   if (texture->isRenderTarget() && ((CD3D9Texture*)texture)->Samples > 0)
+   {
+      IDirect3DSurface9* backBuffer = NULL;
+      IDirect3DSurface9* currentRenderTarget = NULL;
+      pID3DDevice->GetBackBuffer(0, 0, D3DBACKBUFFER_TYPE_MONO, &backBuffer);
+      pID3DDevice->GetRenderTarget(0, &currentRenderTarget);
+
+      if (backBuffer == currentRenderTarget && ((CD3D9Texture*)texture)->RTTSurface && ((CD3D9Texture*)texture)->RTTSurface != currentRenderTarget)
+      {
+         // copy rendertarget to backbuffer
+         pID3DDevice->StretchRect(((CD3D9Texture*)texture)->RTTSurface, NULL, backBuffer, NULL, D3DTEXF_NONE);
+         backBuffer->Release();
+         return;
+      }
+   }
+
    const core::dimension2d<u32>& ss = texture->getOriginalSize();
    core::rect<f32> tcoords;
    tcoords.UpperLeftCorner.X = (f32)sourceRect.UpperLeftCorner.X / (f32)ss.Width;
@@ -1803,6 +1843,23 @@ void CD3D9Driver::draw2DImage(const video::ITextur
    if (!sourceRect.isValid())
        return;
 
+   // if drawing MSAA render texture to screen use StretchRect instead of actually drawing
+   if (texture->isRenderTarget() && ((CD3D9Texture*)texture)->Samples > 0)
+   {
+      IDirect3DSurface9* backBuffer = NULL;
+      IDirect3DSurface9* currentRenderTarget = NULL;
+      pID3DDevice->GetBackBuffer(0, 0, D3DBACKBUFFER_TYPE_MONO, &backBuffer);
+      pID3DDevice->GetRenderTarget(0, &currentRenderTarget);
+
+      if (backBuffer == currentRenderTarget && ((CD3D9Texture*)texture)->RTTSurface && ((CD3D9Texture*)texture)->RTTSurface != currentRenderTarget)
+      {
+         // copy rendertarget to backbuffer
+         pID3DDevice->StretchRect(((CD3D9Texture*)texture)->RTTSurface, NULL, backBuffer, NULL, D3DTEXF_NONE);
+         backBuffer->Release();
+         return;
+      }
+   }
+
    if (!setActiveTexture(0, const_cast<video::ITexture*>(texture)))
        return;
@@ -2977,7 +3040,10 @@ bool CD3D9Driver::reset()
            // so take first one
            if (((CD3D9Texture*)(Textures[j].Surface))->DepthSurface==DepthBuffers[i])
            {
-               ((CD3D9Texture*)(Textures[j].Surface))->Texture->GetLevelDesc(0,&desc2);
+            if (((CD3D9Texture*)(Textures[j].Surface))->Samples > 0)
+               ((CD3D9Texture*)(Textures[j].Surface))->RTTSurface->GetDesc(&desc2);
+            else
+                  ((CD3D9Texture*)(Textures[j].Surface))->Texture->GetLevelDesc(0,&desc2);
                break;
            }
        }
@@ -3256,7 +3322,15 @@ ITexture* CD3D9Driver::addRenderTargetTexture(cons
    CD3D9Texture* tex = new CD3D9Texture(this, size, name, format);
    if (tex)
    {
-       if (!tex->Texture)
+      if (tex->Samples > 0)
+      {
+         if (!tex->RTTSurface)
+         {
+            tex->drop();
+            return 0;
+         }
+      }
+       else if (!tex->Texture)
        {
            tex->drop();
            return 0;
@@ -3516,12 +3590,15 @@ void CD3D9Driver::checkDepthBuffer(ITexture* tex)
            !queryFeature(EVDF_TEXTURE_NSQUARE), true);
    SDepthSurface* depth=0;
    core::dimension2du destSize(0x7fffffff, 0x7fffffff);
-   for (u32 i=0; i<DepthBuffers.size(); ++i)
+   for (u32 i=1; i<DepthBuffers.size(); ++i)
    {
        if ((DepthBuffers[i]->Size.Width>=optSize.Width) &&
            (DepthBuffers[i]->Size.Height>=optSize.Height))
        {
-           if ((DepthBuffers[i]->Size.Width<destSize.Width) &&
+         D3DSURFACE_DESC desc;
+         DepthBuffers[i]->Surface->GetDesc(&desc);
+        
+           if (desc.MultiSampleType == ((CD3D9Texture*)tex)->Samples && (DepthBuffers[i]->Size.Width<destSize.Width) &&
                (DepthBuffers[i]->Size.Height<destSize.Height))
            {
                depth = DepthBuffers[i];
@@ -3535,7 +3612,12 @@ void CD3D9Driver::checkDepthBuffer(ITexture* tex)
        DepthBuffers[0]->Surface->GetDesc(&desc);
        // the multisampling needs to match the RTT
        D3DSURFACE_DESC desc2;
-       ((CD3D9Texture*)tex)->Texture->GetLevelDesc(0,&desc2);
+
+      // If multisampled use RTTSurface because Texture is 0
+      if (((CD3D9Texture*)tex)->Samples > 0)
+         ((CD3D9Texture*)tex)->RTTSurface->GetDesc(&desc2);
+      else
+          ((CD3D9Texture*)tex)->Texture->GetLevelDesc(0,&desc2);
        DepthBuffers.push_back(new SDepthSurface());
        HRESULT hr=pID3DDevice->CreateDepthStencilSurface(optSize.Width,
                optSize.Height,
 

Then you can create an MSAA rendertarget texture like so:

Code: Select all

 
      driver->setTextureCreationFlag(ETCF_MSAA_RENDERTARGET, true);
   ScreenRTT = driver->addRenderTargetTexture(ScreenRTTSize, "screenRTT");
   driver->setTextureCreationFlag(ETCF_MSAA_RENDERTARGET, false);

[devonsoft]

OpenGL is similar, the code I have for that is older but the right way would be to add the Mutlisample renderbuffer extensions to the opengl extension handler wrapper and s32 Samples to the COpenGLFBOTexture as well and then do this in the COpenGLFBOTexture constructor:

Code: Select all

    if (Samples > 0)
    {
#if defined(GL_ARB_framebuffer_object) || defined(GL_EXT_framebuffer_multisample)
    Driver->extGlGenFramebuffers( 1, &ColorFrameBuffer );
    Driver->extGlGenRenderbuffers(1, &ColorRenderBuffer);
    Driver->extGlBindRenderbuffer(GL_RENDERBUFFER_EXT, ColorRenderBuffer);
    bindRTT();
    Driver->extGlRenderbufferStorageMultisample(GL_RENDERBUFFER_EXT, Samples, InternalFormat,  ImageSize.Width, ImageSize.Height);
 
#ifdef _DEBUG
    driver->testGLError();
#endif
 
    // attach color renderbuffer to frame buffer
    Driver->extGlFramebufferRenderbuffer(GL_FRAMEBUFFER_EXT,
                    GL_COLOR_ATTACHMENT0_EXT,
                    GL_RENDERBUFFER_EXT,
                    ColorRenderBuffer);
 
 
#ifdef _DEBUG
    checkFBOStatus(Driver);
#endif
    }
    else
#endif
    {
#ifdef GL_EXT_framebuffer_object
    // generate frame buffer
    Driver->extGlGenFramebuffers(1, &ColorFrameBuffer);
    bindRTT();
 

same for COpenGLFBODepthTexture constructor

Code: Select all

#ifdef GL_EXT_framebuffer_object
    else
    {
        // generate depth buffer
        Driver->extGlGenRenderbuffers(1, &DepthRenderBuffer);
        Driver->extGlBindRenderbuffer(GL_RENDERBUFFER_EXT, DepthRenderBuffer);
 
#if defined(GL_ARB_framebuffer_object) || defined(GL_EXT_framebuffer_multisample)
        if (Samples > 0)
            Driver->extGlRenderbufferStorageMultisample(GL_RENDERBUFFER_EXT, Samples, Driver->getZBufferBits(),  ImageSize.Width, ImageSize.Height);
        else
#endif
        Driver->extGlRenderbufferStorage(GL_RENDERBUFFER_EXT,
                Driver->getZBufferBits(), ImageSize.Width,
                ImageSize.Height);
    }
#endif

To draw the MSAA texture you have to resolve it into a non MSAA texture like this (**I think**, we only did this for borderless window on linux not random size render textures. could handle this in the texture itself by keeping two textures or two textures inside the COpenGLDriver)

Code: Select all

        // Resolve RenderTarget into FinalTexture
        extGlBindFramebuffer(GL_DRAW_FRAMEBUFFER_EXT, AAFinalTexture->ColorFrameBuffer);
        extGlBindFramebuffer(GL_READ_FRAMEBUFFER_EXT, AARenderTarget->ColorFrameBuffer); 
        glDrawBuffer(GL_COLOR_ATTACHMENT0_EXT);
        extGLBlitFramebuffer(0, 0, AARenderTarget->getSize().Width, AARenderTarget->getSize().Height, 0, 0, AAFinalTexture->getSize().Width, AAFinalTexture->getSize().Height, GL_COLOR_BUFFER_BIT, GL_NEAREST);
 
        // Draw Final Texture to screen buffer
        extGlBindFramebuffer(GL_DRAW_FRAMEBUFFER_EXT, 0);  
        extGlBindFramebuffer(GL_READ_FRAMEBUFFER_EXT, AAFinalTexture->ColorFrameBuffer);
        glDrawBuffer(GL_BACK);
        extGLBlitFramebuffer(0, 0, AAFinalTexture->getSize().Width, AAFinalTexture->getSize().Height, 0, 0, ScreenSize.Width, ScreenSize.Height, GL_COLOR_BUFFER_BIT, GL_NEAREST);
        extGlBindFramebuffer( GL_FRAMEBUFFER_EXT, 0 );
        glViewport(0,0,ScreenSize.Width,ScreenSize.Height);

[IrrDeveloper]

devonsoft,

Great! I can confirm this works.

One thing though: in draw2DImage, no need to limit StretchRect to the backbuffer, you can copy from the MSAA texture to a non-MSAA target as well. So no real need to query what the current backbuffer is, just let the user set their other/new rendertarget and call draw2DImage.

Now checking if I can get the OpenGL version to work