Play avi's on BillBoardNode w/ help of opencv lib

[FreakNigh]

here is a full main.cpp to play two avi's at the same time with the help of opencv. Also good for learning how to convert opencv's IplImage to an ITexture.

To see this code running on youtube.. http://www.youtube.com/watch?v=5vvp-08yegI

Code: Select all

#include <irrlicht/irrlicht.h>
#include <opencv/cv.h>
#include <opencv/cxcore.h>
#include <opencv/highgui.h>
#include <time.h>
#include <sys/timeb.h>
 
using namespace irr;

using namespace core;
using namespace scene;
using namespace video;
using namespace io;
using namespace gui;

IrrlichtDevice *device;
IVideoDriver* driver;
ISceneManager* smgr;

ITexture* create_ITexture_from_CvCapture(CvCapture* capture);
ITexture* update_ITexture_from_CvCapture(ITexture *dest, CvCapture* capture, double time_since_last_update, double *over_flow_time);
double current_time();

int main()
{
	device = createDevice( video::EDT_OPENGL, dimension2d<s32>(640, 480), 16, false, false, false, 0);
	driver = device->getVideoDriver();
	smgr = device->getSceneManager();
   
	device->setWindowCaption(L"Hello World! - Irrlicht Engine Demo");
   
	//something for 3d reference
	IAnimatedMesh* mesh = smgr->getMesh("media/sydney.md2");
	IAnimatedMeshSceneNode* node = smgr->addAnimatedMeshSceneNode( mesh );
	if (node)
	{
		node->setMaterialFlag(EMF_LIGHTING, false);
		node->setMD2Animation ( scene::EMAT_STAND );
		node->setMaterialTexture( 0, driver->getTexture("media/sydney.bmp") );
	}
	
	//our billboard
	IBillboardSceneNode* our_bill_node = smgr->addBillboardSceneNode(NULL, dimension2d<f32>(25.0f, 25.0f), vector3df(-30,0,0));
	our_bill_node->setMaterialFlag(video::EMF_LIGHTING, false);
	//#2
	IBillboardSceneNode* our_bill_node2 = smgr->addBillboardSceneNode(NULL, dimension2d<f32>(25.0f, 25.0f), vector3df(30,20,30));
	our_bill_node2->setMaterialFlag(video::EMF_LIGHTING, false);
   
	//camera stuff
	ICameraSceneNode* camera = smgr->addCameraSceneNodeFPS();
	camera->setPosition(vector3df(60,0,0));
	camera->setTarget(vector3df(0,0,0));
   
	//choose your own avi's
	//refer to this if needed to convert your vid to cv usable...
	//mencoder in.avi -ovc raw -vf format=i420 -o out.avi
	CvCapture* capture = cvCaptureFromFile("verona60avi56k.avi");
	CvCapture* capture2 = cvCaptureFromFile("out.avi");
	
	if (!capture)
		printf("Could not initialize capturing1...\n");
	if (!capture2)
		printf("Could not initialize capturing2...\n");

	//get the initial image
	ITexture* CvImage = create_ITexture_from_CvCapture(capture);
	ITexture* CvImage2 = create_ITexture_from_CvCapture(capture2);
	our_bill_node->setMaterialTexture( 0, CvImage );
	our_bill_node2->setMaterialTexture( 0, CvImage2 );
	
	//fps stuff with a bit more precision
	double last_time = current_time();
	double add_on_time = 0, add_on_time2 = 0;

	while(device->run())
	{
		driver->beginScene(true, true, SColor(255,100,101,140));

		smgr->drawAll();

		driver->endScene();
		
		update_ITexture_from_CvCapture(CvImage, capture, current_time() - last_time, &add_on_time);
		update_ITexture_from_CvCapture(CvImage2, capture2, current_time() - last_time, &add_on_time2);
		last_time = current_time();
	}
   
	device->drop();

	return 0;
}

ITexture* update_ITexture_from_CvCapture(ITexture *dest, CvCapture* capture, double time_since_last_update, double *over_flow_time)
{
	//get these variables about the video
	int frameH    = (int) cvGetCaptureProperty(capture, CV_CAP_PROP_FRAME_HEIGHT);
	int frameW    = (int) cvGetCaptureProperty(capture, CV_CAP_PROP_FRAME_WIDTH);
	int fps       = (int) cvGetCaptureProperty(capture, CV_CAP_PROP_FPS);

	//alittle code to try and keep it all at the right speed
	int frames_to_capture = (int)(fps * (time_since_last_update + *over_flow_time));
	if(frames_to_capture)
	{
		*over_flow_time = time_since_last_update - (frames_to_capture / fps * 1.0);
	}
	else
	{
		*over_flow_time = *over_flow_time + time_since_last_update;
		return dest;
	}
	
	
	//grab an image and potentially skip some frames
	IplImage* img = 0;
	for(int i=0;i<frames_to_capture; i++)
	{
		if(!cvGrabFrame(capture))
		{
			printf("Could not grab a frame\n");
			return dest;
		}


	}
	img = cvRetrieveFrame(capture);

	//now put the img data into the texture data
	u8* pixels = (u8*)(dest->lock());
	u8* ardata = (u8*)img->imageData;
	int max_pixels = frameH * frameW;

	for(int i=0;i<max_pixels;i++)
	{
		*pixels = *ardata;
		pixels++; ardata++;
		*pixels = *ardata;
		pixels++; ardata++;
		*pixels = *ardata;
		pixels++; ardata++;
	
		pixels++;
	}

	dest->unlock();

	return dest;
}

ITexture* create_ITexture_from_CvCapture(CvCapture* capture)
{
	char unique_tex_name[50];
	
	//for irrlicht
	sprintf(unique_tex_name, "cv_image:%d", capture);
	
	//grab an image
	IplImage* img = 0; 
	if(!cvGrabFrame(capture))
	{
		printf("Could not grab a frame\n");
		return 0;
	}
	img=cvRetrieveFrame(capture);
	
	//get these variables about the video
	int frameH    = (int) cvGetCaptureProperty(capture, CV_CAP_PROP_FRAME_HEIGHT);
	int frameW    = (int) cvGetCaptureProperty(capture, CV_CAP_PROP_FRAME_WIDTH);
	int fps       = (int) cvGetCaptureProperty(capture, CV_CAP_PROP_FPS);
	
	//make our texture
	ITexture* m_poTileTexture = driver->addTexture(core::dimension2d<s32>(frameW, frameH), unique_tex_name, video::ECF_A1R5G5B5);
   
	// read the pixels directly into the texture
	u8* pixels = (u8*)(m_poTileTexture->lock());
	u8* ardata = (u8*)img->imageData;
	int max_pixels = frameW * frameH;

	for(int i=0;i<max_pixels;i++)
	{
		*pixels = *ardata;
		pixels++; ardata++;
		*pixels = *ardata;
		pixels++; ardata++;
		*pixels = *ardata;
		pixels++; ardata++;
	
		pixels++;
	}

	m_poTileTexture->unlock();

	return m_poTileTexture;
} 

//this function brings back seconds with milliseconds since it was first called
double current_time()
{
	static int first_sec = 0;
	static int first_msec = 0;
	struct timeb new_time;
	
	//set current time
	ftime(&new_time);

	//set if not set
	if(!first_sec)
	{
		first_sec = new_time.time;
		first_msec = new_time.millitm;
	}
	
	return (new_time.time - first_sec) + ((new_time.millitm - first_msec) * 0.001);
}

[hybrid]

I didn't test the code, yet, but it seems wrong. You're creating an A1R5G5B5 texture, but you access it as 32bits. You should always check which format the texture has, and according to this format add the proper conversions. Since the conversions are not yet exposed, you might have to copy some parts from CColorConversion for this to work properly.
Another good idea is to create a non-alpha texture, in order to avoid the extra alpha byte which seems to be unsupported by OpenCV. This woudl allow for a very fast memcpy if both libraries use the same color encoding.

[FreakNigh]

you dont have to look at the code to see if it works or not, check out the youtube video.

I really have no clue about A1R5G5B5, but it definitely is in rgba format or 32bit, and the opencv images are 24bit or rgb which is why when copying the data over I need to skip the last 8 bits of the irrlicht's image pixel data.

This code isn't ment to be a final product but the idea of converting both to 24bit or both to 32bit so that you can use memcpy is smart as far as i know.

What would be even faster of course is recode opencv alittle bit so that it just writes right into an ITexture.

-Or there might be a way to have the ITexture and opencv Image structure share the same data array... ? I dunno I just hacked this up in a few hours for work and thought I'd share it.

[Acki]

hmm, I didn't check this, but would it not be better to create a texture with 24bit RGB format (ECF_R8G8B8) ???
and maybe then you can just do a memcpy or even share the same datas...

[FreakNigh]

Thanks Acki. We actually do need the alpha channel for how we're using it...

[hybrid]

Yes, the code works in this situation, because the driver enables ALWAYS_32_BIT by default, and the OpenGL driver creates an A8R8G8B8 even if you desire an A1R5G5B5. However, once a different texture creation flag is chosen it won't work. I'd suggest to change the format of the texture to A8R8G8B8 and change the texture creation flags to ALWAYS_32_BIT right before creating the texture. This will definitely fix the issues, and since you say that you need the 32bit textures there's also no unnecessary memory usage involved.

[FreakNigh]

is this better?

CvIrrCamTexture.h

Code: Select all

#include <opencv/cv.h>
#include <opencv/highgui.h>
#include <irrlicht/irrlicht.h>
#include <math.h>

//I'm lazy
using namespace irr;

using namespace core;
using namespace scene;
using namespace video;
using namespace io;
using namespace gui;

class CvIrrCamTexture
{
	public:
		CvIrrCamTexture(IVideoDriver* driver, int cvflags = 0, int use_rgba_texture = 0);
		~CvIrrCamTexture();

		ITexture* getTexture();
		int UpdateTexture();
	private:
		//needed to manipulate irr textures
		IVideoDriver* driver;
		
		//our texture
		ITexture* irr_texture;
		
		//cv capture
		CvCapture* cv_capture;
		
		void copy_over_image(IplImage* cv_img);
};

CvIrrCamTexture.cpp

Code: Select all

#include "CvIrrCamTexture.h"

CvIrrCamTexture::CvIrrCamTexture(IVideoDriver* driver, int cvflags, int use_rgba_texture)
{
	char unique_tex_name[50];
	IplImage* cv_img = 0;
	
	//set this
	this->driver = driver;
	
	//start the capture
	this->cv_capture = cvCaptureFromCAM(cvflags);
	if(!this->cv_capture) return;
	
	//grab an image
	cv_img = cvQueryFrame(this->cv_capture);
	if(!cv_img) return;
	
	//for irrlicht scheme
	sprintf(unique_tex_name, "cv_image_CAM:%d", this);
	
	//make the texture
	if(use_rgba_texture)
		this->irr_texture = this->driver->addTexture(core::dimension2d<s32>(cv_img->width, cv_img->height), unique_tex_name, video::ECF_A8R8G8B8);
	else
		this->irr_texture = this->driver->addTexture(core::dimension2d<s32>(cv_img->width, cv_img->height), unique_tex_name, video::ECF_R8G8B8);
	
	//finally
	this->copy_over_image(cv_img);
}

CvIrrCamTexture::~CvIrrCamTexture()
{
	
}

ITexture* CvIrrCamTexture::getTexture()
{
	return this->irr_texture;
}

int CvIrrCamTexture::UpdateTexture()
{
	IplImage* cv_img = 0;
	
	//grab an image
	cv_img = cvQueryFrame(this->cv_capture);
	if(!cv_img) return 0;
	
	this->copy_over_image(cv_img);
}

void CvIrrCamTexture::copy_over_image(IplImage* cv_img)
{
	// read the pixels directly into the texture
	char* pixels = (char*)(this->irr_texture->lock());
	char* ardata = (char*)cv_img->imageData;
	char* final_loc = cv_img->imageSize + cv_img->imageData;

	switch(this->irr_texture->getColorFormat())
	{
		case ECF_R8G8B8:
			while(ardata < final_loc)
			{
				*pixels = *ardata;
				pixels++; ardata++;
				*pixels = *ardata;
				pixels++; ardata++;
				*pixels = *ardata;
				pixels++; ardata++;

			}
			break;
			
		case ECF_A8R8G8B8:
			while(ardata < final_loc)
			{
				*pixels = *ardata;
				pixels++; ardata++;
				*pixels = *ardata;
				pixels++; ardata++;
				*pixels = *ardata;
				pixels++; ardata++;
				
				pixels++;
			}
			break;
	}


	this->irr_texture->unlock();
}

[FreakNigh]

Example using it:

Code: Select all

#include <irrlicht/irrlicht.h>
#include <opencv/cv.h>
#include <opencv/highgui.h>

#include "CvIrrCamTexture.h"

using namespace irr;

using namespace core;
using namespace scene;
using namespace video;
using namespace io;
using namespace gui;

IrrlichtDevice *device;
IVideoDriver* driver;
ISceneManager* smgr;

int main()
{
	device = createDevice( video::EDT_OPENGL, dimension2d<s32>(640, 480), 16, false, false, false, 0);
	driver = device->getVideoDriver();
	smgr = device->getSceneManager();
   
	device->setWindowCaption(L"Hello World! - Irrlicht Engine Demo");
   
	//something for 3d reference
	IAnimatedMesh* mesh = smgr->getMesh("media/sydney.md2");
	IAnimatedMeshSceneNode* node = smgr->addAnimatedMeshSceneNode(mesh);
	if (node)
	{
		node->setMaterialFlag(EMF_LIGHTING, false);
		node->setMD2Animation ( scene::EMAT_STAND );
		node->setMaterialTexture( 0, driver->getTexture("media/sydney.bmp") );
		node->setPosition(vector3df(10,10,-10));
	}
	
	//and some more
	device->getFileSystem()->addZipFileArchive("media/map-20kdm2.pk3");
	IAnimatedMesh* mesh2 = smgr->getMesh("20kdm2.bsp");
	ISceneNode* node2 = 0;
	if (mesh2)
		node2 = smgr->addOctTreeSceneNode(mesh2->getMesh(0), 0, -1, 128);
	if (node2)
		node2->setPosition(vector3df(-1300,-144,-1249));
	
	//our billboard
	IBillboardSceneNode* our_bill_node = smgr->addBillboardSceneNode(NULL, dimension2d<f32>(25.0f, 25.0f), vector3df(0,0,0));
	our_bill_node->setMaterialFlag(video::EMF_LIGHTING, false);
	
	//camera stuff
	ICameraSceneNode* camera = smgr->addCameraSceneNodeFPS();
	camera->setPosition(vector3df(60,10,0));
	camera->setTarget(vector3df(0,0,0));
	
	//start the webcam
	CvIrrCamTexture* cv_text_mgr = new CvIrrCamTexture(driver,0,0);
	//set the texture
	our_bill_node->setMaterialTexture( 0, cv_text_mgr->getTexture() );

	while(device->run())
	{
		//update webcam texture
		cv_text_mgr->UpdateTexture();
		
		driver->beginScene(true, true, SColor(255,100,101,140));

		smgr->drawAll();

		driver->endScene();
	}
   
	device->drop();

	return 0;
}

[Gman3344]

you should try to make it so you can render the video to a render targer then it would be able to be textured on a mesh such as a tv screen.

[hybrid]

It is already a texture, so you can do so. Now

[andrei25ni]

hey guys, can you tell me what version of opencv to use ?
because I am getting the error

Code: Select all

  [Linker error] undefined reference to `cvCaptureFromFile' 

when trying to compile the first code posted by FreakNigh.
I am using WinXp with DevC++.

thanks.

[jeromegz]

hi all i am french, sorry for bad english:

i have tried with :
devcpp _latest version (4.9...)
irrlicht 1.3.1
opencv 1.0

work with it and i think that the code support an newer version of irrlicht but i have no try , but now we are at the version 6 of irrlicht , but irrlichr have a lot of change since version 1.3.1 and if anywone try , thanks.

i am "anywone" too , so, i start to try now.

[jawdy]

Sorry to add to an old post - but a couple of people have asked some questions about getting this to work with newer versions of Irr and CV... well folks, I've been working on this for a few days and have cracked it

The first thing is to make sure you've added the include and library paths to your project (in Visual Studio, use the main Tool->Options and add it to the list of resources in there, then they'll be available to every project. I'm not in front of my machine right now, but I'll write it up properly when I am!).
Another thing I've noticed with OpenCV is that you need to Link the libraries directly (again, in Visual Studio). So in the project properties, under linker, then "input" and add the URL to each library.

In order to make nigh's code work in the latest version(s) of Irr, you'll need to make a few changes to the code, namely anything that says <s32> for signed integers, you need to change to <u32>.

I apologise for this quick post, not giving enough instructions for setting up the project nor providing my code - I promise that as soon as I'm in front of my PC I'll put it up!
Oh, and I've also gotten it multi-threading, so the Irrlicht rendering loop operates at over 300fps, whilst the camera is doing it's capturing thing (I've linked it to a live feed of camera, rather than avi - it's part of cvIrrCaptureCam I think, but I've modified that)... it should all become clear when I put my code up!

[briaj]

I wasn't able to compile it. I try to compile it with eclipse and always appear errors in 2 classes of the opencv 2.0. I changed the s32 to u32 too.
I would like to know how can I compile it please.

Thank you very much in advance.