Augmented Reality with ARToolKit

[FreakNigh]

This has become IrrAR... please goto this thread instead - http://irrlicht.sourceforge.net/phpBB2/ ... hp?t=30000

Here is some code that brings together the libs ARToolKit and Irrlicht... (I still dont have full control over this code you could say, but it is working - and not 100%...) I will be making this into IrrAR when the time comes. If anyone wants to help =)

See the Youtube: http://www.youtube.com/watch?v=6jKCFBVLFxE



Code:

Code: Select all

#include <irrlicht/irrlicht.h>
#include <AR/video.h>
#include <AR/param.h>
#include <AR/ar.h>
 
using namespace irr;

using namespace core;
using namespace scene;
using namespace video;
using namespace io;
using namespace gui;

char		*vconf = "";

int             xsize, ysize;
int             thresh = 100;
int             count = 0;

char           *cparam_name    = "ardata/camera_para.dat";
ARParam         cparam;

char           *patt_name      = "ardata/patt.hiro";
int             patt_id;
double          patt_width     = 80.0;
double          patt_center[2] = {0.0, 0.0};
double          patt_trans[3][4];

IrrlichtDevice *device;
IVideoDriver* driver;
ISceneManager* smgr;

void initAR();
ITexture* create_ITexture_from_ARimage(ARUint8 *ardata, int width, int height);
ITexture* update_ITexture_from_ARimage(ITexture *dest, ARUint8 *ardata, int width, int height);
void AttachNode(ISceneNode *thenode, ARUint8 *dataPtr);
void convert_trans_para( double para[3][4], double gl_para[16] );

//....

#define   MINIWIN_MAX    8
#define   GMINI          2

static int      gl_hmd_flag      = 0;
static double   gZoom;
static int      gImXsize, gImYsize;
static int      gXsize, gYsize;
static int      gMiniXsize, gMiniYsize;
static int      gMiniXnum,  gMiniYnum;
static int      gWinXsize, gWinYsize;
static ARParam  gCparam;
static float   gl_cpara[16];

void argConvGLcpara( ARParam *param, double gnear, double gfar, float m[16] );
static void argConvGLcpara2( double cparam[3][4], int width, int height, double gnear, double gfar, float m[16] );
void init_camera( ARParam *cparam, double zoom, int fullFlag, int xwin, int ywin, int hmd_flag );

class MyEventReceiver : public IEventReceiver
{
	public:
		virtual bool OnEvent(const SEvent &event)
		{ 
			if (event.EventType == EET_KEY_INPUT_EVENT && !event.KeyInput.PressedDown)
			{
				vector3df cur_travel;
				vector2df cur_win_size;
				
				switch(event.KeyInput.Key)
				{
					case KEY_ESCAPE:
						device->closeDevice();
						return true;
				}
			}
			return false;
		}
};

int main()
{
	MyEventReceiver receiver;
	
	//init code ripped from ARToolkit's simpleTest.c
	initAR();
	
	device = createDevice( video::EDT_OPENGL, dimension2d<s32>(352, 288), 16, false, false, false, &receiver);
	
	device->setWindowCaption(L"Hello World! - Irrlicht Engine Demo");
	
	driver = device->getVideoDriver();
	smgr = device->getSceneManager();
	
	IAnimatedMesh* mesh = smgr->getMesh("media/sydney.md2");
	IAnimatedMeshSceneNode* node = smgr->addAnimatedMeshSceneNode( mesh );
	if (node)
	{
		node->setMaterialFlag(EMF_LIGHTING, false);
		node->setMD2Animation ( scene::EMAT_STAND );
		node->setMaterialTexture( 0, driver->getTexture("media/sydney.bmp") );
		node->setScale(vector3df(3,3,3));
	}
	
	ISceneNode* node2 = smgr->addCubeSceneNode(50);
	node2->setPosition(vector3df(0,50,0));
	node2->setMaterialFlag(EMF_LIGHTING, false);
	node2->setMaterialTexture( 0, driver->getTexture("media/sydney.bmp") );
	//node2->setParent(node);
	
		//and some more
	/*
	device->getFileSystem()->addZipFileArchive("media/map-20kdm2.pk3");
	IAnimatedMesh* mesh3 = smgr->getMesh("20kdm2.bsp");
	ISceneNode* node3 = 0;
	if (mesh3)
		node3 = smgr->addOctTreeSceneNode(mesh3->getMesh(0), 0, -1, 128);
	if (node3)
		node3->setPosition(vector3df(-1300,-144,-1249));
	*/
	
	matrix4 projection_matrix;
	vector3df camera_pos = vector3df(0,0,0);
	vector3df camera_target = vector3df(0,0,-10);
	ICameraSceneNode* camera = smgr->addCameraSceneNode(0, camera_pos, camera_target);
	//ICameraSceneNode* camera = smgr->addCameraSceneNodeFPS();
	//gl_cpara[8] = -gl_cpara[8];
	//gl_cpara[9] = -gl_cpara[9];
	//gl_cpara[10] = -gl_cpara[10];
	//gl_cpara[11] = -gl_cpara[11];
	//gl_cpara[14] = -gl_cpara[14];
	projection_matrix.setM(gl_cpara);
	camera->setProjectionMatrix(projection_matrix);
	printf("up vector:%f %f %f\n", camera->getUpVector().X, camera->getUpVector().Y, camera->getUpVector().Z);
	//camera->setUpVector(vector3df(0,-1,0));
	//printf("[%f %f %f %f]\n", gl_cpara[0], gl_cpara[1], gl_cpara[2], gl_cpara[3]);
	//printf("[%f %f %f %f]\n", gl_cpara[4], gl_cpara[5], gl_cpara[6], gl_cpara[7]);
	//printf("[%f %f %f %f]\n", gl_cpara[8], gl_cpara[9], gl_cpara[10], gl_cpara[11]);
	//printf("[%f %f %f %f]\n", gl_cpara[12], gl_cpara[13], gl_cpara[14], gl_cpara[15]);
	
	int lastFPS = -1;
	ARUint8         *dataPtr;
		
	if( (dataPtr = (ARUint8 *)arVideoGetImage()) == NULL ) 
		printf("no image loaded\n");
	
	ITexture* ARimage = create_ITexture_from_ARimage(dataPtr, xsize, ysize);
	
	//our_bill_node->setMaterialTexture( 0, ARimage );
	
	while(device->run())
	{
		if(dataPtr = (ARUint8 *)arVideoGetImage()) 
			update_ITexture_from_ARimage(ARimage, dataPtr, xsize, ysize);
		
		driver->beginScene(true, true, SColor(255,100,101,140));

		driver->draw2DImage(ARimage, position2d<s32>(0,0));
		
		AttachNode(node2, dataPtr);
		
		smgr->drawAll();
		//guienv->drawAll();

		driver->endScene();
		
		int fps = driver->getFPS();

		if (lastFPS != fps)
		{
			core::stringw str = L"Irrlicht Engine [";
			str += driver->getName();
			str += "] FPS:";
			str += fps;

			device->setWindowCaption(str.c_str());
			lastFPS = fps;
		}
	}
	
	device->drop();

	return 0;
}

ITexture* update_ITexture_from_ARimage(ITexture *dest, ARUint8 *ardata, int width, int height)
{
	u8* pixels;
	pixels = (u8*)(dest->lock());
	if( pixels )
	{
		int max_pixels = width * height;
		
		for(int i=0;i<max_pixels;i++)
		{
			*pixels = *ardata;
			pixels++; ardata++;
			*pixels = *ardata;
			pixels++; ardata++;
			*pixels = *ardata;
			pixels++; ardata++;
			
			pixels++;
		}
		
		dest->unlock();
	}
	else
	{
		printf("some hellish error\n");
	}

	return dest;
}

ITexture* create_ITexture_from_ARimage(ARUint8 *ardata, int width, int height)
{

	ITexture* m_poTileTexture = driver->addTexture(core::dimension2d<s32>(width, height), "TileTex", video::ECF_A1R5G5B5);

	ITexture* m_poTileNormalMap = driver->addTexture(core::dimension2d<s32>(width, height), "NormTex", video::ECF_A1R5G5B5); 
	
	   // read the pixels directly into the texture
	u8* pixels;
	pixels = (u8*)(m_poTileTexture->lock());
	if( pixels )
	{
		int max_pixels = width * height;
		
		for(int i=0;i<max_pixels;i++)
		{
			*pixels = *ardata;
			pixels++; ardata++;
			*pixels = *ardata;
			pixels++; ardata++;
			*pixels = *ardata;
			pixels++; ardata++;
			
			pixels++;
		}
		
		m_poTileTexture->unlock();
	}
	else
	{
		printf("some hellish error\n");
	}

	return m_poTileTexture;
}

void initAR()
{
	//nearly verbatum rip from ARToolKit's simpleTest.c, minus the "g" stuff that is used for integrating with glut
	ARParam  wparam;
	
	/* open the video path */
	if( arVideoOpen( vconf ) < 0 ) exit(0);
	/* find the size of the window */
	if( arVideoInqSize(&xsize, &ysize) < 0 ) exit(0);
	printf("Image size (x,y) = (%d,%d)\n", xsize, ysize);

	/* set the initial camera parameters */
	if( arParamLoad(cparam_name, 1, &wparam) < 0 ) {
		printf("Camera parameter load error !!\n");
		exit(0);
	}
	arParamChangeSize( &wparam, xsize, ysize, &cparam );
	arInitCparam( &cparam );
	printf("*** Camera Parameter ***\n");
	arParamDisp( &cparam );

	if( (patt_id=arLoadPatt(patt_name)) < 0 ) {
		printf("pattern load error !!\n");
		exit(0);
	}
	
	init_camera( &cparam, 1.0, 0, 0, 0, 0 );
	
	arVideoCapStart();
}

void AttachNode(ISceneNode *thenode, ARUint8 *dataPtr)
{
	ARMarkerInfo    *marker_info;
	int             marker_num;
	int             j, k;
	double gl_para[16];
	float glf_para[16];
	int i;
	CMatrix4<float> mat;

	arDetectMarker(dataPtr, thresh, &marker_info, &marker_num);

	arVideoCapNext();

	/* check for object visibility */
	k = -1;
	for( j = 0; j < marker_num; j++ ) {
		if( patt_id == marker_info[j].id ) {
			if( k == -1 ) k = j;
			else if( marker_info[k].cf < marker_info[j].cf ) k = j;
		}
	}
	if( k == -1 ) {
		return;
	}

	/* get the transformation between the marker and the real camera */
	arGetTransMat(&marker_info[k], patt_center, patt_width, patt_trans);
	
	convert_trans_para(patt_trans, gl_para);
	for(i=0;i<16;i++) glf_para[i] = (float)gl_para[i];
	mat.setM(glf_para);
	
	vector3d<f32> scale_vec = mat.getScale();
	vector3d<f32> rot_vec = mat.getRotationDegrees();
	vector3d<f32> pos_vec = mat.getTranslation();
	
	pos_vec.X = -pos_vec.X;

	rot_vec.X += 90;
	rot_vec.Y = -rot_vec.Y;
	rot_vec.Z = -rot_vec.Z;
	
	//scale_vec.X = 2;
	//scale_vec.Y = 2;
	//scale_vec.Z = 2;
	
	//driver->setTransform(video::ETS_PROJECTION, gl_para);
	//printf("whoa man\n");
	printf("scale=<%f %f %f>\n", scale_vec.X, scale_vec.Y, scale_vec.Z);
	//printf("rot=<%f %f %f>\n", rot_vec.X, rot_vec.Y, rot_vec.Z);
	//printf("pos=<%f %f %f>\n", pos_vec.X, pos_vec.Y, pos_vec.Z);
	//printf("[%f %f %f %f]\n", gl_para[0], gl_para[1], gl_para[2], gl_para[3]);
	//printf("[%f %f %f %f]\n", gl_para[4], gl_para[5], gl_para[6], gl_para[7]);
	//printf("[%f %f %f %f]\n", gl_para[8], gl_para[9], gl_para[10], gl_para[11]);
	//printf("[%f %f %f %f]\n", gl_para[12], gl_para[13], gl_para[14], gl_para[15]);
	thenode->setScale(scale_vec);
	thenode->setRotation(rot_vec);
	thenode->setPosition(pos_vec + vector3df(0,-25,0));
	
	//driver->setTransform(ETS_WORLD, mat);
}

void convert_trans_para( double para[3][4], double gl_para[16] )
{
	int     i, j;

	for( j = 0; j < 3; j++ ) {
		for( i = 0; i < 4; i++ ) {
			gl_para[i*4+j] = para[j][i];
		}
	}
	gl_para[0*4+3] = gl_para[1*4+3] = gl_para[2*4+3] = 0.0;
	gl_para[3*4+3] = 1.0;
}


void init_camera( ARParam *cparam, double zoom, int fullFlag, int xwin, int ywin, int hmd_flag )
{
	int       i;

	gl_hmd_flag = hmd_flag;
	gZoom  = zoom;
	gImXsize = cparam->xsize;
	gImYsize = cparam->ysize;
	if( gl_hmd_flag == 0 ) {
		gXsize = (int)((double)cparam->xsize * gZoom);
		gYsize = (int)((double)cparam->ysize * gZoom);
	}
	else {
		gXsize = AR_HMD_XSIZE;
		gYsize = AR_HMD_YSIZE;
	}
	gMiniXsize = (int)((double)cparam->xsize * gZoom / GMINI);
	gMiniYsize = (int)((double)cparam->ysize * gZoom / GMINI);

	if( xwin * ywin > MINIWIN_MAX ) {
		if( xwin > MINIWIN_MAX ) xwin = MINIWIN_MAX;
		ywin = MINIWIN_MAX / xwin;
	}
	gMiniXnum = xwin;
	gMiniYnum = ywin;
	gWinXsize = (gMiniXsize*gMiniXnum > gXsize)?
			gMiniXsize*gMiniXnum: gXsize;
	gWinYsize = gYsize + gMiniYsize*gMiniYnum;

	gCparam = *cparam;
	for( i = 0; i < 4; i++ ) {
		gCparam.mat[1][i] = (gCparam.ysize-1)*(gCparam.mat[2][i]) - gCparam.mat[1][i];
	}
	argConvGLcpara( &gCparam, AR_GL_CLIP_NEAR, AR_GL_CLIP_FAR, gl_cpara );

	//argInit2( fullFlag );
}

void argConvGLcpara( ARParam *param, double gnear, double gfar, float m[16] )
{
	argConvGLcpara2( param->mat, param->xsize, param->ysize, gnear, gfar, m );
}

static void argConvGLcpara2( double cparam[3][4], int width, int height, double gnear, double gfar, float m[16] )
{
	double   icpara[3][4];
	double   trans[3][4];
	double   p[3][3], q[4][4];
	int      i, j;
	

	if( arParamDecompMat(cparam, icpara, trans) < 0 ) {
		printf("gConvGLcpara: Parameter error!!\n");
		exit(0);
	}

	for( i = 0; i < 3; i++ ) {
		for( j = 0; j < 3; j++ ) {
			p[i][j] = icpara[i][j] / icpara[2][2];
		}
	}
	m[0] = q[0][0] = (2.0 * p[0][0] / width);
	m[1] = q[0][1] = (2.0 * p[0][1] / width);
	m[2] = q[0][2] = ((2.0 * p[0][2] / width)  - 1.0);
	m[3] = q[0][3] = 0.0;

	m[4] = q[1][0] = 0.0;
	m[5] = q[1][1] = (2.0 * p[1][1] / height);
	m[6] = q[1][2] = ((2.0 * p[1][2] / height) - 1.0);
	m[7] = q[1][3] = 0.0;

	m[8] = q[2][0] = 0.0;
	m[9] = q[2][1] = 0.0;
	m[10] = q[2][2] = -(gfar + gnear)/(gfar - gnear);
	m[11] = q[2][3] = -2.0 * gfar * gnear / (gfar - gnear);

	m[12] = q[3][0] = 0.0;
	m[13] = q[3][1] = 0.0;
	m[14] = q[3][2] = -1.0;
	m[15] = q[3][3] = 0.0;
	
	for( i = 0; i < 4; i++ ) {
		for( j = 0; j < 3; j++ ) {
			m[i+j*4] = q[i][0] * trans[0][j] 
					+ q[i][1] * trans[1][j]
					+ q[i][2] * trans[2][j];
		}
		m[i+3*4] = q[i][0] * trans[0][3]
				+ q[i][1] * trans[1][3]
				+ q[i][2] * trans[2][3]
				+ q[i][3];
	}
}

/*
static void argConvGLcpara2( double cparam[3][4], int width, int height, double gnear, double gfar, float m[16] )
{
	double   icpara[3][4];
	double   trans[3][4];
	double   p[3][3], q[4][4];
	int      i, j;
	

	if( arParamDecompMat(cparam, icpara, trans) < 0 ) {
		printf("gConvGLcpara: Parameter error!!\n");
		exit(0);
}

	for( i = 0; i < 3; i++ ) {
		for( j = 0; j < 3; j++ ) {
			p[i][j] = icpara[i][j] / icpara[2][2];
}
}
	q[0][0] = (2.0 * p[0][0] / width);
	q[0][1] = (2.0 * p[0][1] / width);
	q[0][2] = ((2.0 * p[0][2] / width)  - 1.0);
	q[0][3] = 0.0;

	q[1][0] = 0.0;
	q[1][1] = (2.0 * p[1][1] / height);
	q[1][2] = ((2.0 * p[1][2] / height) - 1.0);
	q[1][3] = 0.0;

	q[2][0] = 0.0;
	q[2][1] = 0.0;
	q[2][2] = (gfar + gnear)/(gfar - gnear);
	q[2][3] = -2.0 * gfar * gnear / (gfar - gnear);

	q[3][0] = 0.0;
	q[3][1] = 0.0;
	q[3][2] = 1.0;
	q[3][3] = 0.0;

	for( i = 0; i < 4; i++ ) {
		for( j = 0; j < 3; j++ ) {
			m[i+j*4] = q[i][0] * trans[0][j]
					+ q[i][1] * trans[1][j]
					+ q[i][2] * trans[2][j];
}
		m[i+3*4] = q[i][0] * trans[0][3]
				+ q[i][1] * trans[1][3]
				+ q[i][2] * trans[2][3]
				+ q[i][3];
}
}
*/

[JP]

Shame the FPS is so low and that sydney bobs around so much but it's pretty cool when you turn the bit of paper round!

[BlindSide]

Been there, done that

http://irrlichtirc.g0dsoft.com/BlindSide/rotanim.JPG

Jk, good job

What kind of problems are you experiencing?

[FreakNigh]

Ya JP the effect isn't perfect but I'm pretty sure it'd be alot better because right now I got the position and rotation right but not the scale of the translation matrix. Also for the FPS you might be able to get away with putting the AR part of the code on another thread (I did that with "my" head tracking code with good success..)

Blidside.. I got the camera projection matrix in (as far as I know its right, it sure is wacked because it inverts all the axis's), Aside from the messed up projection matrix, I've converted the pos and rot parts of the translation matrix for the model but I can't figure out the scale. I got some ideas on how to fix it right but I'm leaving for a 10 day vacation tomorrow so..

[BlindSide]

Beware that ARToolkit is GPL... You should release separatwra packages for the OpenCV and ARToolkit wrappers so that people can use the first one freely.

[FreakNigh]

Oh ya I'm not mixing the projects.. btw I found out I don't need the scale and the problem I had with just rotation + position was something I added in myself. My CvIrrCamController project is slowing down because we have now moved out of the building phase and into the perfecting the effect phase. So when I get back from vacation look for IrrAR =)

[godmode2k]

great job FreakNigh !
i've pretty impressed your AR job.
i had had tested AR only by ARToolkit.
but i haven't to try apply AR with Irrlicht.
you did.

great job again.